Linux Audio

Check our new training course

Linux BSP development engineering services

Need help to port Linux and bootloaders to your hardware?
Loading...
v6.8
   1/*
   2 * Copyright 2011 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24
  25#include <linux/firmware.h>
  26#include <linux/module.h>
  27#include <linux/pci.h>
  28#include <linux/slab.h>
  29
  30#include <drm/drm_vblank.h>
  31#include <drm/radeon_drm.h>
  32
  33#include "atom.h"
  34#include "clearstate_si.h"
  35#include "evergreen.h"
  36#include "r600.h"
  37#include "radeon.h"
  38#include "radeon_asic.h"
  39#include "radeon_audio.h"
  40#include "radeon_ucode.h"
  41#include "si_blit_shaders.h"
  42#include "si.h"
  43#include "sid.h"
 
 
 
 
  44
  45
  46MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
  47MODULE_FIRMWARE("radeon/TAHITI_me.bin");
  48MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
  49MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
  50MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
  51MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
  52MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
  53
  54MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
  55MODULE_FIRMWARE("radeon/tahiti_me.bin");
  56MODULE_FIRMWARE("radeon/tahiti_ce.bin");
  57MODULE_FIRMWARE("radeon/tahiti_mc.bin");
  58MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
  59MODULE_FIRMWARE("radeon/tahiti_smc.bin");
  60
  61MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
  62MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
  63MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
  64MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
  65MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
  66MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
  67MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
  68
  69MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
  70MODULE_FIRMWARE("radeon/pitcairn_me.bin");
  71MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
  72MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
  73MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
  74MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
  75MODULE_FIRMWARE("radeon/pitcairn_k_smc.bin");
  76
  77MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
  78MODULE_FIRMWARE("radeon/VERDE_me.bin");
  79MODULE_FIRMWARE("radeon/VERDE_ce.bin");
  80MODULE_FIRMWARE("radeon/VERDE_mc.bin");
  81MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
  82MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
  83MODULE_FIRMWARE("radeon/VERDE_smc.bin");
  84
  85MODULE_FIRMWARE("radeon/verde_pfp.bin");
  86MODULE_FIRMWARE("radeon/verde_me.bin");
  87MODULE_FIRMWARE("radeon/verde_ce.bin");
  88MODULE_FIRMWARE("radeon/verde_mc.bin");
  89MODULE_FIRMWARE("radeon/verde_rlc.bin");
  90MODULE_FIRMWARE("radeon/verde_smc.bin");
  91MODULE_FIRMWARE("radeon/verde_k_smc.bin");
  92
  93MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
  94MODULE_FIRMWARE("radeon/OLAND_me.bin");
  95MODULE_FIRMWARE("radeon/OLAND_ce.bin");
  96MODULE_FIRMWARE("radeon/OLAND_mc.bin");
  97MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
  98MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
  99MODULE_FIRMWARE("radeon/OLAND_smc.bin");
 100
 101MODULE_FIRMWARE("radeon/oland_pfp.bin");
 102MODULE_FIRMWARE("radeon/oland_me.bin");
 103MODULE_FIRMWARE("radeon/oland_ce.bin");
 104MODULE_FIRMWARE("radeon/oland_mc.bin");
 105MODULE_FIRMWARE("radeon/oland_rlc.bin");
 106MODULE_FIRMWARE("radeon/oland_smc.bin");
 107MODULE_FIRMWARE("radeon/oland_k_smc.bin");
 108
 109MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
 110MODULE_FIRMWARE("radeon/HAINAN_me.bin");
 111MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
 112MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
 113MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
 114MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
 115MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
 116
 117MODULE_FIRMWARE("radeon/hainan_pfp.bin");
 118MODULE_FIRMWARE("radeon/hainan_me.bin");
 119MODULE_FIRMWARE("radeon/hainan_ce.bin");
 120MODULE_FIRMWARE("radeon/hainan_mc.bin");
 121MODULE_FIRMWARE("radeon/hainan_rlc.bin");
 122MODULE_FIRMWARE("radeon/hainan_smc.bin");
 123MODULE_FIRMWARE("radeon/hainan_k_smc.bin");
 124MODULE_FIRMWARE("radeon/banks_k_2_smc.bin");
 125
 126MODULE_FIRMWARE("radeon/si58_mc.bin");
 127
 128static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
 129static void si_pcie_gen3_enable(struct radeon_device *rdev);
 130static void si_program_aspm(struct radeon_device *rdev);
 131extern void sumo_rlc_fini(struct radeon_device *rdev);
 132extern int sumo_rlc_init(struct radeon_device *rdev);
 
 
 
 
 
 
 
 
 133static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
 134					 bool enable);
 135static void si_init_pg(struct radeon_device *rdev);
 136static void si_init_cg(struct radeon_device *rdev);
 137static void si_fini_pg(struct radeon_device *rdev);
 138static void si_fini_cg(struct radeon_device *rdev);
 139static void si_rlc_stop(struct radeon_device *rdev);
 140
 141static const u32 crtc_offsets[] =
 142{
 143	EVERGREEN_CRTC0_REGISTER_OFFSET,
 144	EVERGREEN_CRTC1_REGISTER_OFFSET,
 145	EVERGREEN_CRTC2_REGISTER_OFFSET,
 146	EVERGREEN_CRTC3_REGISTER_OFFSET,
 147	EVERGREEN_CRTC4_REGISTER_OFFSET,
 148	EVERGREEN_CRTC5_REGISTER_OFFSET
 149};
 150
 151static const u32 si_disp_int_status[] =
 152{
 153	DISP_INTERRUPT_STATUS,
 154	DISP_INTERRUPT_STATUS_CONTINUE,
 155	DISP_INTERRUPT_STATUS_CONTINUE2,
 156	DISP_INTERRUPT_STATUS_CONTINUE3,
 157	DISP_INTERRUPT_STATUS_CONTINUE4,
 158	DISP_INTERRUPT_STATUS_CONTINUE5
 159};
 160
 161#define DC_HPDx_CONTROL(x)        (DC_HPD1_CONTROL     + (x * 0xc))
 162#define DC_HPDx_INT_CONTROL(x)    (DC_HPD1_INT_CONTROL + (x * 0xc))
 163#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS  + (x * 0xc))
 164
 165static const u32 verde_rlc_save_restore_register_list[] =
 166{
 167	(0x8000 << 16) | (0x98f4 >> 2),
 168	0x00000000,
 169	(0x8040 << 16) | (0x98f4 >> 2),
 170	0x00000000,
 171	(0x8000 << 16) | (0xe80 >> 2),
 172	0x00000000,
 173	(0x8040 << 16) | (0xe80 >> 2),
 174	0x00000000,
 175	(0x8000 << 16) | (0x89bc >> 2),
 176	0x00000000,
 177	(0x8040 << 16) | (0x89bc >> 2),
 178	0x00000000,
 179	(0x8000 << 16) | (0x8c1c >> 2),
 180	0x00000000,
 181	(0x8040 << 16) | (0x8c1c >> 2),
 182	0x00000000,
 183	(0x9c00 << 16) | (0x98f0 >> 2),
 184	0x00000000,
 185	(0x9c00 << 16) | (0xe7c >> 2),
 186	0x00000000,
 187	(0x8000 << 16) | (0x9148 >> 2),
 188	0x00000000,
 189	(0x8040 << 16) | (0x9148 >> 2),
 190	0x00000000,
 191	(0x9c00 << 16) | (0x9150 >> 2),
 192	0x00000000,
 193	(0x9c00 << 16) | (0x897c >> 2),
 194	0x00000000,
 195	(0x9c00 << 16) | (0x8d8c >> 2),
 196	0x00000000,
 197	(0x9c00 << 16) | (0xac54 >> 2),
 198	0X00000000,
 199	0x3,
 200	(0x9c00 << 16) | (0x98f8 >> 2),
 201	0x00000000,
 202	(0x9c00 << 16) | (0x9910 >> 2),
 203	0x00000000,
 204	(0x9c00 << 16) | (0x9914 >> 2),
 205	0x00000000,
 206	(0x9c00 << 16) | (0x9918 >> 2),
 207	0x00000000,
 208	(0x9c00 << 16) | (0x991c >> 2),
 209	0x00000000,
 210	(0x9c00 << 16) | (0x9920 >> 2),
 211	0x00000000,
 212	(0x9c00 << 16) | (0x9924 >> 2),
 213	0x00000000,
 214	(0x9c00 << 16) | (0x9928 >> 2),
 215	0x00000000,
 216	(0x9c00 << 16) | (0x992c >> 2),
 217	0x00000000,
 218	(0x9c00 << 16) | (0x9930 >> 2),
 219	0x00000000,
 220	(0x9c00 << 16) | (0x9934 >> 2),
 221	0x00000000,
 222	(0x9c00 << 16) | (0x9938 >> 2),
 223	0x00000000,
 224	(0x9c00 << 16) | (0x993c >> 2),
 225	0x00000000,
 226	(0x9c00 << 16) | (0x9940 >> 2),
 227	0x00000000,
 228	(0x9c00 << 16) | (0x9944 >> 2),
 229	0x00000000,
 230	(0x9c00 << 16) | (0x9948 >> 2),
 231	0x00000000,
 232	(0x9c00 << 16) | (0x994c >> 2),
 233	0x00000000,
 234	(0x9c00 << 16) | (0x9950 >> 2),
 235	0x00000000,
 236	(0x9c00 << 16) | (0x9954 >> 2),
 237	0x00000000,
 238	(0x9c00 << 16) | (0x9958 >> 2),
 239	0x00000000,
 240	(0x9c00 << 16) | (0x995c >> 2),
 241	0x00000000,
 242	(0x9c00 << 16) | (0x9960 >> 2),
 243	0x00000000,
 244	(0x9c00 << 16) | (0x9964 >> 2),
 245	0x00000000,
 246	(0x9c00 << 16) | (0x9968 >> 2),
 247	0x00000000,
 248	(0x9c00 << 16) | (0x996c >> 2),
 249	0x00000000,
 250	(0x9c00 << 16) | (0x9970 >> 2),
 251	0x00000000,
 252	(0x9c00 << 16) | (0x9974 >> 2),
 253	0x00000000,
 254	(0x9c00 << 16) | (0x9978 >> 2),
 255	0x00000000,
 256	(0x9c00 << 16) | (0x997c >> 2),
 257	0x00000000,
 258	(0x9c00 << 16) | (0x9980 >> 2),
 259	0x00000000,
 260	(0x9c00 << 16) | (0x9984 >> 2),
 261	0x00000000,
 262	(0x9c00 << 16) | (0x9988 >> 2),
 263	0x00000000,
 264	(0x9c00 << 16) | (0x998c >> 2),
 265	0x00000000,
 266	(0x9c00 << 16) | (0x8c00 >> 2),
 267	0x00000000,
 268	(0x9c00 << 16) | (0x8c14 >> 2),
 269	0x00000000,
 270	(0x9c00 << 16) | (0x8c04 >> 2),
 271	0x00000000,
 272	(0x9c00 << 16) | (0x8c08 >> 2),
 273	0x00000000,
 274	(0x8000 << 16) | (0x9b7c >> 2),
 275	0x00000000,
 276	(0x8040 << 16) | (0x9b7c >> 2),
 277	0x00000000,
 278	(0x8000 << 16) | (0xe84 >> 2),
 279	0x00000000,
 280	(0x8040 << 16) | (0xe84 >> 2),
 281	0x00000000,
 282	(0x8000 << 16) | (0x89c0 >> 2),
 283	0x00000000,
 284	(0x8040 << 16) | (0x89c0 >> 2),
 285	0x00000000,
 286	(0x8000 << 16) | (0x914c >> 2),
 287	0x00000000,
 288	(0x8040 << 16) | (0x914c >> 2),
 289	0x00000000,
 290	(0x8000 << 16) | (0x8c20 >> 2),
 291	0x00000000,
 292	(0x8040 << 16) | (0x8c20 >> 2),
 293	0x00000000,
 294	(0x8000 << 16) | (0x9354 >> 2),
 295	0x00000000,
 296	(0x8040 << 16) | (0x9354 >> 2),
 297	0x00000000,
 298	(0x9c00 << 16) | (0x9060 >> 2),
 299	0x00000000,
 300	(0x9c00 << 16) | (0x9364 >> 2),
 301	0x00000000,
 302	(0x9c00 << 16) | (0x9100 >> 2),
 303	0x00000000,
 304	(0x9c00 << 16) | (0x913c >> 2),
 305	0x00000000,
 306	(0x8000 << 16) | (0x90e0 >> 2),
 307	0x00000000,
 308	(0x8000 << 16) | (0x90e4 >> 2),
 309	0x00000000,
 310	(0x8000 << 16) | (0x90e8 >> 2),
 311	0x00000000,
 312	(0x8040 << 16) | (0x90e0 >> 2),
 313	0x00000000,
 314	(0x8040 << 16) | (0x90e4 >> 2),
 315	0x00000000,
 316	(0x8040 << 16) | (0x90e8 >> 2),
 317	0x00000000,
 318	(0x9c00 << 16) | (0x8bcc >> 2),
 319	0x00000000,
 320	(0x9c00 << 16) | (0x8b24 >> 2),
 321	0x00000000,
 322	(0x9c00 << 16) | (0x88c4 >> 2),
 323	0x00000000,
 324	(0x9c00 << 16) | (0x8e50 >> 2),
 325	0x00000000,
 326	(0x9c00 << 16) | (0x8c0c >> 2),
 327	0x00000000,
 328	(0x9c00 << 16) | (0x8e58 >> 2),
 329	0x00000000,
 330	(0x9c00 << 16) | (0x8e5c >> 2),
 331	0x00000000,
 332	(0x9c00 << 16) | (0x9508 >> 2),
 333	0x00000000,
 334	(0x9c00 << 16) | (0x950c >> 2),
 335	0x00000000,
 336	(0x9c00 << 16) | (0x9494 >> 2),
 337	0x00000000,
 338	(0x9c00 << 16) | (0xac0c >> 2),
 339	0x00000000,
 340	(0x9c00 << 16) | (0xac10 >> 2),
 341	0x00000000,
 342	(0x9c00 << 16) | (0xac14 >> 2),
 343	0x00000000,
 344	(0x9c00 << 16) | (0xae00 >> 2),
 345	0x00000000,
 346	(0x9c00 << 16) | (0xac08 >> 2),
 347	0x00000000,
 348	(0x9c00 << 16) | (0x88d4 >> 2),
 349	0x00000000,
 350	(0x9c00 << 16) | (0x88c8 >> 2),
 351	0x00000000,
 352	(0x9c00 << 16) | (0x88cc >> 2),
 353	0x00000000,
 354	(0x9c00 << 16) | (0x89b0 >> 2),
 355	0x00000000,
 356	(0x9c00 << 16) | (0x8b10 >> 2),
 357	0x00000000,
 358	(0x9c00 << 16) | (0x8a14 >> 2),
 359	0x00000000,
 360	(0x9c00 << 16) | (0x9830 >> 2),
 361	0x00000000,
 362	(0x9c00 << 16) | (0x9834 >> 2),
 363	0x00000000,
 364	(0x9c00 << 16) | (0x9838 >> 2),
 365	0x00000000,
 366	(0x9c00 << 16) | (0x9a10 >> 2),
 367	0x00000000,
 368	(0x8000 << 16) | (0x9870 >> 2),
 369	0x00000000,
 370	(0x8000 << 16) | (0x9874 >> 2),
 371	0x00000000,
 372	(0x8001 << 16) | (0x9870 >> 2),
 373	0x00000000,
 374	(0x8001 << 16) | (0x9874 >> 2),
 375	0x00000000,
 376	(0x8040 << 16) | (0x9870 >> 2),
 377	0x00000000,
 378	(0x8040 << 16) | (0x9874 >> 2),
 379	0x00000000,
 380	(0x8041 << 16) | (0x9870 >> 2),
 381	0x00000000,
 382	(0x8041 << 16) | (0x9874 >> 2),
 383	0x00000000,
 384	0x00000000
 385};
 386
 387static const u32 tahiti_golden_rlc_registers[] =
 388{
 389	0xc424, 0xffffffff, 0x00601005,
 390	0xc47c, 0xffffffff, 0x10104040,
 391	0xc488, 0xffffffff, 0x0100000a,
 392	0xc314, 0xffffffff, 0x00000800,
 393	0xc30c, 0xffffffff, 0x800000f4,
 394	0xf4a8, 0xffffffff, 0x00000000
 395};
 396
 397static const u32 tahiti_golden_registers[] =
 398{
 399	0x9a10, 0x00010000, 0x00018208,
 400	0x9830, 0xffffffff, 0x00000000,
 401	0x9834, 0xf00fffff, 0x00000400,
 402	0x9838, 0x0002021c, 0x00020200,
 403	0xc78, 0x00000080, 0x00000000,
 404	0xd030, 0x000300c0, 0x00800040,
 405	0xd830, 0x000300c0, 0x00800040,
 406	0x5bb0, 0x000000f0, 0x00000070,
 407	0x5bc0, 0x00200000, 0x50100000,
 408	0x7030, 0x31000311, 0x00000011,
 409	0x277c, 0x00000003, 0x000007ff,
 410	0x240c, 0x000007ff, 0x00000000,
 411	0x8a14, 0xf000001f, 0x00000007,
 412	0x8b24, 0xffffffff, 0x00ffffff,
 413	0x8b10, 0x0000ff0f, 0x00000000,
 414	0x28a4c, 0x07ffffff, 0x4e000000,
 415	0x28350, 0x3f3f3fff, 0x2a00126a,
 416	0x30, 0x000000ff, 0x0040,
 417	0x34, 0x00000040, 0x00004040,
 418	0x9100, 0x07ffffff, 0x03000000,
 419	0x8e88, 0x01ff1f3f, 0x00000000,
 420	0x8e84, 0x01ff1f3f, 0x00000000,
 421	0x9060, 0x0000007f, 0x00000020,
 422	0x9508, 0x00010000, 0x00010000,
 423	0xac14, 0x00000200, 0x000002fb,
 424	0xac10, 0xffffffff, 0x0000543b,
 425	0xac0c, 0xffffffff, 0xa9210876,
 426	0x88d0, 0xffffffff, 0x000fff40,
 427	0x88d4, 0x0000001f, 0x00000010,
 428	0x1410, 0x20000000, 0x20fffed8,
 429	0x15c0, 0x000c0fc0, 0x000c0400
 430};
 431
 432static const u32 tahiti_golden_registers2[] =
 433{
 434	0xc64, 0x00000001, 0x00000001
 435};
 436
 437static const u32 pitcairn_golden_rlc_registers[] =
 438{
 439	0xc424, 0xffffffff, 0x00601004,
 440	0xc47c, 0xffffffff, 0x10102020,
 441	0xc488, 0xffffffff, 0x01000020,
 442	0xc314, 0xffffffff, 0x00000800,
 443	0xc30c, 0xffffffff, 0x800000a4
 444};
 445
 446static const u32 pitcairn_golden_registers[] =
 447{
 448	0x9a10, 0x00010000, 0x00018208,
 449	0x9830, 0xffffffff, 0x00000000,
 450	0x9834, 0xf00fffff, 0x00000400,
 451	0x9838, 0x0002021c, 0x00020200,
 452	0xc78, 0x00000080, 0x00000000,
 453	0xd030, 0x000300c0, 0x00800040,
 454	0xd830, 0x000300c0, 0x00800040,
 455	0x5bb0, 0x000000f0, 0x00000070,
 456	0x5bc0, 0x00200000, 0x50100000,
 457	0x7030, 0x31000311, 0x00000011,
 458	0x2ae4, 0x00073ffe, 0x000022a2,
 459	0x240c, 0x000007ff, 0x00000000,
 460	0x8a14, 0xf000001f, 0x00000007,
 461	0x8b24, 0xffffffff, 0x00ffffff,
 462	0x8b10, 0x0000ff0f, 0x00000000,
 463	0x28a4c, 0x07ffffff, 0x4e000000,
 464	0x28350, 0x3f3f3fff, 0x2a00126a,
 465	0x30, 0x000000ff, 0x0040,
 466	0x34, 0x00000040, 0x00004040,
 467	0x9100, 0x07ffffff, 0x03000000,
 468	0x9060, 0x0000007f, 0x00000020,
 469	0x9508, 0x00010000, 0x00010000,
 470	0xac14, 0x000003ff, 0x000000f7,
 471	0xac10, 0xffffffff, 0x00000000,
 472	0xac0c, 0xffffffff, 0x32761054,
 473	0x88d4, 0x0000001f, 0x00000010,
 474	0x15c0, 0x000c0fc0, 0x000c0400
 475};
 476
 477static const u32 verde_golden_rlc_registers[] =
 478{
 479	0xc424, 0xffffffff, 0x033f1005,
 480	0xc47c, 0xffffffff, 0x10808020,
 481	0xc488, 0xffffffff, 0x00800008,
 482	0xc314, 0xffffffff, 0x00001000,
 483	0xc30c, 0xffffffff, 0x80010014
 484};
 485
 486static const u32 verde_golden_registers[] =
 487{
 488	0x9a10, 0x00010000, 0x00018208,
 489	0x9830, 0xffffffff, 0x00000000,
 490	0x9834, 0xf00fffff, 0x00000400,
 491	0x9838, 0x0002021c, 0x00020200,
 492	0xc78, 0x00000080, 0x00000000,
 493	0xd030, 0x000300c0, 0x00800040,
 494	0xd030, 0x000300c0, 0x00800040,
 495	0xd830, 0x000300c0, 0x00800040,
 496	0xd830, 0x000300c0, 0x00800040,
 497	0x5bb0, 0x000000f0, 0x00000070,
 498	0x5bc0, 0x00200000, 0x50100000,
 499	0x7030, 0x31000311, 0x00000011,
 500	0x2ae4, 0x00073ffe, 0x000022a2,
 501	0x2ae4, 0x00073ffe, 0x000022a2,
 502	0x2ae4, 0x00073ffe, 0x000022a2,
 503	0x240c, 0x000007ff, 0x00000000,
 504	0x240c, 0x000007ff, 0x00000000,
 505	0x240c, 0x000007ff, 0x00000000,
 506	0x8a14, 0xf000001f, 0x00000007,
 507	0x8a14, 0xf000001f, 0x00000007,
 508	0x8a14, 0xf000001f, 0x00000007,
 509	0x8b24, 0xffffffff, 0x00ffffff,
 510	0x8b10, 0x0000ff0f, 0x00000000,
 511	0x28a4c, 0x07ffffff, 0x4e000000,
 512	0x28350, 0x3f3f3fff, 0x0000124a,
 513	0x28350, 0x3f3f3fff, 0x0000124a,
 514	0x28350, 0x3f3f3fff, 0x0000124a,
 515	0x30, 0x000000ff, 0x0040,
 516	0x34, 0x00000040, 0x00004040,
 517	0x9100, 0x07ffffff, 0x03000000,
 518	0x9100, 0x07ffffff, 0x03000000,
 519	0x8e88, 0x01ff1f3f, 0x00000000,
 520	0x8e88, 0x01ff1f3f, 0x00000000,
 521	0x8e88, 0x01ff1f3f, 0x00000000,
 522	0x8e84, 0x01ff1f3f, 0x00000000,
 523	0x8e84, 0x01ff1f3f, 0x00000000,
 524	0x8e84, 0x01ff1f3f, 0x00000000,
 525	0x9060, 0x0000007f, 0x00000020,
 526	0x9508, 0x00010000, 0x00010000,
 527	0xac14, 0x000003ff, 0x00000003,
 528	0xac14, 0x000003ff, 0x00000003,
 529	0xac14, 0x000003ff, 0x00000003,
 530	0xac10, 0xffffffff, 0x00000000,
 531	0xac10, 0xffffffff, 0x00000000,
 532	0xac10, 0xffffffff, 0x00000000,
 533	0xac0c, 0xffffffff, 0x00001032,
 534	0xac0c, 0xffffffff, 0x00001032,
 535	0xac0c, 0xffffffff, 0x00001032,
 536	0x88d4, 0x0000001f, 0x00000010,
 537	0x88d4, 0x0000001f, 0x00000010,
 538	0x88d4, 0x0000001f, 0x00000010,
 539	0x15c0, 0x000c0fc0, 0x000c0400
 540};
 541
 542static const u32 oland_golden_rlc_registers[] =
 543{
 544	0xc424, 0xffffffff, 0x00601005,
 545	0xc47c, 0xffffffff, 0x10104040,
 546	0xc488, 0xffffffff, 0x0100000a,
 547	0xc314, 0xffffffff, 0x00000800,
 548	0xc30c, 0xffffffff, 0x800000f4
 549};
 550
 551static const u32 oland_golden_registers[] =
 552{
 553	0x9a10, 0x00010000, 0x00018208,
 554	0x9830, 0xffffffff, 0x00000000,
 555	0x9834, 0xf00fffff, 0x00000400,
 556	0x9838, 0x0002021c, 0x00020200,
 557	0xc78, 0x00000080, 0x00000000,
 558	0xd030, 0x000300c0, 0x00800040,
 559	0xd830, 0x000300c0, 0x00800040,
 560	0x5bb0, 0x000000f0, 0x00000070,
 561	0x5bc0, 0x00200000, 0x50100000,
 562	0x7030, 0x31000311, 0x00000011,
 563	0x2ae4, 0x00073ffe, 0x000022a2,
 564	0x240c, 0x000007ff, 0x00000000,
 565	0x8a14, 0xf000001f, 0x00000007,
 566	0x8b24, 0xffffffff, 0x00ffffff,
 567	0x8b10, 0x0000ff0f, 0x00000000,
 568	0x28a4c, 0x07ffffff, 0x4e000000,
 569	0x28350, 0x3f3f3fff, 0x00000082,
 570	0x30, 0x000000ff, 0x0040,
 571	0x34, 0x00000040, 0x00004040,
 572	0x9100, 0x07ffffff, 0x03000000,
 573	0x9060, 0x0000007f, 0x00000020,
 574	0x9508, 0x00010000, 0x00010000,
 575	0xac14, 0x000003ff, 0x000000f3,
 576	0xac10, 0xffffffff, 0x00000000,
 577	0xac0c, 0xffffffff, 0x00003210,
 578	0x88d4, 0x0000001f, 0x00000010,
 579	0x15c0, 0x000c0fc0, 0x000c0400
 580};
 581
 582static const u32 hainan_golden_registers[] =
 583{
 584	0x9a10, 0x00010000, 0x00018208,
 585	0x9830, 0xffffffff, 0x00000000,
 586	0x9834, 0xf00fffff, 0x00000400,
 587	0x9838, 0x0002021c, 0x00020200,
 588	0xd0c0, 0xff000fff, 0x00000100,
 589	0xd030, 0x000300c0, 0x00800040,
 590	0xd8c0, 0xff000fff, 0x00000100,
 591	0xd830, 0x000300c0, 0x00800040,
 592	0x2ae4, 0x00073ffe, 0x000022a2,
 593	0x240c, 0x000007ff, 0x00000000,
 594	0x8a14, 0xf000001f, 0x00000007,
 595	0x8b24, 0xffffffff, 0x00ffffff,
 596	0x8b10, 0x0000ff0f, 0x00000000,
 597	0x28a4c, 0x07ffffff, 0x4e000000,
 598	0x28350, 0x3f3f3fff, 0x00000000,
 599	0x30, 0x000000ff, 0x0040,
 600	0x34, 0x00000040, 0x00004040,
 601	0x9100, 0x03e00000, 0x03600000,
 602	0x9060, 0x0000007f, 0x00000020,
 603	0x9508, 0x00010000, 0x00010000,
 604	0xac14, 0x000003ff, 0x000000f1,
 605	0xac10, 0xffffffff, 0x00000000,
 606	0xac0c, 0xffffffff, 0x00003210,
 607	0x88d4, 0x0000001f, 0x00000010,
 608	0x15c0, 0x000c0fc0, 0x000c0400
 609};
 610
 611static const u32 hainan_golden_registers2[] =
 612{
 613	0x98f8, 0xffffffff, 0x02010001
 614};
 615
 616static const u32 tahiti_mgcg_cgcg_init[] =
 617{
 618	0xc400, 0xffffffff, 0xfffffffc,
 619	0x802c, 0xffffffff, 0xe0000000,
 620	0x9a60, 0xffffffff, 0x00000100,
 621	0x92a4, 0xffffffff, 0x00000100,
 622	0xc164, 0xffffffff, 0x00000100,
 623	0x9774, 0xffffffff, 0x00000100,
 624	0x8984, 0xffffffff, 0x06000100,
 625	0x8a18, 0xffffffff, 0x00000100,
 626	0x92a0, 0xffffffff, 0x00000100,
 627	0xc380, 0xffffffff, 0x00000100,
 628	0x8b28, 0xffffffff, 0x00000100,
 629	0x9144, 0xffffffff, 0x00000100,
 630	0x8d88, 0xffffffff, 0x00000100,
 631	0x8d8c, 0xffffffff, 0x00000100,
 632	0x9030, 0xffffffff, 0x00000100,
 633	0x9034, 0xffffffff, 0x00000100,
 634	0x9038, 0xffffffff, 0x00000100,
 635	0x903c, 0xffffffff, 0x00000100,
 636	0xad80, 0xffffffff, 0x00000100,
 637	0xac54, 0xffffffff, 0x00000100,
 638	0x897c, 0xffffffff, 0x06000100,
 639	0x9868, 0xffffffff, 0x00000100,
 640	0x9510, 0xffffffff, 0x00000100,
 641	0xaf04, 0xffffffff, 0x00000100,
 642	0xae04, 0xffffffff, 0x00000100,
 643	0x949c, 0xffffffff, 0x00000100,
 644	0x802c, 0xffffffff, 0xe0000000,
 645	0x9160, 0xffffffff, 0x00010000,
 646	0x9164, 0xffffffff, 0x00030002,
 647	0x9168, 0xffffffff, 0x00040007,
 648	0x916c, 0xffffffff, 0x00060005,
 649	0x9170, 0xffffffff, 0x00090008,
 650	0x9174, 0xffffffff, 0x00020001,
 651	0x9178, 0xffffffff, 0x00040003,
 652	0x917c, 0xffffffff, 0x00000007,
 653	0x9180, 0xffffffff, 0x00060005,
 654	0x9184, 0xffffffff, 0x00090008,
 655	0x9188, 0xffffffff, 0x00030002,
 656	0x918c, 0xffffffff, 0x00050004,
 657	0x9190, 0xffffffff, 0x00000008,
 658	0x9194, 0xffffffff, 0x00070006,
 659	0x9198, 0xffffffff, 0x000a0009,
 660	0x919c, 0xffffffff, 0x00040003,
 661	0x91a0, 0xffffffff, 0x00060005,
 662	0x91a4, 0xffffffff, 0x00000009,
 663	0x91a8, 0xffffffff, 0x00080007,
 664	0x91ac, 0xffffffff, 0x000b000a,
 665	0x91b0, 0xffffffff, 0x00050004,
 666	0x91b4, 0xffffffff, 0x00070006,
 667	0x91b8, 0xffffffff, 0x0008000b,
 668	0x91bc, 0xffffffff, 0x000a0009,
 669	0x91c0, 0xffffffff, 0x000d000c,
 670	0x91c4, 0xffffffff, 0x00060005,
 671	0x91c8, 0xffffffff, 0x00080007,
 672	0x91cc, 0xffffffff, 0x0000000b,
 673	0x91d0, 0xffffffff, 0x000a0009,
 674	0x91d4, 0xffffffff, 0x000d000c,
 675	0x91d8, 0xffffffff, 0x00070006,
 676	0x91dc, 0xffffffff, 0x00090008,
 677	0x91e0, 0xffffffff, 0x0000000c,
 678	0x91e4, 0xffffffff, 0x000b000a,
 679	0x91e8, 0xffffffff, 0x000e000d,
 680	0x91ec, 0xffffffff, 0x00080007,
 681	0x91f0, 0xffffffff, 0x000a0009,
 682	0x91f4, 0xffffffff, 0x0000000d,
 683	0x91f8, 0xffffffff, 0x000c000b,
 684	0x91fc, 0xffffffff, 0x000f000e,
 685	0x9200, 0xffffffff, 0x00090008,
 686	0x9204, 0xffffffff, 0x000b000a,
 687	0x9208, 0xffffffff, 0x000c000f,
 688	0x920c, 0xffffffff, 0x000e000d,
 689	0x9210, 0xffffffff, 0x00110010,
 690	0x9214, 0xffffffff, 0x000a0009,
 691	0x9218, 0xffffffff, 0x000c000b,
 692	0x921c, 0xffffffff, 0x0000000f,
 693	0x9220, 0xffffffff, 0x000e000d,
 694	0x9224, 0xffffffff, 0x00110010,
 695	0x9228, 0xffffffff, 0x000b000a,
 696	0x922c, 0xffffffff, 0x000d000c,
 697	0x9230, 0xffffffff, 0x00000010,
 698	0x9234, 0xffffffff, 0x000f000e,
 699	0x9238, 0xffffffff, 0x00120011,
 700	0x923c, 0xffffffff, 0x000c000b,
 701	0x9240, 0xffffffff, 0x000e000d,
 702	0x9244, 0xffffffff, 0x00000011,
 703	0x9248, 0xffffffff, 0x0010000f,
 704	0x924c, 0xffffffff, 0x00130012,
 705	0x9250, 0xffffffff, 0x000d000c,
 706	0x9254, 0xffffffff, 0x000f000e,
 707	0x9258, 0xffffffff, 0x00100013,
 708	0x925c, 0xffffffff, 0x00120011,
 709	0x9260, 0xffffffff, 0x00150014,
 710	0x9264, 0xffffffff, 0x000e000d,
 711	0x9268, 0xffffffff, 0x0010000f,
 712	0x926c, 0xffffffff, 0x00000013,
 713	0x9270, 0xffffffff, 0x00120011,
 714	0x9274, 0xffffffff, 0x00150014,
 715	0x9278, 0xffffffff, 0x000f000e,
 716	0x927c, 0xffffffff, 0x00110010,
 717	0x9280, 0xffffffff, 0x00000014,
 718	0x9284, 0xffffffff, 0x00130012,
 719	0x9288, 0xffffffff, 0x00160015,
 720	0x928c, 0xffffffff, 0x0010000f,
 721	0x9290, 0xffffffff, 0x00120011,
 722	0x9294, 0xffffffff, 0x00000015,
 723	0x9298, 0xffffffff, 0x00140013,
 724	0x929c, 0xffffffff, 0x00170016,
 725	0x9150, 0xffffffff, 0x96940200,
 726	0x8708, 0xffffffff, 0x00900100,
 727	0xc478, 0xffffffff, 0x00000080,
 728	0xc404, 0xffffffff, 0x0020003f,
 729	0x30, 0xffffffff, 0x0000001c,
 730	0x34, 0x000f0000, 0x000f0000,
 731	0x160c, 0xffffffff, 0x00000100,
 732	0x1024, 0xffffffff, 0x00000100,
 733	0x102c, 0x00000101, 0x00000000,
 734	0x20a8, 0xffffffff, 0x00000104,
 735	0x264c, 0x000c0000, 0x000c0000,
 736	0x2648, 0x000c0000, 0x000c0000,
 737	0x55e4, 0xff000fff, 0x00000100,
 738	0x55e8, 0x00000001, 0x00000001,
 739	0x2f50, 0x00000001, 0x00000001,
 740	0x30cc, 0xc0000fff, 0x00000104,
 741	0xc1e4, 0x00000001, 0x00000001,
 742	0xd0c0, 0xfffffff0, 0x00000100,
 743	0xd8c0, 0xfffffff0, 0x00000100
 744};
 745
 746static const u32 pitcairn_mgcg_cgcg_init[] =
 747{
 748	0xc400, 0xffffffff, 0xfffffffc,
 749	0x802c, 0xffffffff, 0xe0000000,
 750	0x9a60, 0xffffffff, 0x00000100,
 751	0x92a4, 0xffffffff, 0x00000100,
 752	0xc164, 0xffffffff, 0x00000100,
 753	0x9774, 0xffffffff, 0x00000100,
 754	0x8984, 0xffffffff, 0x06000100,
 755	0x8a18, 0xffffffff, 0x00000100,
 756	0x92a0, 0xffffffff, 0x00000100,
 757	0xc380, 0xffffffff, 0x00000100,
 758	0x8b28, 0xffffffff, 0x00000100,
 759	0x9144, 0xffffffff, 0x00000100,
 760	0x8d88, 0xffffffff, 0x00000100,
 761	0x8d8c, 0xffffffff, 0x00000100,
 762	0x9030, 0xffffffff, 0x00000100,
 763	0x9034, 0xffffffff, 0x00000100,
 764	0x9038, 0xffffffff, 0x00000100,
 765	0x903c, 0xffffffff, 0x00000100,
 766	0xad80, 0xffffffff, 0x00000100,
 767	0xac54, 0xffffffff, 0x00000100,
 768	0x897c, 0xffffffff, 0x06000100,
 769	0x9868, 0xffffffff, 0x00000100,
 770	0x9510, 0xffffffff, 0x00000100,
 771	0xaf04, 0xffffffff, 0x00000100,
 772	0xae04, 0xffffffff, 0x00000100,
 773	0x949c, 0xffffffff, 0x00000100,
 774	0x802c, 0xffffffff, 0xe0000000,
 775	0x9160, 0xffffffff, 0x00010000,
 776	0x9164, 0xffffffff, 0x00030002,
 777	0x9168, 0xffffffff, 0x00040007,
 778	0x916c, 0xffffffff, 0x00060005,
 779	0x9170, 0xffffffff, 0x00090008,
 780	0x9174, 0xffffffff, 0x00020001,
 781	0x9178, 0xffffffff, 0x00040003,
 782	0x917c, 0xffffffff, 0x00000007,
 783	0x9180, 0xffffffff, 0x00060005,
 784	0x9184, 0xffffffff, 0x00090008,
 785	0x9188, 0xffffffff, 0x00030002,
 786	0x918c, 0xffffffff, 0x00050004,
 787	0x9190, 0xffffffff, 0x00000008,
 788	0x9194, 0xffffffff, 0x00070006,
 789	0x9198, 0xffffffff, 0x000a0009,
 790	0x919c, 0xffffffff, 0x00040003,
 791	0x91a0, 0xffffffff, 0x00060005,
 792	0x91a4, 0xffffffff, 0x00000009,
 793	0x91a8, 0xffffffff, 0x00080007,
 794	0x91ac, 0xffffffff, 0x000b000a,
 795	0x91b0, 0xffffffff, 0x00050004,
 796	0x91b4, 0xffffffff, 0x00070006,
 797	0x91b8, 0xffffffff, 0x0008000b,
 798	0x91bc, 0xffffffff, 0x000a0009,
 799	0x91c0, 0xffffffff, 0x000d000c,
 800	0x9200, 0xffffffff, 0x00090008,
 801	0x9204, 0xffffffff, 0x000b000a,
 802	0x9208, 0xffffffff, 0x000c000f,
 803	0x920c, 0xffffffff, 0x000e000d,
 804	0x9210, 0xffffffff, 0x00110010,
 805	0x9214, 0xffffffff, 0x000a0009,
 806	0x9218, 0xffffffff, 0x000c000b,
 807	0x921c, 0xffffffff, 0x0000000f,
 808	0x9220, 0xffffffff, 0x000e000d,
 809	0x9224, 0xffffffff, 0x00110010,
 810	0x9228, 0xffffffff, 0x000b000a,
 811	0x922c, 0xffffffff, 0x000d000c,
 812	0x9230, 0xffffffff, 0x00000010,
 813	0x9234, 0xffffffff, 0x000f000e,
 814	0x9238, 0xffffffff, 0x00120011,
 815	0x923c, 0xffffffff, 0x000c000b,
 816	0x9240, 0xffffffff, 0x000e000d,
 817	0x9244, 0xffffffff, 0x00000011,
 818	0x9248, 0xffffffff, 0x0010000f,
 819	0x924c, 0xffffffff, 0x00130012,
 820	0x9250, 0xffffffff, 0x000d000c,
 821	0x9254, 0xffffffff, 0x000f000e,
 822	0x9258, 0xffffffff, 0x00100013,
 823	0x925c, 0xffffffff, 0x00120011,
 824	0x9260, 0xffffffff, 0x00150014,
 825	0x9150, 0xffffffff, 0x96940200,
 826	0x8708, 0xffffffff, 0x00900100,
 827	0xc478, 0xffffffff, 0x00000080,
 828	0xc404, 0xffffffff, 0x0020003f,
 829	0x30, 0xffffffff, 0x0000001c,
 830	0x34, 0x000f0000, 0x000f0000,
 831	0x160c, 0xffffffff, 0x00000100,
 832	0x1024, 0xffffffff, 0x00000100,
 833	0x102c, 0x00000101, 0x00000000,
 834	0x20a8, 0xffffffff, 0x00000104,
 835	0x55e4, 0xff000fff, 0x00000100,
 836	0x55e8, 0x00000001, 0x00000001,
 837	0x2f50, 0x00000001, 0x00000001,
 838	0x30cc, 0xc0000fff, 0x00000104,
 839	0xc1e4, 0x00000001, 0x00000001,
 840	0xd0c0, 0xfffffff0, 0x00000100,
 841	0xd8c0, 0xfffffff0, 0x00000100
 842};
 843
 844static const u32 verde_mgcg_cgcg_init[] =
 845{
 846	0xc400, 0xffffffff, 0xfffffffc,
 847	0x802c, 0xffffffff, 0xe0000000,
 848	0x9a60, 0xffffffff, 0x00000100,
 849	0x92a4, 0xffffffff, 0x00000100,
 850	0xc164, 0xffffffff, 0x00000100,
 851	0x9774, 0xffffffff, 0x00000100,
 852	0x8984, 0xffffffff, 0x06000100,
 853	0x8a18, 0xffffffff, 0x00000100,
 854	0x92a0, 0xffffffff, 0x00000100,
 855	0xc380, 0xffffffff, 0x00000100,
 856	0x8b28, 0xffffffff, 0x00000100,
 857	0x9144, 0xffffffff, 0x00000100,
 858	0x8d88, 0xffffffff, 0x00000100,
 859	0x8d8c, 0xffffffff, 0x00000100,
 860	0x9030, 0xffffffff, 0x00000100,
 861	0x9034, 0xffffffff, 0x00000100,
 862	0x9038, 0xffffffff, 0x00000100,
 863	0x903c, 0xffffffff, 0x00000100,
 864	0xad80, 0xffffffff, 0x00000100,
 865	0xac54, 0xffffffff, 0x00000100,
 866	0x897c, 0xffffffff, 0x06000100,
 867	0x9868, 0xffffffff, 0x00000100,
 868	0x9510, 0xffffffff, 0x00000100,
 869	0xaf04, 0xffffffff, 0x00000100,
 870	0xae04, 0xffffffff, 0x00000100,
 871	0x949c, 0xffffffff, 0x00000100,
 872	0x802c, 0xffffffff, 0xe0000000,
 873	0x9160, 0xffffffff, 0x00010000,
 874	0x9164, 0xffffffff, 0x00030002,
 875	0x9168, 0xffffffff, 0x00040007,
 876	0x916c, 0xffffffff, 0x00060005,
 877	0x9170, 0xffffffff, 0x00090008,
 878	0x9174, 0xffffffff, 0x00020001,
 879	0x9178, 0xffffffff, 0x00040003,
 880	0x917c, 0xffffffff, 0x00000007,
 881	0x9180, 0xffffffff, 0x00060005,
 882	0x9184, 0xffffffff, 0x00090008,
 883	0x9188, 0xffffffff, 0x00030002,
 884	0x918c, 0xffffffff, 0x00050004,
 885	0x9190, 0xffffffff, 0x00000008,
 886	0x9194, 0xffffffff, 0x00070006,
 887	0x9198, 0xffffffff, 0x000a0009,
 888	0x919c, 0xffffffff, 0x00040003,
 889	0x91a0, 0xffffffff, 0x00060005,
 890	0x91a4, 0xffffffff, 0x00000009,
 891	0x91a8, 0xffffffff, 0x00080007,
 892	0x91ac, 0xffffffff, 0x000b000a,
 893	0x91b0, 0xffffffff, 0x00050004,
 894	0x91b4, 0xffffffff, 0x00070006,
 895	0x91b8, 0xffffffff, 0x0008000b,
 896	0x91bc, 0xffffffff, 0x000a0009,
 897	0x91c0, 0xffffffff, 0x000d000c,
 898	0x9200, 0xffffffff, 0x00090008,
 899	0x9204, 0xffffffff, 0x000b000a,
 900	0x9208, 0xffffffff, 0x000c000f,
 901	0x920c, 0xffffffff, 0x000e000d,
 902	0x9210, 0xffffffff, 0x00110010,
 903	0x9214, 0xffffffff, 0x000a0009,
 904	0x9218, 0xffffffff, 0x000c000b,
 905	0x921c, 0xffffffff, 0x0000000f,
 906	0x9220, 0xffffffff, 0x000e000d,
 907	0x9224, 0xffffffff, 0x00110010,
 908	0x9228, 0xffffffff, 0x000b000a,
 909	0x922c, 0xffffffff, 0x000d000c,
 910	0x9230, 0xffffffff, 0x00000010,
 911	0x9234, 0xffffffff, 0x000f000e,
 912	0x9238, 0xffffffff, 0x00120011,
 913	0x923c, 0xffffffff, 0x000c000b,
 914	0x9240, 0xffffffff, 0x000e000d,
 915	0x9244, 0xffffffff, 0x00000011,
 916	0x9248, 0xffffffff, 0x0010000f,
 917	0x924c, 0xffffffff, 0x00130012,
 918	0x9250, 0xffffffff, 0x000d000c,
 919	0x9254, 0xffffffff, 0x000f000e,
 920	0x9258, 0xffffffff, 0x00100013,
 921	0x925c, 0xffffffff, 0x00120011,
 922	0x9260, 0xffffffff, 0x00150014,
 923	0x9150, 0xffffffff, 0x96940200,
 924	0x8708, 0xffffffff, 0x00900100,
 925	0xc478, 0xffffffff, 0x00000080,
 926	0xc404, 0xffffffff, 0x0020003f,
 927	0x30, 0xffffffff, 0x0000001c,
 928	0x34, 0x000f0000, 0x000f0000,
 929	0x160c, 0xffffffff, 0x00000100,
 930	0x1024, 0xffffffff, 0x00000100,
 931	0x102c, 0x00000101, 0x00000000,
 932	0x20a8, 0xffffffff, 0x00000104,
 933	0x264c, 0x000c0000, 0x000c0000,
 934	0x2648, 0x000c0000, 0x000c0000,
 935	0x55e4, 0xff000fff, 0x00000100,
 936	0x55e8, 0x00000001, 0x00000001,
 937	0x2f50, 0x00000001, 0x00000001,
 938	0x30cc, 0xc0000fff, 0x00000104,
 939	0xc1e4, 0x00000001, 0x00000001,
 940	0xd0c0, 0xfffffff0, 0x00000100,
 941	0xd8c0, 0xfffffff0, 0x00000100
 942};
 943
 944static const u32 oland_mgcg_cgcg_init[] =
 945{
 946	0xc400, 0xffffffff, 0xfffffffc,
 947	0x802c, 0xffffffff, 0xe0000000,
 948	0x9a60, 0xffffffff, 0x00000100,
 949	0x92a4, 0xffffffff, 0x00000100,
 950	0xc164, 0xffffffff, 0x00000100,
 951	0x9774, 0xffffffff, 0x00000100,
 952	0x8984, 0xffffffff, 0x06000100,
 953	0x8a18, 0xffffffff, 0x00000100,
 954	0x92a0, 0xffffffff, 0x00000100,
 955	0xc380, 0xffffffff, 0x00000100,
 956	0x8b28, 0xffffffff, 0x00000100,
 957	0x9144, 0xffffffff, 0x00000100,
 958	0x8d88, 0xffffffff, 0x00000100,
 959	0x8d8c, 0xffffffff, 0x00000100,
 960	0x9030, 0xffffffff, 0x00000100,
 961	0x9034, 0xffffffff, 0x00000100,
 962	0x9038, 0xffffffff, 0x00000100,
 963	0x903c, 0xffffffff, 0x00000100,
 964	0xad80, 0xffffffff, 0x00000100,
 965	0xac54, 0xffffffff, 0x00000100,
 966	0x897c, 0xffffffff, 0x06000100,
 967	0x9868, 0xffffffff, 0x00000100,
 968	0x9510, 0xffffffff, 0x00000100,
 969	0xaf04, 0xffffffff, 0x00000100,
 970	0xae04, 0xffffffff, 0x00000100,
 971	0x949c, 0xffffffff, 0x00000100,
 972	0x802c, 0xffffffff, 0xe0000000,
 973	0x9160, 0xffffffff, 0x00010000,
 974	0x9164, 0xffffffff, 0x00030002,
 975	0x9168, 0xffffffff, 0x00040007,
 976	0x916c, 0xffffffff, 0x00060005,
 977	0x9170, 0xffffffff, 0x00090008,
 978	0x9174, 0xffffffff, 0x00020001,
 979	0x9178, 0xffffffff, 0x00040003,
 980	0x917c, 0xffffffff, 0x00000007,
 981	0x9180, 0xffffffff, 0x00060005,
 982	0x9184, 0xffffffff, 0x00090008,
 983	0x9188, 0xffffffff, 0x00030002,
 984	0x918c, 0xffffffff, 0x00050004,
 985	0x9190, 0xffffffff, 0x00000008,
 986	0x9194, 0xffffffff, 0x00070006,
 987	0x9198, 0xffffffff, 0x000a0009,
 988	0x919c, 0xffffffff, 0x00040003,
 989	0x91a0, 0xffffffff, 0x00060005,
 990	0x91a4, 0xffffffff, 0x00000009,
 991	0x91a8, 0xffffffff, 0x00080007,
 992	0x91ac, 0xffffffff, 0x000b000a,
 993	0x91b0, 0xffffffff, 0x00050004,
 994	0x91b4, 0xffffffff, 0x00070006,
 995	0x91b8, 0xffffffff, 0x0008000b,
 996	0x91bc, 0xffffffff, 0x000a0009,
 997	0x91c0, 0xffffffff, 0x000d000c,
 998	0x91c4, 0xffffffff, 0x00060005,
 999	0x91c8, 0xffffffff, 0x00080007,
1000	0x91cc, 0xffffffff, 0x0000000b,
1001	0x91d0, 0xffffffff, 0x000a0009,
1002	0x91d4, 0xffffffff, 0x000d000c,
1003	0x9150, 0xffffffff, 0x96940200,
1004	0x8708, 0xffffffff, 0x00900100,
1005	0xc478, 0xffffffff, 0x00000080,
1006	0xc404, 0xffffffff, 0x0020003f,
1007	0x30, 0xffffffff, 0x0000001c,
1008	0x34, 0x000f0000, 0x000f0000,
1009	0x160c, 0xffffffff, 0x00000100,
1010	0x1024, 0xffffffff, 0x00000100,
1011	0x102c, 0x00000101, 0x00000000,
1012	0x20a8, 0xffffffff, 0x00000104,
1013	0x264c, 0x000c0000, 0x000c0000,
1014	0x2648, 0x000c0000, 0x000c0000,
1015	0x55e4, 0xff000fff, 0x00000100,
1016	0x55e8, 0x00000001, 0x00000001,
1017	0x2f50, 0x00000001, 0x00000001,
1018	0x30cc, 0xc0000fff, 0x00000104,
1019	0xc1e4, 0x00000001, 0x00000001,
1020	0xd0c0, 0xfffffff0, 0x00000100,
1021	0xd8c0, 0xfffffff0, 0x00000100
1022};
1023
1024static const u32 hainan_mgcg_cgcg_init[] =
1025{
1026	0xc400, 0xffffffff, 0xfffffffc,
1027	0x802c, 0xffffffff, 0xe0000000,
1028	0x9a60, 0xffffffff, 0x00000100,
1029	0x92a4, 0xffffffff, 0x00000100,
1030	0xc164, 0xffffffff, 0x00000100,
1031	0x9774, 0xffffffff, 0x00000100,
1032	0x8984, 0xffffffff, 0x06000100,
1033	0x8a18, 0xffffffff, 0x00000100,
1034	0x92a0, 0xffffffff, 0x00000100,
1035	0xc380, 0xffffffff, 0x00000100,
1036	0x8b28, 0xffffffff, 0x00000100,
1037	0x9144, 0xffffffff, 0x00000100,
1038	0x8d88, 0xffffffff, 0x00000100,
1039	0x8d8c, 0xffffffff, 0x00000100,
1040	0x9030, 0xffffffff, 0x00000100,
1041	0x9034, 0xffffffff, 0x00000100,
1042	0x9038, 0xffffffff, 0x00000100,
1043	0x903c, 0xffffffff, 0x00000100,
1044	0xad80, 0xffffffff, 0x00000100,
1045	0xac54, 0xffffffff, 0x00000100,
1046	0x897c, 0xffffffff, 0x06000100,
1047	0x9868, 0xffffffff, 0x00000100,
1048	0x9510, 0xffffffff, 0x00000100,
1049	0xaf04, 0xffffffff, 0x00000100,
1050	0xae04, 0xffffffff, 0x00000100,
1051	0x949c, 0xffffffff, 0x00000100,
1052	0x802c, 0xffffffff, 0xe0000000,
1053	0x9160, 0xffffffff, 0x00010000,
1054	0x9164, 0xffffffff, 0x00030002,
1055	0x9168, 0xffffffff, 0x00040007,
1056	0x916c, 0xffffffff, 0x00060005,
1057	0x9170, 0xffffffff, 0x00090008,
1058	0x9174, 0xffffffff, 0x00020001,
1059	0x9178, 0xffffffff, 0x00040003,
1060	0x917c, 0xffffffff, 0x00000007,
1061	0x9180, 0xffffffff, 0x00060005,
1062	0x9184, 0xffffffff, 0x00090008,
1063	0x9188, 0xffffffff, 0x00030002,
1064	0x918c, 0xffffffff, 0x00050004,
1065	0x9190, 0xffffffff, 0x00000008,
1066	0x9194, 0xffffffff, 0x00070006,
1067	0x9198, 0xffffffff, 0x000a0009,
1068	0x919c, 0xffffffff, 0x00040003,
1069	0x91a0, 0xffffffff, 0x00060005,
1070	0x91a4, 0xffffffff, 0x00000009,
1071	0x91a8, 0xffffffff, 0x00080007,
1072	0x91ac, 0xffffffff, 0x000b000a,
1073	0x91b0, 0xffffffff, 0x00050004,
1074	0x91b4, 0xffffffff, 0x00070006,
1075	0x91b8, 0xffffffff, 0x0008000b,
1076	0x91bc, 0xffffffff, 0x000a0009,
1077	0x91c0, 0xffffffff, 0x000d000c,
1078	0x91c4, 0xffffffff, 0x00060005,
1079	0x91c8, 0xffffffff, 0x00080007,
1080	0x91cc, 0xffffffff, 0x0000000b,
1081	0x91d0, 0xffffffff, 0x000a0009,
1082	0x91d4, 0xffffffff, 0x000d000c,
1083	0x9150, 0xffffffff, 0x96940200,
1084	0x8708, 0xffffffff, 0x00900100,
1085	0xc478, 0xffffffff, 0x00000080,
1086	0xc404, 0xffffffff, 0x0020003f,
1087	0x30, 0xffffffff, 0x0000001c,
1088	0x34, 0x000f0000, 0x000f0000,
1089	0x160c, 0xffffffff, 0x00000100,
1090	0x1024, 0xffffffff, 0x00000100,
1091	0x20a8, 0xffffffff, 0x00000104,
1092	0x264c, 0x000c0000, 0x000c0000,
1093	0x2648, 0x000c0000, 0x000c0000,
1094	0x2f50, 0x00000001, 0x00000001,
1095	0x30cc, 0xc0000fff, 0x00000104,
1096	0xc1e4, 0x00000001, 0x00000001,
1097	0xd0c0, 0xfffffff0, 0x00000100,
1098	0xd8c0, 0xfffffff0, 0x00000100
1099};
1100
1101static u32 verde_pg_init[] =
1102{
1103	0x353c, 0xffffffff, 0x40000,
1104	0x3538, 0xffffffff, 0x200010ff,
1105	0x353c, 0xffffffff, 0x0,
1106	0x353c, 0xffffffff, 0x0,
1107	0x353c, 0xffffffff, 0x0,
1108	0x353c, 0xffffffff, 0x0,
1109	0x353c, 0xffffffff, 0x0,
1110	0x353c, 0xffffffff, 0x7007,
1111	0x3538, 0xffffffff, 0x300010ff,
1112	0x353c, 0xffffffff, 0x0,
1113	0x353c, 0xffffffff, 0x0,
1114	0x353c, 0xffffffff, 0x0,
1115	0x353c, 0xffffffff, 0x0,
1116	0x353c, 0xffffffff, 0x0,
1117	0x353c, 0xffffffff, 0x400000,
1118	0x3538, 0xffffffff, 0x100010ff,
1119	0x353c, 0xffffffff, 0x0,
1120	0x353c, 0xffffffff, 0x0,
1121	0x353c, 0xffffffff, 0x0,
1122	0x353c, 0xffffffff, 0x0,
1123	0x353c, 0xffffffff, 0x0,
1124	0x353c, 0xffffffff, 0x120200,
1125	0x3538, 0xffffffff, 0x500010ff,
1126	0x353c, 0xffffffff, 0x0,
1127	0x353c, 0xffffffff, 0x0,
1128	0x353c, 0xffffffff, 0x0,
1129	0x353c, 0xffffffff, 0x0,
1130	0x353c, 0xffffffff, 0x0,
1131	0x353c, 0xffffffff, 0x1e1e16,
1132	0x3538, 0xffffffff, 0x600010ff,
1133	0x353c, 0xffffffff, 0x0,
1134	0x353c, 0xffffffff, 0x0,
1135	0x353c, 0xffffffff, 0x0,
1136	0x353c, 0xffffffff, 0x0,
1137	0x353c, 0xffffffff, 0x0,
1138	0x353c, 0xffffffff, 0x171f1e,
1139	0x3538, 0xffffffff, 0x700010ff,
1140	0x353c, 0xffffffff, 0x0,
1141	0x353c, 0xffffffff, 0x0,
1142	0x353c, 0xffffffff, 0x0,
1143	0x353c, 0xffffffff, 0x0,
1144	0x353c, 0xffffffff, 0x0,
1145	0x353c, 0xffffffff, 0x0,
1146	0x3538, 0xffffffff, 0x9ff,
1147	0x3500, 0xffffffff, 0x0,
1148	0x3504, 0xffffffff, 0x10000800,
1149	0x3504, 0xffffffff, 0xf,
1150	0x3504, 0xffffffff, 0xf,
1151	0x3500, 0xffffffff, 0x4,
1152	0x3504, 0xffffffff, 0x1000051e,
1153	0x3504, 0xffffffff, 0xffff,
1154	0x3504, 0xffffffff, 0xffff,
1155	0x3500, 0xffffffff, 0x8,
1156	0x3504, 0xffffffff, 0x80500,
1157	0x3500, 0xffffffff, 0x12,
1158	0x3504, 0xffffffff, 0x9050c,
1159	0x3500, 0xffffffff, 0x1d,
1160	0x3504, 0xffffffff, 0xb052c,
1161	0x3500, 0xffffffff, 0x2a,
1162	0x3504, 0xffffffff, 0x1053e,
1163	0x3500, 0xffffffff, 0x2d,
1164	0x3504, 0xffffffff, 0x10546,
1165	0x3500, 0xffffffff, 0x30,
1166	0x3504, 0xffffffff, 0xa054e,
1167	0x3500, 0xffffffff, 0x3c,
1168	0x3504, 0xffffffff, 0x1055f,
1169	0x3500, 0xffffffff, 0x3f,
1170	0x3504, 0xffffffff, 0x10567,
1171	0x3500, 0xffffffff, 0x42,
1172	0x3504, 0xffffffff, 0x1056f,
1173	0x3500, 0xffffffff, 0x45,
1174	0x3504, 0xffffffff, 0x10572,
1175	0x3500, 0xffffffff, 0x48,
1176	0x3504, 0xffffffff, 0x20575,
1177	0x3500, 0xffffffff, 0x4c,
1178	0x3504, 0xffffffff, 0x190801,
1179	0x3500, 0xffffffff, 0x67,
1180	0x3504, 0xffffffff, 0x1082a,
1181	0x3500, 0xffffffff, 0x6a,
1182	0x3504, 0xffffffff, 0x1b082d,
1183	0x3500, 0xffffffff, 0x87,
1184	0x3504, 0xffffffff, 0x310851,
1185	0x3500, 0xffffffff, 0xba,
1186	0x3504, 0xffffffff, 0x891,
1187	0x3500, 0xffffffff, 0xbc,
1188	0x3504, 0xffffffff, 0x893,
1189	0x3500, 0xffffffff, 0xbe,
1190	0x3504, 0xffffffff, 0x20895,
1191	0x3500, 0xffffffff, 0xc2,
1192	0x3504, 0xffffffff, 0x20899,
1193	0x3500, 0xffffffff, 0xc6,
1194	0x3504, 0xffffffff, 0x2089d,
1195	0x3500, 0xffffffff, 0xca,
1196	0x3504, 0xffffffff, 0x8a1,
1197	0x3500, 0xffffffff, 0xcc,
1198	0x3504, 0xffffffff, 0x8a3,
1199	0x3500, 0xffffffff, 0xce,
1200	0x3504, 0xffffffff, 0x308a5,
1201	0x3500, 0xffffffff, 0xd3,
1202	0x3504, 0xffffffff, 0x6d08cd,
1203	0x3500, 0xffffffff, 0x142,
1204	0x3504, 0xffffffff, 0x2000095a,
1205	0x3504, 0xffffffff, 0x1,
1206	0x3500, 0xffffffff, 0x144,
1207	0x3504, 0xffffffff, 0x301f095b,
1208	0x3500, 0xffffffff, 0x165,
1209	0x3504, 0xffffffff, 0xc094d,
1210	0x3500, 0xffffffff, 0x173,
1211	0x3504, 0xffffffff, 0xf096d,
1212	0x3500, 0xffffffff, 0x184,
1213	0x3504, 0xffffffff, 0x15097f,
1214	0x3500, 0xffffffff, 0x19b,
1215	0x3504, 0xffffffff, 0xc0998,
1216	0x3500, 0xffffffff, 0x1a9,
1217	0x3504, 0xffffffff, 0x409a7,
1218	0x3500, 0xffffffff, 0x1af,
1219	0x3504, 0xffffffff, 0xcdc,
1220	0x3500, 0xffffffff, 0x1b1,
1221	0x3504, 0xffffffff, 0x800,
1222	0x3508, 0xffffffff, 0x6c9b2000,
1223	0x3510, 0xfc00, 0x2000,
1224	0x3544, 0xffffffff, 0xfc0,
1225	0x28d4, 0x00000100, 0x100
1226};
1227
1228static void si_init_golden_registers(struct radeon_device *rdev)
1229{
1230	switch (rdev->family) {
1231	case CHIP_TAHITI:
1232		radeon_program_register_sequence(rdev,
1233						 tahiti_golden_registers,
1234						 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1235		radeon_program_register_sequence(rdev,
1236						 tahiti_golden_rlc_registers,
1237						 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1238		radeon_program_register_sequence(rdev,
1239						 tahiti_mgcg_cgcg_init,
1240						 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1241		radeon_program_register_sequence(rdev,
1242						 tahiti_golden_registers2,
1243						 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1244		break;
1245	case CHIP_PITCAIRN:
1246		radeon_program_register_sequence(rdev,
1247						 pitcairn_golden_registers,
1248						 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1249		radeon_program_register_sequence(rdev,
1250						 pitcairn_golden_rlc_registers,
1251						 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1252		radeon_program_register_sequence(rdev,
1253						 pitcairn_mgcg_cgcg_init,
1254						 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1255		break;
1256	case CHIP_VERDE:
1257		radeon_program_register_sequence(rdev,
1258						 verde_golden_registers,
1259						 (const u32)ARRAY_SIZE(verde_golden_registers));
1260		radeon_program_register_sequence(rdev,
1261						 verde_golden_rlc_registers,
1262						 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1263		radeon_program_register_sequence(rdev,
1264						 verde_mgcg_cgcg_init,
1265						 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1266		radeon_program_register_sequence(rdev,
1267						 verde_pg_init,
1268						 (const u32)ARRAY_SIZE(verde_pg_init));
1269		break;
1270	case CHIP_OLAND:
1271		radeon_program_register_sequence(rdev,
1272						 oland_golden_registers,
1273						 (const u32)ARRAY_SIZE(oland_golden_registers));
1274		radeon_program_register_sequence(rdev,
1275						 oland_golden_rlc_registers,
1276						 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1277		radeon_program_register_sequence(rdev,
1278						 oland_mgcg_cgcg_init,
1279						 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1280		break;
1281	case CHIP_HAINAN:
1282		radeon_program_register_sequence(rdev,
1283						 hainan_golden_registers,
1284						 (const u32)ARRAY_SIZE(hainan_golden_registers));
1285		radeon_program_register_sequence(rdev,
1286						 hainan_golden_registers2,
1287						 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1288		radeon_program_register_sequence(rdev,
1289						 hainan_mgcg_cgcg_init,
1290						 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1291		break;
1292	default:
1293		break;
1294	}
1295}
1296
1297/**
1298 * si_get_allowed_info_register - fetch the register for the info ioctl
1299 *
1300 * @rdev: radeon_device pointer
1301 * @reg: register offset in bytes
1302 * @val: register value
1303 *
1304 * Returns 0 for success or -EINVAL for an invalid register
1305 *
1306 */
1307int si_get_allowed_info_register(struct radeon_device *rdev,
1308				 u32 reg, u32 *val)
1309{
1310	switch (reg) {
1311	case GRBM_STATUS:
1312	case GRBM_STATUS2:
1313	case GRBM_STATUS_SE0:
1314	case GRBM_STATUS_SE1:
1315	case SRBM_STATUS:
1316	case SRBM_STATUS2:
1317	case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
1318	case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
1319	case UVD_STATUS:
1320		*val = RREG32(reg);
1321		return 0;
1322	default:
1323		return -EINVAL;
1324	}
1325}
1326
1327#define PCIE_BUS_CLK                10000
1328#define TCLK                        (PCIE_BUS_CLK / 10)
1329
1330/**
1331 * si_get_xclk - get the xclk
1332 *
1333 * @rdev: radeon_device pointer
1334 *
1335 * Returns the reference clock used by the gfx engine
1336 * (SI).
1337 */
1338u32 si_get_xclk(struct radeon_device *rdev)
1339{
1340	u32 reference_clock = rdev->clock.spll.reference_freq;
1341	u32 tmp;
1342
1343	tmp = RREG32(CG_CLKPIN_CNTL_2);
1344	if (tmp & MUX_TCLK_TO_XCLK)
1345		return TCLK;
1346
1347	tmp = RREG32(CG_CLKPIN_CNTL);
1348	if (tmp & XTALIN_DIVIDE)
1349		return reference_clock / 4;
1350
1351	return reference_clock;
1352}
1353
1354/* get temperature in millidegrees */
1355int si_get_temp(struct radeon_device *rdev)
1356{
1357	u32 temp;
1358	int actual_temp = 0;
1359
1360	temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1361		CTF_TEMP_SHIFT;
1362
1363	if (temp & 0x200)
1364		actual_temp = 255;
1365	else
1366		actual_temp = temp & 0x1ff;
1367
1368	actual_temp = (actual_temp * 1000);
1369
1370	return actual_temp;
1371}
1372
1373#define TAHITI_IO_MC_REGS_SIZE 36
1374
1375static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1376	{0x0000006f, 0x03044000},
1377	{0x00000070, 0x0480c018},
1378	{0x00000071, 0x00000040},
1379	{0x00000072, 0x01000000},
1380	{0x00000074, 0x000000ff},
1381	{0x00000075, 0x00143400},
1382	{0x00000076, 0x08ec0800},
1383	{0x00000077, 0x040000cc},
1384	{0x00000079, 0x00000000},
1385	{0x0000007a, 0x21000409},
1386	{0x0000007c, 0x00000000},
1387	{0x0000007d, 0xe8000000},
1388	{0x0000007e, 0x044408a8},
1389	{0x0000007f, 0x00000003},
1390	{0x00000080, 0x00000000},
1391	{0x00000081, 0x01000000},
1392	{0x00000082, 0x02000000},
1393	{0x00000083, 0x00000000},
1394	{0x00000084, 0xe3f3e4f4},
1395	{0x00000085, 0x00052024},
1396	{0x00000087, 0x00000000},
1397	{0x00000088, 0x66036603},
1398	{0x00000089, 0x01000000},
1399	{0x0000008b, 0x1c0a0000},
1400	{0x0000008c, 0xff010000},
1401	{0x0000008e, 0xffffefff},
1402	{0x0000008f, 0xfff3efff},
1403	{0x00000090, 0xfff3efbf},
1404	{0x00000094, 0x00101101},
1405	{0x00000095, 0x00000fff},
1406	{0x00000096, 0x00116fff},
1407	{0x00000097, 0x60010000},
1408	{0x00000098, 0x10010000},
1409	{0x00000099, 0x00006000},
1410	{0x0000009a, 0x00001000},
1411	{0x0000009f, 0x00a77400}
1412};
1413
1414static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1415	{0x0000006f, 0x03044000},
1416	{0x00000070, 0x0480c018},
1417	{0x00000071, 0x00000040},
1418	{0x00000072, 0x01000000},
1419	{0x00000074, 0x000000ff},
1420	{0x00000075, 0x00143400},
1421	{0x00000076, 0x08ec0800},
1422	{0x00000077, 0x040000cc},
1423	{0x00000079, 0x00000000},
1424	{0x0000007a, 0x21000409},
1425	{0x0000007c, 0x00000000},
1426	{0x0000007d, 0xe8000000},
1427	{0x0000007e, 0x044408a8},
1428	{0x0000007f, 0x00000003},
1429	{0x00000080, 0x00000000},
1430	{0x00000081, 0x01000000},
1431	{0x00000082, 0x02000000},
1432	{0x00000083, 0x00000000},
1433	{0x00000084, 0xe3f3e4f4},
1434	{0x00000085, 0x00052024},
1435	{0x00000087, 0x00000000},
1436	{0x00000088, 0x66036603},
1437	{0x00000089, 0x01000000},
1438	{0x0000008b, 0x1c0a0000},
1439	{0x0000008c, 0xff010000},
1440	{0x0000008e, 0xffffefff},
1441	{0x0000008f, 0xfff3efff},
1442	{0x00000090, 0xfff3efbf},
1443	{0x00000094, 0x00101101},
1444	{0x00000095, 0x00000fff},
1445	{0x00000096, 0x00116fff},
1446	{0x00000097, 0x60010000},
1447	{0x00000098, 0x10010000},
1448	{0x00000099, 0x00006000},
1449	{0x0000009a, 0x00001000},
1450	{0x0000009f, 0x00a47400}
1451};
1452
1453static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1454	{0x0000006f, 0x03044000},
1455	{0x00000070, 0x0480c018},
1456	{0x00000071, 0x00000040},
1457	{0x00000072, 0x01000000},
1458	{0x00000074, 0x000000ff},
1459	{0x00000075, 0x00143400},
1460	{0x00000076, 0x08ec0800},
1461	{0x00000077, 0x040000cc},
1462	{0x00000079, 0x00000000},
1463	{0x0000007a, 0x21000409},
1464	{0x0000007c, 0x00000000},
1465	{0x0000007d, 0xe8000000},
1466	{0x0000007e, 0x044408a8},
1467	{0x0000007f, 0x00000003},
1468	{0x00000080, 0x00000000},
1469	{0x00000081, 0x01000000},
1470	{0x00000082, 0x02000000},
1471	{0x00000083, 0x00000000},
1472	{0x00000084, 0xe3f3e4f4},
1473	{0x00000085, 0x00052024},
1474	{0x00000087, 0x00000000},
1475	{0x00000088, 0x66036603},
1476	{0x00000089, 0x01000000},
1477	{0x0000008b, 0x1c0a0000},
1478	{0x0000008c, 0xff010000},
1479	{0x0000008e, 0xffffefff},
1480	{0x0000008f, 0xfff3efff},
1481	{0x00000090, 0xfff3efbf},
1482	{0x00000094, 0x00101101},
1483	{0x00000095, 0x00000fff},
1484	{0x00000096, 0x00116fff},
1485	{0x00000097, 0x60010000},
1486	{0x00000098, 0x10010000},
1487	{0x00000099, 0x00006000},
1488	{0x0000009a, 0x00001000},
1489	{0x0000009f, 0x00a37400}
1490};
1491
1492static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1493	{0x0000006f, 0x03044000},
1494	{0x00000070, 0x0480c018},
1495	{0x00000071, 0x00000040},
1496	{0x00000072, 0x01000000},
1497	{0x00000074, 0x000000ff},
1498	{0x00000075, 0x00143400},
1499	{0x00000076, 0x08ec0800},
1500	{0x00000077, 0x040000cc},
1501	{0x00000079, 0x00000000},
1502	{0x0000007a, 0x21000409},
1503	{0x0000007c, 0x00000000},
1504	{0x0000007d, 0xe8000000},
1505	{0x0000007e, 0x044408a8},
1506	{0x0000007f, 0x00000003},
1507	{0x00000080, 0x00000000},
1508	{0x00000081, 0x01000000},
1509	{0x00000082, 0x02000000},
1510	{0x00000083, 0x00000000},
1511	{0x00000084, 0xe3f3e4f4},
1512	{0x00000085, 0x00052024},
1513	{0x00000087, 0x00000000},
1514	{0x00000088, 0x66036603},
1515	{0x00000089, 0x01000000},
1516	{0x0000008b, 0x1c0a0000},
1517	{0x0000008c, 0xff010000},
1518	{0x0000008e, 0xffffefff},
1519	{0x0000008f, 0xfff3efff},
1520	{0x00000090, 0xfff3efbf},
1521	{0x00000094, 0x00101101},
1522	{0x00000095, 0x00000fff},
1523	{0x00000096, 0x00116fff},
1524	{0x00000097, 0x60010000},
1525	{0x00000098, 0x10010000},
1526	{0x00000099, 0x00006000},
1527	{0x0000009a, 0x00001000},
1528	{0x0000009f, 0x00a17730}
1529};
1530
1531static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1532	{0x0000006f, 0x03044000},
1533	{0x00000070, 0x0480c018},
1534	{0x00000071, 0x00000040},
1535	{0x00000072, 0x01000000},
1536	{0x00000074, 0x000000ff},
1537	{0x00000075, 0x00143400},
1538	{0x00000076, 0x08ec0800},
1539	{0x00000077, 0x040000cc},
1540	{0x00000079, 0x00000000},
1541	{0x0000007a, 0x21000409},
1542	{0x0000007c, 0x00000000},
1543	{0x0000007d, 0xe8000000},
1544	{0x0000007e, 0x044408a8},
1545	{0x0000007f, 0x00000003},
1546	{0x00000080, 0x00000000},
1547	{0x00000081, 0x01000000},
1548	{0x00000082, 0x02000000},
1549	{0x00000083, 0x00000000},
1550	{0x00000084, 0xe3f3e4f4},
1551	{0x00000085, 0x00052024},
1552	{0x00000087, 0x00000000},
1553	{0x00000088, 0x66036603},
1554	{0x00000089, 0x01000000},
1555	{0x0000008b, 0x1c0a0000},
1556	{0x0000008c, 0xff010000},
1557	{0x0000008e, 0xffffefff},
1558	{0x0000008f, 0xfff3efff},
1559	{0x00000090, 0xfff3efbf},
1560	{0x00000094, 0x00101101},
1561	{0x00000095, 0x00000fff},
1562	{0x00000096, 0x00116fff},
1563	{0x00000097, 0x60010000},
1564	{0x00000098, 0x10010000},
1565	{0x00000099, 0x00006000},
1566	{0x0000009a, 0x00001000},
1567	{0x0000009f, 0x00a07730}
1568};
1569
1570/* ucode loading */
1571int si_mc_load_microcode(struct radeon_device *rdev)
1572{
1573	const __be32 *fw_data = NULL;
1574	const __le32 *new_fw_data = NULL;
1575	u32 running;
1576	u32 *io_mc_regs = NULL;
1577	const __le32 *new_io_mc_regs = NULL;
1578	int i, regs_size, ucode_size;
1579
1580	if (!rdev->mc_fw)
1581		return -EINVAL;
1582
1583	if (rdev->new_fw) {
1584		const struct mc_firmware_header_v1_0 *hdr =
1585			(const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1586
1587		radeon_ucode_print_mc_hdr(&hdr->header);
1588		regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1589		new_io_mc_regs = (const __le32 *)
1590			(rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1591		ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1592		new_fw_data = (const __le32 *)
1593			(rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1594	} else {
1595		ucode_size = rdev->mc_fw->size / 4;
1596
1597		switch (rdev->family) {
1598		case CHIP_TAHITI:
1599			io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1600			regs_size = TAHITI_IO_MC_REGS_SIZE;
1601			break;
1602		case CHIP_PITCAIRN:
1603			io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1604			regs_size = TAHITI_IO_MC_REGS_SIZE;
1605			break;
1606		case CHIP_VERDE:
1607		default:
1608			io_mc_regs = (u32 *)&verde_io_mc_regs;
1609			regs_size = TAHITI_IO_MC_REGS_SIZE;
1610			break;
1611		case CHIP_OLAND:
1612			io_mc_regs = (u32 *)&oland_io_mc_regs;
1613			regs_size = TAHITI_IO_MC_REGS_SIZE;
1614			break;
1615		case CHIP_HAINAN:
1616			io_mc_regs = (u32 *)&hainan_io_mc_regs;
1617			regs_size = TAHITI_IO_MC_REGS_SIZE;
1618			break;
1619		}
1620		fw_data = (const __be32 *)rdev->mc_fw->data;
1621	}
1622
1623	running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1624
1625	if (running == 0) {
 
 
 
 
 
1626		/* reset the engine and set to writable */
1627		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1628		WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1629
1630		/* load mc io regs */
1631		for (i = 0; i < regs_size; i++) {
1632			if (rdev->new_fw) {
1633				WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1634				WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1635			} else {
1636				WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1637				WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1638			}
1639		}
1640		/* load the MC ucode */
1641		for (i = 0; i < ucode_size; i++) {
1642			if (rdev->new_fw)
1643				WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1644			else
1645				WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1646		}
1647
1648		/* put the engine back into the active state */
1649		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1650		WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1651		WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1652
1653		/* wait for training to complete */
1654		for (i = 0; i < rdev->usec_timeout; i++) {
1655			if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1656				break;
1657			udelay(1);
1658		}
1659		for (i = 0; i < rdev->usec_timeout; i++) {
1660			if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1661				break;
1662			udelay(1);
1663		}
 
 
 
1664	}
1665
1666	return 0;
1667}
1668
1669static int si_init_microcode(struct radeon_device *rdev)
1670{
1671	const char *chip_name;
1672	const char *new_chip_name;
1673	size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1674	size_t smc_req_size, mc2_req_size;
1675	char fw_name[30];
1676	int err;
1677	int new_fw = 0;
1678	bool new_smc = false;
1679	bool si58_fw = false;
1680	bool banks2_fw = false;
1681
1682	DRM_DEBUG("\n");
1683
1684	switch (rdev->family) {
1685	case CHIP_TAHITI:
1686		chip_name = "TAHITI";
1687		new_chip_name = "tahiti";
1688		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1689		me_req_size = SI_PM4_UCODE_SIZE * 4;
1690		ce_req_size = SI_CE_UCODE_SIZE * 4;
1691		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1692		mc_req_size = SI_MC_UCODE_SIZE * 4;
1693		mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1694		smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1695		break;
1696	case CHIP_PITCAIRN:
1697		chip_name = "PITCAIRN";
1698		if ((rdev->pdev->revision == 0x81) &&
1699		    ((rdev->pdev->device == 0x6810) ||
1700		     (rdev->pdev->device == 0x6811)))
1701			new_smc = true;
1702		new_chip_name = "pitcairn";
1703		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1704		me_req_size = SI_PM4_UCODE_SIZE * 4;
1705		ce_req_size = SI_CE_UCODE_SIZE * 4;
1706		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1707		mc_req_size = SI_MC_UCODE_SIZE * 4;
1708		mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1709		smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1710		break;
1711	case CHIP_VERDE:
1712		chip_name = "VERDE";
1713		if (((rdev->pdev->device == 0x6820) &&
1714		     ((rdev->pdev->revision == 0x81) ||
1715		      (rdev->pdev->revision == 0x83))) ||
1716		    ((rdev->pdev->device == 0x6821) &&
1717		     ((rdev->pdev->revision == 0x83) ||
1718		      (rdev->pdev->revision == 0x87))) ||
1719		    ((rdev->pdev->revision == 0x87) &&
1720		     ((rdev->pdev->device == 0x6823) ||
1721		      (rdev->pdev->device == 0x682b))))
1722			new_smc = true;
1723		new_chip_name = "verde";
1724		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1725		me_req_size = SI_PM4_UCODE_SIZE * 4;
1726		ce_req_size = SI_CE_UCODE_SIZE * 4;
1727		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1728		mc_req_size = SI_MC_UCODE_SIZE * 4;
1729		mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1730		smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1731		break;
1732	case CHIP_OLAND:
1733		chip_name = "OLAND";
1734		if (((rdev->pdev->revision == 0x81) &&
1735		     ((rdev->pdev->device == 0x6600) ||
1736		      (rdev->pdev->device == 0x6604) ||
1737		      (rdev->pdev->device == 0x6605) ||
1738		      (rdev->pdev->device == 0x6610))) ||
1739		    ((rdev->pdev->revision == 0x83) &&
1740		     (rdev->pdev->device == 0x6610)))
1741			new_smc = true;
1742		new_chip_name = "oland";
1743		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1744		me_req_size = SI_PM4_UCODE_SIZE * 4;
1745		ce_req_size = SI_CE_UCODE_SIZE * 4;
1746		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1747		mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1748		smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1749		break;
1750	case CHIP_HAINAN:
1751		chip_name = "HAINAN";
1752		if (((rdev->pdev->revision == 0x81) &&
1753		     (rdev->pdev->device == 0x6660)) ||
1754		    ((rdev->pdev->revision == 0x83) &&
1755		     ((rdev->pdev->device == 0x6660) ||
1756		      (rdev->pdev->device == 0x6663) ||
1757		      (rdev->pdev->device == 0x6665) ||
1758		      (rdev->pdev->device == 0x6667))))
1759			new_smc = true;
1760		else if ((rdev->pdev->revision == 0xc3) &&
1761			 (rdev->pdev->device == 0x6665))
1762			banks2_fw = true;
1763		new_chip_name = "hainan";
1764		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1765		me_req_size = SI_PM4_UCODE_SIZE * 4;
1766		ce_req_size = SI_CE_UCODE_SIZE * 4;
1767		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1768		mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1769		smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1770		break;
1771	default: BUG();
1772	}
1773
1774	/* this memory configuration requires special firmware */
1775	if (((RREG32(MC_SEQ_MISC0) & 0xff000000) >> 24) == 0x58)
1776		si58_fw = true;
1777
1778	DRM_INFO("Loading %s Microcode\n", new_chip_name);
1779
1780	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1781	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1782	if (err) {
1783		snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1784		err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1785		if (err)
1786			goto out;
1787		if (rdev->pfp_fw->size != pfp_req_size) {
1788			pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
 
1789			       rdev->pfp_fw->size, fw_name);
1790			err = -EINVAL;
1791			goto out;
1792		}
1793	} else {
1794		err = radeon_ucode_validate(rdev->pfp_fw);
1795		if (err) {
1796			pr_err("si_cp: validation failed for firmware \"%s\"\n",
 
1797			       fw_name);
1798			goto out;
1799		} else {
1800			new_fw++;
1801		}
1802	}
1803
1804	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
1805	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1806	if (err) {
1807		snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1808		err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1809		if (err)
1810			goto out;
1811		if (rdev->me_fw->size != me_req_size) {
1812			pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
 
1813			       rdev->me_fw->size, fw_name);
1814			err = -EINVAL;
1815		}
1816	} else {
1817		err = radeon_ucode_validate(rdev->me_fw);
1818		if (err) {
1819			pr_err("si_cp: validation failed for firmware \"%s\"\n",
 
1820			       fw_name);
1821			goto out;
1822		} else {
1823			new_fw++;
1824		}
1825	}
1826
1827	snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
1828	err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1829	if (err) {
1830		snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1831		err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1832		if (err)
1833			goto out;
1834		if (rdev->ce_fw->size != ce_req_size) {
1835			pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
 
1836			       rdev->ce_fw->size, fw_name);
1837			err = -EINVAL;
1838		}
1839	} else {
1840		err = radeon_ucode_validate(rdev->ce_fw);
1841		if (err) {
1842			pr_err("si_cp: validation failed for firmware \"%s\"\n",
 
1843			       fw_name);
1844			goto out;
1845		} else {
1846			new_fw++;
1847		}
1848	}
1849
1850	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
1851	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1852	if (err) {
1853		snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1854		err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1855		if (err)
1856			goto out;
1857		if (rdev->rlc_fw->size != rlc_req_size) {
1858			pr_err("si_rlc: Bogus length %zu in firmware \"%s\"\n",
 
1859			       rdev->rlc_fw->size, fw_name);
1860			err = -EINVAL;
1861		}
1862	} else {
1863		err = radeon_ucode_validate(rdev->rlc_fw);
1864		if (err) {
1865			pr_err("si_cp: validation failed for firmware \"%s\"\n",
 
1866			       fw_name);
1867			goto out;
1868		} else {
1869			new_fw++;
1870		}
1871	}
1872
1873	if (si58_fw)
1874		snprintf(fw_name, sizeof(fw_name), "radeon/si58_mc.bin");
1875	else
1876		snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
1877	err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1878	if (err) {
1879		snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1880		err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1881		if (err) {
1882			snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1883			err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1884			if (err)
1885				goto out;
1886		}
1887		if ((rdev->mc_fw->size != mc_req_size) &&
1888		    (rdev->mc_fw->size != mc2_req_size)) {
1889			pr_err("si_mc: Bogus length %zu in firmware \"%s\"\n",
 
1890			       rdev->mc_fw->size, fw_name);
1891			err = -EINVAL;
1892		}
1893		DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1894	} else {
1895		err = radeon_ucode_validate(rdev->mc_fw);
1896		if (err) {
1897			pr_err("si_cp: validation failed for firmware \"%s\"\n",
 
1898			       fw_name);
1899			goto out;
1900		} else {
1901			new_fw++;
1902		}
1903	}
1904
1905	if (banks2_fw)
1906		snprintf(fw_name, sizeof(fw_name), "radeon/banks_k_2_smc.bin");
1907	else if (new_smc)
1908		snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
1909	else
1910		snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
1911	err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1912	if (err) {
1913		snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1914		err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1915		if (err) {
1916			pr_err("smc: error loading firmware \"%s\"\n", fw_name);
 
 
1917			release_firmware(rdev->smc_fw);
1918			rdev->smc_fw = NULL;
1919			err = 0;
1920		} else if (rdev->smc_fw->size != smc_req_size) {
1921			pr_err("si_smc: Bogus length %zu in firmware \"%s\"\n",
 
1922			       rdev->smc_fw->size, fw_name);
1923			err = -EINVAL;
1924		}
1925	} else {
1926		err = radeon_ucode_validate(rdev->smc_fw);
1927		if (err) {
1928			pr_err("si_cp: validation failed for firmware \"%s\"\n",
 
1929			       fw_name);
1930			goto out;
1931		} else {
1932			new_fw++;
1933		}
1934	}
1935
1936	if (new_fw == 0) {
1937		rdev->new_fw = false;
1938	} else if (new_fw < 6) {
1939		pr_err("si_fw: mixing new and old firmware!\n");
1940		err = -EINVAL;
1941	} else {
1942		rdev->new_fw = true;
1943	}
1944out:
1945	if (err) {
1946		if (err != -EINVAL)
1947			pr_err("si_cp: Failed to load firmware \"%s\"\n",
 
1948			       fw_name);
1949		release_firmware(rdev->pfp_fw);
1950		rdev->pfp_fw = NULL;
1951		release_firmware(rdev->me_fw);
1952		rdev->me_fw = NULL;
1953		release_firmware(rdev->ce_fw);
1954		rdev->ce_fw = NULL;
1955		release_firmware(rdev->rlc_fw);
1956		rdev->rlc_fw = NULL;
1957		release_firmware(rdev->mc_fw);
1958		rdev->mc_fw = NULL;
1959		release_firmware(rdev->smc_fw);
1960		rdev->smc_fw = NULL;
1961	}
1962	return err;
1963}
1964
1965/* watermark setup */
1966static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1967				   struct radeon_crtc *radeon_crtc,
1968				   struct drm_display_mode *mode,
1969				   struct drm_display_mode *other_mode)
1970{
1971	u32 tmp, buffer_alloc, i;
1972	u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1973	/*
1974	 * Line Buffer Setup
1975	 * There are 3 line buffers, each one shared by 2 display controllers.
1976	 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1977	 * the display controllers.  The paritioning is done via one of four
1978	 * preset allocations specified in bits 21:20:
1979	 *  0 - half lb
1980	 *  2 - whole lb, other crtc must be disabled
1981	 */
1982	/* this can get tricky if we have two large displays on a paired group
1983	 * of crtcs.  Ideally for multiple large displays we'd assign them to
1984	 * non-linked crtcs for maximum line buffer allocation.
1985	 */
1986	if (radeon_crtc->base.enabled && mode) {
1987		if (other_mode) {
1988			tmp = 0; /* 1/2 */
1989			buffer_alloc = 1;
1990		} else {
1991			tmp = 2; /* whole */
1992			buffer_alloc = 2;
1993		}
1994	} else {
1995		tmp = 0;
1996		buffer_alloc = 0;
1997	}
1998
1999	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
2000	       DC_LB_MEMORY_CONFIG(tmp));
2001
2002	WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
2003	       DMIF_BUFFERS_ALLOCATED(buffer_alloc));
2004	for (i = 0; i < rdev->usec_timeout; i++) {
2005		if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
2006		    DMIF_BUFFERS_ALLOCATED_COMPLETED)
2007			break;
2008		udelay(1);
2009	}
2010
2011	if (radeon_crtc->base.enabled && mode) {
2012		switch (tmp) {
2013		case 0:
2014		default:
2015			return 4096 * 2;
2016		case 2:
2017			return 8192 * 2;
2018		}
2019	}
2020
2021	/* controller not enabled, so no lb used */
2022	return 0;
2023}
2024
2025static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
2026{
2027	u32 tmp = RREG32(MC_SHARED_CHMAP);
2028
2029	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2030	case 0:
2031	default:
2032		return 1;
2033	case 1:
2034		return 2;
2035	case 2:
2036		return 4;
2037	case 3:
2038		return 8;
2039	case 4:
2040		return 3;
2041	case 5:
2042		return 6;
2043	case 6:
2044		return 10;
2045	case 7:
2046		return 12;
2047	case 8:
2048		return 16;
2049	}
2050}
2051
2052struct dce6_wm_params {
2053	u32 dram_channels; /* number of dram channels */
2054	u32 yclk;          /* bandwidth per dram data pin in kHz */
2055	u32 sclk;          /* engine clock in kHz */
2056	u32 disp_clk;      /* display clock in kHz */
2057	u32 src_width;     /* viewport width */
2058	u32 active_time;   /* active display time in ns */
2059	u32 blank_time;    /* blank time in ns */
2060	bool interlaced;    /* mode is interlaced */
2061	fixed20_12 vsc;    /* vertical scale ratio */
2062	u32 num_heads;     /* number of active crtcs */
2063	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2064	u32 lb_size;       /* line buffer allocated to pipe */
2065	u32 vtaps;         /* vertical scaler taps */
2066};
2067
2068static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
2069{
2070	/* Calculate raw DRAM Bandwidth */
2071	fixed20_12 dram_efficiency; /* 0.7 */
2072	fixed20_12 yclk, dram_channels, bandwidth;
2073	fixed20_12 a;
2074
2075	a.full = dfixed_const(1000);
2076	yclk.full = dfixed_const(wm->yclk);
2077	yclk.full = dfixed_div(yclk, a);
2078	dram_channels.full = dfixed_const(wm->dram_channels * 4);
2079	a.full = dfixed_const(10);
2080	dram_efficiency.full = dfixed_const(7);
2081	dram_efficiency.full = dfixed_div(dram_efficiency, a);
2082	bandwidth.full = dfixed_mul(dram_channels, yclk);
2083	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2084
2085	return dfixed_trunc(bandwidth);
2086}
2087
2088static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2089{
2090	/* Calculate DRAM Bandwidth and the part allocated to display. */
2091	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2092	fixed20_12 yclk, dram_channels, bandwidth;
2093	fixed20_12 a;
2094
2095	a.full = dfixed_const(1000);
2096	yclk.full = dfixed_const(wm->yclk);
2097	yclk.full = dfixed_div(yclk, a);
2098	dram_channels.full = dfixed_const(wm->dram_channels * 4);
2099	a.full = dfixed_const(10);
2100	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2101	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2102	bandwidth.full = dfixed_mul(dram_channels, yclk);
2103	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2104
2105	return dfixed_trunc(bandwidth);
2106}
2107
2108static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2109{
2110	/* Calculate the display Data return Bandwidth */
2111	fixed20_12 return_efficiency; /* 0.8 */
2112	fixed20_12 sclk, bandwidth;
2113	fixed20_12 a;
2114
2115	a.full = dfixed_const(1000);
2116	sclk.full = dfixed_const(wm->sclk);
2117	sclk.full = dfixed_div(sclk, a);
2118	a.full = dfixed_const(10);
2119	return_efficiency.full = dfixed_const(8);
2120	return_efficiency.full = dfixed_div(return_efficiency, a);
2121	a.full = dfixed_const(32);
2122	bandwidth.full = dfixed_mul(a, sclk);
2123	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2124
2125	return dfixed_trunc(bandwidth);
2126}
2127
2128static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2129{
2130	return 32;
2131}
2132
2133static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2134{
2135	/* Calculate the DMIF Request Bandwidth */
2136	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2137	fixed20_12 disp_clk, sclk, bandwidth;
2138	fixed20_12 a, b1, b2;
2139	u32 min_bandwidth;
2140
2141	a.full = dfixed_const(1000);
2142	disp_clk.full = dfixed_const(wm->disp_clk);
2143	disp_clk.full = dfixed_div(disp_clk, a);
2144	a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2145	b1.full = dfixed_mul(a, disp_clk);
2146
2147	a.full = dfixed_const(1000);
2148	sclk.full = dfixed_const(wm->sclk);
2149	sclk.full = dfixed_div(sclk, a);
2150	a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2151	b2.full = dfixed_mul(a, sclk);
2152
2153	a.full = dfixed_const(10);
2154	disp_clk_request_efficiency.full = dfixed_const(8);
2155	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2156
2157	min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2158
2159	a.full = dfixed_const(min_bandwidth);
2160	bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2161
2162	return dfixed_trunc(bandwidth);
2163}
2164
2165static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2166{
2167	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2168	u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2169	u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2170	u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2171
2172	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2173}
2174
2175static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2176{
2177	/* Calculate the display mode Average Bandwidth
2178	 * DisplayMode should contain the source and destination dimensions,
2179	 * timing, etc.
2180	 */
2181	fixed20_12 bpp;
2182	fixed20_12 line_time;
2183	fixed20_12 src_width;
2184	fixed20_12 bandwidth;
2185	fixed20_12 a;
2186
2187	a.full = dfixed_const(1000);
2188	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2189	line_time.full = dfixed_div(line_time, a);
2190	bpp.full = dfixed_const(wm->bytes_per_pixel);
2191	src_width.full = dfixed_const(wm->src_width);
2192	bandwidth.full = dfixed_mul(src_width, bpp);
2193	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2194	bandwidth.full = dfixed_div(bandwidth, line_time);
2195
2196	return dfixed_trunc(bandwidth);
2197}
2198
2199static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2200{
2201	/* First calcualte the latency in ns */
2202	u32 mc_latency = 2000; /* 2000 ns. */
2203	u32 available_bandwidth = dce6_available_bandwidth(wm);
2204	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2205	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2206	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2207	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2208		(wm->num_heads * cursor_line_pair_return_time);
2209	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2210	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2211	u32 tmp, dmif_size = 12288;
2212	fixed20_12 a, b, c;
2213
2214	if (wm->num_heads == 0)
2215		return 0;
2216
2217	a.full = dfixed_const(2);
2218	b.full = dfixed_const(1);
2219	if ((wm->vsc.full > a.full) ||
2220	    ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2221	    (wm->vtaps >= 5) ||
2222	    ((wm->vsc.full >= a.full) && wm->interlaced))
2223		max_src_lines_per_dst_line = 4;
2224	else
2225		max_src_lines_per_dst_line = 2;
2226
2227	a.full = dfixed_const(available_bandwidth);
2228	b.full = dfixed_const(wm->num_heads);
2229	a.full = dfixed_div(a, b);
2230	tmp = div_u64((u64) dmif_size * (u64) wm->disp_clk, mc_latency + 512);
2231	tmp = min(dfixed_trunc(a), tmp);
2232
2233	lb_fill_bw = min(tmp, wm->disp_clk * wm->bytes_per_pixel / 1000);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2234
2235	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2236	b.full = dfixed_const(1000);
2237	c.full = dfixed_const(lb_fill_bw);
2238	b.full = dfixed_div(c, b);
2239	a.full = dfixed_div(a, b);
2240	line_fill_time = dfixed_trunc(a);
2241
2242	if (line_fill_time < wm->active_time)
2243		return latency;
2244	else
2245		return latency + (line_fill_time - wm->active_time);
2246
2247}
2248
2249static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2250{
2251	if (dce6_average_bandwidth(wm) <=
2252	    (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2253		return true;
2254	else
2255		return false;
2256};
2257
2258static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2259{
2260	if (dce6_average_bandwidth(wm) <=
2261	    (dce6_available_bandwidth(wm) / wm->num_heads))
2262		return true;
2263	else
2264		return false;
2265};
2266
2267static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2268{
2269	u32 lb_partitions = wm->lb_size / wm->src_width;
2270	u32 line_time = wm->active_time + wm->blank_time;
2271	u32 latency_tolerant_lines;
2272	u32 latency_hiding;
2273	fixed20_12 a;
2274
2275	a.full = dfixed_const(1);
2276	if (wm->vsc.full > a.full)
2277		latency_tolerant_lines = 1;
2278	else {
2279		if (lb_partitions <= (wm->vtaps + 1))
2280			latency_tolerant_lines = 1;
2281		else
2282			latency_tolerant_lines = 2;
2283	}
2284
2285	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2286
2287	if (dce6_latency_watermark(wm) <= latency_hiding)
2288		return true;
2289	else
2290		return false;
2291}
2292
2293static void dce6_program_watermarks(struct radeon_device *rdev,
2294					 struct radeon_crtc *radeon_crtc,
2295					 u32 lb_size, u32 num_heads)
2296{
2297	struct drm_display_mode *mode = &radeon_crtc->base.mode;
2298	struct dce6_wm_params wm_low, wm_high;
2299	u32 dram_channels;
2300	u32 active_time;
2301	u32 line_time = 0;
2302	u32 latency_watermark_a = 0, latency_watermark_b = 0;
2303	u32 priority_a_mark = 0, priority_b_mark = 0;
2304	u32 priority_a_cnt = PRIORITY_OFF;
2305	u32 priority_b_cnt = PRIORITY_OFF;
2306	u32 tmp, arb_control3;
2307	fixed20_12 a, b, c;
2308
2309	if (radeon_crtc->base.enabled && num_heads && mode) {
2310		active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2311					    (u32)mode->clock);
2312		line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2313					  (u32)mode->clock);
2314		line_time = min(line_time, (u32)65535);
2315		priority_a_cnt = 0;
2316		priority_b_cnt = 0;
2317
2318		if (rdev->family == CHIP_ARUBA)
2319			dram_channels = evergreen_get_number_of_dram_channels(rdev);
2320		else
2321			dram_channels = si_get_number_of_dram_channels(rdev);
2322
2323		/* watermark for high clocks */
2324		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2325			wm_high.yclk =
2326				radeon_dpm_get_mclk(rdev, false) * 10;
2327			wm_high.sclk =
2328				radeon_dpm_get_sclk(rdev, false) * 10;
2329		} else {
2330			wm_high.yclk = rdev->pm.current_mclk * 10;
2331			wm_high.sclk = rdev->pm.current_sclk * 10;
2332		}
2333
2334		wm_high.disp_clk = mode->clock;
2335		wm_high.src_width = mode->crtc_hdisplay;
2336		wm_high.active_time = active_time;
2337		wm_high.blank_time = line_time - wm_high.active_time;
2338		wm_high.interlaced = false;
2339		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2340			wm_high.interlaced = true;
2341		wm_high.vsc = radeon_crtc->vsc;
2342		wm_high.vtaps = 1;
2343		if (radeon_crtc->rmx_type != RMX_OFF)
2344			wm_high.vtaps = 2;
2345		wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2346		wm_high.lb_size = lb_size;
2347		wm_high.dram_channels = dram_channels;
2348		wm_high.num_heads = num_heads;
2349
2350		/* watermark for low clocks */
2351		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2352			wm_low.yclk =
2353				radeon_dpm_get_mclk(rdev, true) * 10;
2354			wm_low.sclk =
2355				radeon_dpm_get_sclk(rdev, true) * 10;
2356		} else {
2357			wm_low.yclk = rdev->pm.current_mclk * 10;
2358			wm_low.sclk = rdev->pm.current_sclk * 10;
2359		}
2360
2361		wm_low.disp_clk = mode->clock;
2362		wm_low.src_width = mode->crtc_hdisplay;
2363		wm_low.active_time = active_time;
2364		wm_low.blank_time = line_time - wm_low.active_time;
2365		wm_low.interlaced = false;
2366		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2367			wm_low.interlaced = true;
2368		wm_low.vsc = radeon_crtc->vsc;
2369		wm_low.vtaps = 1;
2370		if (radeon_crtc->rmx_type != RMX_OFF)
2371			wm_low.vtaps = 2;
2372		wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2373		wm_low.lb_size = lb_size;
2374		wm_low.dram_channels = dram_channels;
2375		wm_low.num_heads = num_heads;
2376
2377		/* set for high clocks */
2378		latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2379		/* set for low clocks */
2380		latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2381
2382		/* possibly force display priority to high */
2383		/* should really do this at mode validation time... */
2384		if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2385		    !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2386		    !dce6_check_latency_hiding(&wm_high) ||
2387		    (rdev->disp_priority == 2)) {
2388			DRM_DEBUG_KMS("force priority to high\n");
2389			priority_a_cnt |= PRIORITY_ALWAYS_ON;
2390			priority_b_cnt |= PRIORITY_ALWAYS_ON;
2391		}
2392		if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2393		    !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2394		    !dce6_check_latency_hiding(&wm_low) ||
2395		    (rdev->disp_priority == 2)) {
2396			DRM_DEBUG_KMS("force priority to high\n");
2397			priority_a_cnt |= PRIORITY_ALWAYS_ON;
2398			priority_b_cnt |= PRIORITY_ALWAYS_ON;
2399		}
2400
2401		a.full = dfixed_const(1000);
2402		b.full = dfixed_const(mode->clock);
2403		b.full = dfixed_div(b, a);
2404		c.full = dfixed_const(latency_watermark_a);
2405		c.full = dfixed_mul(c, b);
2406		c.full = dfixed_mul(c, radeon_crtc->hsc);
2407		c.full = dfixed_div(c, a);
2408		a.full = dfixed_const(16);
2409		c.full = dfixed_div(c, a);
2410		priority_a_mark = dfixed_trunc(c);
2411		priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2412
2413		a.full = dfixed_const(1000);
2414		b.full = dfixed_const(mode->clock);
2415		b.full = dfixed_div(b, a);
2416		c.full = dfixed_const(latency_watermark_b);
2417		c.full = dfixed_mul(c, b);
2418		c.full = dfixed_mul(c, radeon_crtc->hsc);
2419		c.full = dfixed_div(c, a);
2420		a.full = dfixed_const(16);
2421		c.full = dfixed_div(c, a);
2422		priority_b_mark = dfixed_trunc(c);
2423		priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2424
2425		/* Save number of lines the linebuffer leads before the scanout */
2426		radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2427	}
2428
2429	/* select wm A */
2430	arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2431	tmp = arb_control3;
2432	tmp &= ~LATENCY_WATERMARK_MASK(3);
2433	tmp |= LATENCY_WATERMARK_MASK(1);
2434	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2435	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2436	       (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2437		LATENCY_HIGH_WATERMARK(line_time)));
2438	/* select wm B */
2439	tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2440	tmp &= ~LATENCY_WATERMARK_MASK(3);
2441	tmp |= LATENCY_WATERMARK_MASK(2);
2442	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2443	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2444	       (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2445		LATENCY_HIGH_WATERMARK(line_time)));
2446	/* restore original selection */
2447	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2448
2449	/* write the priority marks */
2450	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2451	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2452
2453	/* save values for DPM */
2454	radeon_crtc->line_time = line_time;
2455	radeon_crtc->wm_high = latency_watermark_a;
2456	radeon_crtc->wm_low = latency_watermark_b;
2457}
2458
2459void dce6_bandwidth_update(struct radeon_device *rdev)
2460{
2461	struct drm_display_mode *mode0 = NULL;
2462	struct drm_display_mode *mode1 = NULL;
2463	u32 num_heads = 0, lb_size;
2464	int i;
2465
2466	if (!rdev->mode_info.mode_config_initialized)
2467		return;
2468
2469	radeon_update_display_priority(rdev);
2470
2471	for (i = 0; i < rdev->num_crtc; i++) {
2472		if (rdev->mode_info.crtcs[i]->base.enabled)
2473			num_heads++;
2474	}
2475	for (i = 0; i < rdev->num_crtc; i += 2) {
2476		mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2477		mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2478		lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2479		dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2480		lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2481		dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2482	}
2483}
2484
2485/*
2486 * Core functions
2487 */
2488static void si_tiling_mode_table_init(struct radeon_device *rdev)
2489{
2490	u32 *tile = rdev->config.si.tile_mode_array;
2491	const u32 num_tile_mode_states =
2492			ARRAY_SIZE(rdev->config.si.tile_mode_array);
2493	u32 reg_offset, split_equal_to_row_size;
2494
2495	switch (rdev->config.si.mem_row_size_in_kb) {
2496	case 1:
2497		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2498		break;
2499	case 2:
2500	default:
2501		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2502		break;
2503	case 4:
2504		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2505		break;
2506	}
2507
2508	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2509		tile[reg_offset] = 0;
2510
2511	switch(rdev->family) {
2512	case CHIP_TAHITI:
2513	case CHIP_PITCAIRN:
2514		/* non-AA compressed depth or any compressed stencil */
2515		tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2516			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2517			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2518			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2519			   NUM_BANKS(ADDR_SURF_16_BANK) |
2520			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2521			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2522			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2523		/* 2xAA/4xAA compressed depth only */
2524		tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2525			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2526			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2527			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2528			   NUM_BANKS(ADDR_SURF_16_BANK) |
2529			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2530			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2531			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2532		/* 8xAA compressed depth only */
2533		tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2534			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2535			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2536			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2537			   NUM_BANKS(ADDR_SURF_16_BANK) |
2538			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2539			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2540			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2541		/* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2542		tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2543			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2544			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2545			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2546			   NUM_BANKS(ADDR_SURF_16_BANK) |
2547			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2548			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2549			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2550		/* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2551		tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2552			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2553			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2554			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2555			   NUM_BANKS(ADDR_SURF_16_BANK) |
2556			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2557			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2558			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2559		/* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2560		tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2561			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2562			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2563			   TILE_SPLIT(split_equal_to_row_size) |
2564			   NUM_BANKS(ADDR_SURF_16_BANK) |
2565			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2566			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2567			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2568		/* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2569		tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2570			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2571			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2572			   TILE_SPLIT(split_equal_to_row_size) |
2573			   NUM_BANKS(ADDR_SURF_16_BANK) |
2574			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2575			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2576			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2577		/* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2578		tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2579			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2580			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2581			   TILE_SPLIT(split_equal_to_row_size) |
2582			   NUM_BANKS(ADDR_SURF_16_BANK) |
2583			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2584			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2585			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2586		/* 1D and 1D Array Surfaces */
2587		tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2588			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2589			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2590			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2591			   NUM_BANKS(ADDR_SURF_16_BANK) |
2592			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2593			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2594			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2595		/* Displayable maps. */
2596		tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2597			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2598			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2599			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2600			   NUM_BANKS(ADDR_SURF_16_BANK) |
2601			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2602			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2603			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2604		/* Display 8bpp. */
2605		tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2606			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2607			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2608			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2609			   NUM_BANKS(ADDR_SURF_16_BANK) |
2610			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2611			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2612			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2613		/* Display 16bpp. */
2614		tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2615			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2616			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2617			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2618			   NUM_BANKS(ADDR_SURF_16_BANK) |
2619			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2620			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2621			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2622		/* Display 32bpp. */
2623		tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2624			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2625			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2626			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2627			   NUM_BANKS(ADDR_SURF_16_BANK) |
2628			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2629			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2630			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2631		/* Thin. */
2632		tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2633			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2634			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2635			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2636			   NUM_BANKS(ADDR_SURF_16_BANK) |
2637			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2638			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2639			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2640		/* Thin 8 bpp. */
2641		tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2643			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2644			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2645			   NUM_BANKS(ADDR_SURF_16_BANK) |
2646			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2647			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2648			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2649		/* Thin 16 bpp. */
2650		tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2651			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2652			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2653			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2654			   NUM_BANKS(ADDR_SURF_16_BANK) |
2655			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2656			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2657			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2658		/* Thin 32 bpp. */
2659		tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2660			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2661			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2662			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2663			   NUM_BANKS(ADDR_SURF_16_BANK) |
2664			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2665			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2666			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2667		/* Thin 64 bpp. */
2668		tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2669			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2670			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2671			   TILE_SPLIT(split_equal_to_row_size) |
2672			   NUM_BANKS(ADDR_SURF_16_BANK) |
2673			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2674			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2675			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2676		/* 8 bpp PRT. */
2677		tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2678			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2679			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2680			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2681			   NUM_BANKS(ADDR_SURF_16_BANK) |
2682			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2683			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2684			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2685		/* 16 bpp PRT */
2686		tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2687			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2688			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2689			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2690			   NUM_BANKS(ADDR_SURF_16_BANK) |
2691			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2692			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2693			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2694		/* 32 bpp PRT */
2695		tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2696			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2697			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2698			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2699			   NUM_BANKS(ADDR_SURF_16_BANK) |
2700			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2701			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2702			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2703		/* 64 bpp PRT */
2704		tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2705			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2706			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2707			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2708			   NUM_BANKS(ADDR_SURF_16_BANK) |
2709			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2710			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2711			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2712		/* 128 bpp PRT */
2713		tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2714			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2715			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2716			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2717			   NUM_BANKS(ADDR_SURF_8_BANK) |
2718			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2719			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2720			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2721
2722		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2723			WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2724		break;
2725
2726	case CHIP_VERDE:
2727	case CHIP_OLAND:
2728	case CHIP_HAINAN:
2729		/* non-AA compressed depth or any compressed stencil */
2730		tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2731			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2732			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2733			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2734			   NUM_BANKS(ADDR_SURF_16_BANK) |
2735			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2736			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2737			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2738		/* 2xAA/4xAA compressed depth only */
2739		tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2740			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2741			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2742			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2743			   NUM_BANKS(ADDR_SURF_16_BANK) |
2744			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2745			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2746			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2747		/* 8xAA compressed depth only */
2748		tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2749			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2750			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2751			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2752			   NUM_BANKS(ADDR_SURF_16_BANK) |
2753			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2754			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2755			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2756		/* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2757		tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2758			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2759			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2760			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2761			   NUM_BANKS(ADDR_SURF_16_BANK) |
2762			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2763			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2764			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2765		/* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2766		tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2767			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2768			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2769			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2770			   NUM_BANKS(ADDR_SURF_16_BANK) |
2771			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2772			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2773			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2774		/* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2775		tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2776			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2777			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2778			   TILE_SPLIT(split_equal_to_row_size) |
2779			   NUM_BANKS(ADDR_SURF_16_BANK) |
2780			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2781			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2782			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2783		/* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2784		tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2785			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2786			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2787			   TILE_SPLIT(split_equal_to_row_size) |
2788			   NUM_BANKS(ADDR_SURF_16_BANK) |
2789			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2790			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2791			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2792		/* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2793		tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2794			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2795			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2796			   TILE_SPLIT(split_equal_to_row_size) |
2797			   NUM_BANKS(ADDR_SURF_16_BANK) |
2798			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2799			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2800			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2801		/* 1D and 1D Array Surfaces */
2802		tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2803			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2804			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2805			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2806			   NUM_BANKS(ADDR_SURF_16_BANK) |
2807			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2808			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2809			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2810		/* Displayable maps. */
2811		tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2812			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2813			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2815			   NUM_BANKS(ADDR_SURF_16_BANK) |
2816			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2817			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2818			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2819		/* Display 8bpp. */
2820		tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2821			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2822			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2823			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2824			   NUM_BANKS(ADDR_SURF_16_BANK) |
2825			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2826			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2827			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2828		/* Display 16bpp. */
2829		tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2830			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2831			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2832			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2833			   NUM_BANKS(ADDR_SURF_16_BANK) |
2834			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2835			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2836			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2837		/* Display 32bpp. */
2838		tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2839			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2840			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2841			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2842			   NUM_BANKS(ADDR_SURF_16_BANK) |
2843			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2844			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2845			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2846		/* Thin. */
2847		tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2848			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2849			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2850			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2851			   NUM_BANKS(ADDR_SURF_16_BANK) |
2852			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2853			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2854			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2855		/* Thin 8 bpp. */
2856		tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2857			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2858			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2859			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2860			   NUM_BANKS(ADDR_SURF_16_BANK) |
2861			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2862			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2863			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2864		/* Thin 16 bpp. */
2865		tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2866			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2867			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2868			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2869			   NUM_BANKS(ADDR_SURF_16_BANK) |
2870			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2871			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2872			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2873		/* Thin 32 bpp. */
2874		tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2875			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2876			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2877			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2878			   NUM_BANKS(ADDR_SURF_16_BANK) |
2879			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2880			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2881			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2882		/* Thin 64 bpp. */
2883		tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2884			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2885			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2886			   TILE_SPLIT(split_equal_to_row_size) |
2887			   NUM_BANKS(ADDR_SURF_16_BANK) |
2888			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2889			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2890			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2891		/* 8 bpp PRT. */
2892		tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2893			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2894			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2895			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2896			   NUM_BANKS(ADDR_SURF_16_BANK) |
2897			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2898			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2899			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2900		/* 16 bpp PRT */
2901		tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2902			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2903			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2904			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2905			   NUM_BANKS(ADDR_SURF_16_BANK) |
2906			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2907			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2908			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2909		/* 32 bpp PRT */
2910		tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2911			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2912			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2913			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2914			   NUM_BANKS(ADDR_SURF_16_BANK) |
2915			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2916			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2917			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2918		/* 64 bpp PRT */
2919		tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2920			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2921			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2922			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2923			   NUM_BANKS(ADDR_SURF_16_BANK) |
2924			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2925			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2926			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2927		/* 128 bpp PRT */
2928		tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2929			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2930			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2931			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2932			   NUM_BANKS(ADDR_SURF_8_BANK) |
2933			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2934			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2935			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2936
2937		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2938			WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2939		break;
2940
2941	default:
2942		DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2943	}
2944}
2945
2946static void si_select_se_sh(struct radeon_device *rdev,
2947			    u32 se_num, u32 sh_num)
2948{
2949	u32 data = INSTANCE_BROADCAST_WRITES;
2950
2951	if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2952		data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2953	else if (se_num == 0xffffffff)
2954		data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2955	else if (sh_num == 0xffffffff)
2956		data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2957	else
2958		data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2959	WREG32(GRBM_GFX_INDEX, data);
2960}
2961
2962static u32 si_create_bitmask(u32 bit_width)
2963{
2964	u32 i, mask = 0;
2965
2966	for (i = 0; i < bit_width; i++) {
2967		mask <<= 1;
2968		mask |= 1;
2969	}
2970	return mask;
2971}
2972
2973static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2974{
2975	u32 data, mask;
2976
2977	data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2978	if (data & 1)
2979		data &= INACTIVE_CUS_MASK;
2980	else
2981		data = 0;
2982	data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2983
2984	data >>= INACTIVE_CUS_SHIFT;
2985
2986	mask = si_create_bitmask(cu_per_sh);
2987
2988	return ~data & mask;
2989}
2990
2991static void si_setup_spi(struct radeon_device *rdev,
2992			 u32 se_num, u32 sh_per_se,
2993			 u32 cu_per_sh)
2994{
2995	int i, j, k;
2996	u32 data, mask, active_cu;
2997
2998	for (i = 0; i < se_num; i++) {
2999		for (j = 0; j < sh_per_se; j++) {
3000			si_select_se_sh(rdev, i, j);
3001			data = RREG32(SPI_STATIC_THREAD_MGMT_3);
3002			active_cu = si_get_cu_enabled(rdev, cu_per_sh);
3003
3004			mask = 1;
3005			for (k = 0; k < 16; k++) {
3006				mask <<= k;
3007				if (active_cu & mask) {
3008					data &= ~mask;
3009					WREG32(SPI_STATIC_THREAD_MGMT_3, data);
3010					break;
3011				}
3012			}
3013		}
3014	}
3015	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3016}
3017
3018static u32 si_get_rb_disabled(struct radeon_device *rdev,
3019			      u32 max_rb_num_per_se,
3020			      u32 sh_per_se)
3021{
3022	u32 data, mask;
3023
3024	data = RREG32(CC_RB_BACKEND_DISABLE);
3025	if (data & 1)
3026		data &= BACKEND_DISABLE_MASK;
3027	else
3028		data = 0;
3029	data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3030
3031	data >>= BACKEND_DISABLE_SHIFT;
3032
3033	mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
3034
3035	return data & mask;
3036}
3037
3038static void si_setup_rb(struct radeon_device *rdev,
3039			u32 se_num, u32 sh_per_se,
3040			u32 max_rb_num_per_se)
3041{
3042	int i, j;
3043	u32 data, mask;
3044	u32 disabled_rbs = 0;
3045	u32 enabled_rbs = 0;
3046
3047	for (i = 0; i < se_num; i++) {
3048		for (j = 0; j < sh_per_se; j++) {
3049			si_select_se_sh(rdev, i, j);
3050			data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3051			disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
3052		}
3053	}
3054	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3055
3056	mask = 1;
3057	for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3058		if (!(disabled_rbs & mask))
3059			enabled_rbs |= mask;
3060		mask <<= 1;
3061	}
3062
3063	rdev->config.si.backend_enable_mask = enabled_rbs;
3064
3065	for (i = 0; i < se_num; i++) {
3066		si_select_se_sh(rdev, i, 0xffffffff);
3067		data = 0;
3068		for (j = 0; j < sh_per_se; j++) {
3069			switch (enabled_rbs & 3) {
3070			case 1:
3071				data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3072				break;
3073			case 2:
3074				data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3075				break;
3076			case 3:
3077			default:
3078				data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3079				break;
3080			}
3081			enabled_rbs >>= 2;
3082		}
3083		WREG32(PA_SC_RASTER_CONFIG, data);
3084	}
3085	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3086}
3087
3088static void si_gpu_init(struct radeon_device *rdev)
3089{
3090	u32 gb_addr_config = 0;
3091	u32 mc_arb_ramcfg;
3092	u32 sx_debug_1;
3093	u32 hdp_host_path_cntl;
3094	u32 tmp;
3095	int i, j;
3096
3097	switch (rdev->family) {
3098	case CHIP_TAHITI:
3099		rdev->config.si.max_shader_engines = 2;
3100		rdev->config.si.max_tile_pipes = 12;
3101		rdev->config.si.max_cu_per_sh = 8;
3102		rdev->config.si.max_sh_per_se = 2;
3103		rdev->config.si.max_backends_per_se = 4;
3104		rdev->config.si.max_texture_channel_caches = 12;
3105		rdev->config.si.max_gprs = 256;
3106		rdev->config.si.max_gs_threads = 32;
3107		rdev->config.si.max_hw_contexts = 8;
3108
3109		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3110		rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3111		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3112		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3113		gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3114		break;
3115	case CHIP_PITCAIRN:
3116		rdev->config.si.max_shader_engines = 2;
3117		rdev->config.si.max_tile_pipes = 8;
3118		rdev->config.si.max_cu_per_sh = 5;
3119		rdev->config.si.max_sh_per_se = 2;
3120		rdev->config.si.max_backends_per_se = 4;
3121		rdev->config.si.max_texture_channel_caches = 8;
3122		rdev->config.si.max_gprs = 256;
3123		rdev->config.si.max_gs_threads = 32;
3124		rdev->config.si.max_hw_contexts = 8;
3125
3126		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3127		rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3128		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3129		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3130		gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3131		break;
3132	case CHIP_VERDE:
3133	default:
3134		rdev->config.si.max_shader_engines = 1;
3135		rdev->config.si.max_tile_pipes = 4;
3136		rdev->config.si.max_cu_per_sh = 5;
3137		rdev->config.si.max_sh_per_se = 2;
3138		rdev->config.si.max_backends_per_se = 4;
3139		rdev->config.si.max_texture_channel_caches = 4;
3140		rdev->config.si.max_gprs = 256;
3141		rdev->config.si.max_gs_threads = 32;
3142		rdev->config.si.max_hw_contexts = 8;
3143
3144		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3145		rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3146		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3147		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3148		gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3149		break;
3150	case CHIP_OLAND:
3151		rdev->config.si.max_shader_engines = 1;
3152		rdev->config.si.max_tile_pipes = 4;
3153		rdev->config.si.max_cu_per_sh = 6;
3154		rdev->config.si.max_sh_per_se = 1;
3155		rdev->config.si.max_backends_per_se = 2;
3156		rdev->config.si.max_texture_channel_caches = 4;
3157		rdev->config.si.max_gprs = 256;
3158		rdev->config.si.max_gs_threads = 16;
3159		rdev->config.si.max_hw_contexts = 8;
3160
3161		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3162		rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3163		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3164		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3165		gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3166		break;
3167	case CHIP_HAINAN:
3168		rdev->config.si.max_shader_engines = 1;
3169		rdev->config.si.max_tile_pipes = 4;
3170		rdev->config.si.max_cu_per_sh = 5;
3171		rdev->config.si.max_sh_per_se = 1;
3172		rdev->config.si.max_backends_per_se = 1;
3173		rdev->config.si.max_texture_channel_caches = 2;
3174		rdev->config.si.max_gprs = 256;
3175		rdev->config.si.max_gs_threads = 16;
3176		rdev->config.si.max_hw_contexts = 8;
3177
3178		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3179		rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3180		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3181		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3182		gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3183		break;
3184	}
3185
3186	/* Initialize HDP */
3187	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3188		WREG32((0x2c14 + j), 0x00000000);
3189		WREG32((0x2c18 + j), 0x00000000);
3190		WREG32((0x2c1c + j), 0x00000000);
3191		WREG32((0x2c20 + j), 0x00000000);
3192		WREG32((0x2c24 + j), 0x00000000);
3193	}
3194
3195	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3196	WREG32(SRBM_INT_CNTL, 1);
3197	WREG32(SRBM_INT_ACK, 1);
3198
3199	evergreen_fix_pci_max_read_req_size(rdev);
3200
3201	WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3202
3203	RREG32(MC_SHARED_CHMAP);
3204	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3205
3206	rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3207	rdev->config.si.mem_max_burst_length_bytes = 256;
3208	tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3209	rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3210	if (rdev->config.si.mem_row_size_in_kb > 4)
3211		rdev->config.si.mem_row_size_in_kb = 4;
3212	/* XXX use MC settings? */
3213	rdev->config.si.shader_engine_tile_size = 32;
3214	rdev->config.si.num_gpus = 1;
3215	rdev->config.si.multi_gpu_tile_size = 64;
3216
3217	/* fix up row size */
3218	gb_addr_config &= ~ROW_SIZE_MASK;
3219	switch (rdev->config.si.mem_row_size_in_kb) {
3220	case 1:
3221	default:
3222		gb_addr_config |= ROW_SIZE(0);
3223		break;
3224	case 2:
3225		gb_addr_config |= ROW_SIZE(1);
3226		break;
3227	case 4:
3228		gb_addr_config |= ROW_SIZE(2);
3229		break;
3230	}
3231
3232	/* setup tiling info dword.  gb_addr_config is not adequate since it does
3233	 * not have bank info, so create a custom tiling dword.
3234	 * bits 3:0   num_pipes
3235	 * bits 7:4   num_banks
3236	 * bits 11:8  group_size
3237	 * bits 15:12 row_size
3238	 */
3239	rdev->config.si.tile_config = 0;
3240	switch (rdev->config.si.num_tile_pipes) {
3241	case 1:
3242		rdev->config.si.tile_config |= (0 << 0);
3243		break;
3244	case 2:
3245		rdev->config.si.tile_config |= (1 << 0);
3246		break;
3247	case 4:
3248		rdev->config.si.tile_config |= (2 << 0);
3249		break;
3250	case 8:
3251	default:
3252		/* XXX what about 12? */
3253		rdev->config.si.tile_config |= (3 << 0);
3254		break;
3255	}
3256	switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3257	case 0: /* four banks */
3258		rdev->config.si.tile_config |= 0 << 4;
3259		break;
3260	case 1: /* eight banks */
3261		rdev->config.si.tile_config |= 1 << 4;
3262		break;
3263	case 2: /* sixteen banks */
3264	default:
3265		rdev->config.si.tile_config |= 2 << 4;
3266		break;
3267	}
3268	rdev->config.si.tile_config |=
3269		((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3270	rdev->config.si.tile_config |=
3271		((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3272
3273	WREG32(GB_ADDR_CONFIG, gb_addr_config);
3274	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3275	WREG32(DMIF_ADDR_CALC, gb_addr_config);
3276	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3277	WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3278	WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3279	if (rdev->has_uvd) {
3280		WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3281		WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3282		WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3283	}
3284
3285	si_tiling_mode_table_init(rdev);
3286
3287	si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3288		    rdev->config.si.max_sh_per_se,
3289		    rdev->config.si.max_backends_per_se);
3290
3291	si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3292		     rdev->config.si.max_sh_per_se,
3293		     rdev->config.si.max_cu_per_sh);
3294
3295	rdev->config.si.active_cus = 0;
3296	for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3297		for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3298			rdev->config.si.active_cus +=
3299				hweight32(si_get_cu_active_bitmap(rdev, i, j));
3300		}
3301	}
3302
3303	/* set HW defaults for 3D engine */
3304	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3305				     ROQ_IB2_START(0x2b)));
3306	WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3307
3308	sx_debug_1 = RREG32(SX_DEBUG_1);
3309	WREG32(SX_DEBUG_1, sx_debug_1);
3310
3311	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3312
3313	WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3314				 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3315				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3316				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3317
3318	WREG32(VGT_NUM_INSTANCES, 1);
3319
3320	WREG32(CP_PERFMON_CNTL, 0);
3321
3322	WREG32(SQ_CONFIG, 0);
3323
3324	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3325					  FORCE_EOV_MAX_REZ_CNT(255)));
3326
3327	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3328	       AUTO_INVLD_EN(ES_AND_GS_AUTO));
3329
3330	WREG32(VGT_GS_VERTEX_REUSE, 16);
3331	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3332
3333	WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3334	WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3335	WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3336	WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3337	WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3338	WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3339	WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3340	WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3341
3342	tmp = RREG32(HDP_MISC_CNTL);
3343	tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3344	WREG32(HDP_MISC_CNTL, tmp);
3345
3346	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3347	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3348
3349	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3350
3351	udelay(50);
3352}
3353
3354/*
3355 * GPU scratch registers helpers function.
3356 */
3357static void si_scratch_init(struct radeon_device *rdev)
3358{
3359	int i;
3360
3361	rdev->scratch.num_reg = 7;
3362	rdev->scratch.reg_base = SCRATCH_REG0;
3363	for (i = 0; i < rdev->scratch.num_reg; i++) {
3364		rdev->scratch.free[i] = true;
3365		rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3366	}
3367}
3368
3369void si_fence_ring_emit(struct radeon_device *rdev,
3370			struct radeon_fence *fence)
3371{
3372	struct radeon_ring *ring = &rdev->ring[fence->ring];
3373	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3374
3375	/* flush read cache over gart */
3376	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3377	radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3378	radeon_ring_write(ring, 0);
3379	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3380	radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3381			  PACKET3_TC_ACTION_ENA |
3382			  PACKET3_SH_KCACHE_ACTION_ENA |
3383			  PACKET3_SH_ICACHE_ACTION_ENA);
3384	radeon_ring_write(ring, 0xFFFFFFFF);
3385	radeon_ring_write(ring, 0);
3386	radeon_ring_write(ring, 10); /* poll interval */
3387	/* EVENT_WRITE_EOP - flush caches, send int */
3388	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3389	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3390	radeon_ring_write(ring, lower_32_bits(addr));
3391	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3392	radeon_ring_write(ring, fence->seq);
3393	radeon_ring_write(ring, 0);
3394}
3395
3396/*
3397 * IB stuff
3398 */
3399void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3400{
3401	struct radeon_ring *ring = &rdev->ring[ib->ring];
3402	unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3403	u32 header;
3404
3405	if (ib->is_const_ib) {
3406		/* set switch buffer packet before const IB */
3407		radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3408		radeon_ring_write(ring, 0);
3409
3410		header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3411	} else {
3412		u32 next_rptr;
3413		if (ring->rptr_save_reg) {
3414			next_rptr = ring->wptr + 3 + 4 + 8;
3415			radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3416			radeon_ring_write(ring, ((ring->rptr_save_reg -
3417						  PACKET3_SET_CONFIG_REG_START) >> 2));
3418			radeon_ring_write(ring, next_rptr);
3419		} else if (rdev->wb.enabled) {
3420			next_rptr = ring->wptr + 5 + 4 + 8;
3421			radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3422			radeon_ring_write(ring, (1 << 8));
3423			radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3424			radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3425			radeon_ring_write(ring, next_rptr);
3426		}
3427
3428		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3429	}
3430
3431	radeon_ring_write(ring, header);
3432	radeon_ring_write(ring,
3433#ifdef __BIG_ENDIAN
3434			  (2 << 0) |
3435#endif
3436			  (ib->gpu_addr & 0xFFFFFFFC));
3437	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3438	radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
3439
3440	if (!ib->is_const_ib) {
3441		/* flush read cache over gart for this vmid */
3442		radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3443		radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3444		radeon_ring_write(ring, vm_id);
3445		radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3446		radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3447				  PACKET3_TC_ACTION_ENA |
3448				  PACKET3_SH_KCACHE_ACTION_ENA |
3449				  PACKET3_SH_ICACHE_ACTION_ENA);
3450		radeon_ring_write(ring, 0xFFFFFFFF);
3451		radeon_ring_write(ring, 0);
3452		radeon_ring_write(ring, 10); /* poll interval */
3453	}
3454}
3455
3456/*
3457 * CP.
3458 */
3459static void si_cp_enable(struct radeon_device *rdev, bool enable)
3460{
3461	if (enable)
3462		WREG32(CP_ME_CNTL, 0);
3463	else {
3464		if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3465			radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3466		WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3467		WREG32(SCRATCH_UMSK, 0);
3468		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3469		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3470		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3471	}
3472	udelay(50);
3473}
3474
3475static int si_cp_load_microcode(struct radeon_device *rdev)
3476{
3477	int i;
3478
3479	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3480		return -EINVAL;
3481
3482	si_cp_enable(rdev, false);
3483
3484	if (rdev->new_fw) {
3485		const struct gfx_firmware_header_v1_0 *pfp_hdr =
3486			(const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3487		const struct gfx_firmware_header_v1_0 *ce_hdr =
3488			(const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3489		const struct gfx_firmware_header_v1_0 *me_hdr =
3490			(const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3491		const __le32 *fw_data;
3492		u32 fw_size;
3493
3494		radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3495		radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3496		radeon_ucode_print_gfx_hdr(&me_hdr->header);
3497
3498		/* PFP */
3499		fw_data = (const __le32 *)
3500			(rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3501		fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3502		WREG32(CP_PFP_UCODE_ADDR, 0);
3503		for (i = 0; i < fw_size; i++)
3504			WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3505		WREG32(CP_PFP_UCODE_ADDR, 0);
3506
3507		/* CE */
3508		fw_data = (const __le32 *)
3509			(rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3510		fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3511		WREG32(CP_CE_UCODE_ADDR, 0);
3512		for (i = 0; i < fw_size; i++)
3513			WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3514		WREG32(CP_CE_UCODE_ADDR, 0);
3515
3516		/* ME */
3517		fw_data = (const __be32 *)
3518			(rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3519		fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3520		WREG32(CP_ME_RAM_WADDR, 0);
3521		for (i = 0; i < fw_size; i++)
3522			WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3523		WREG32(CP_ME_RAM_WADDR, 0);
3524	} else {
3525		const __be32 *fw_data;
3526
3527		/* PFP */
3528		fw_data = (const __be32 *)rdev->pfp_fw->data;
3529		WREG32(CP_PFP_UCODE_ADDR, 0);
3530		for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3531			WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3532		WREG32(CP_PFP_UCODE_ADDR, 0);
3533
3534		/* CE */
3535		fw_data = (const __be32 *)rdev->ce_fw->data;
3536		WREG32(CP_CE_UCODE_ADDR, 0);
3537		for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3538			WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3539		WREG32(CP_CE_UCODE_ADDR, 0);
3540
3541		/* ME */
3542		fw_data = (const __be32 *)rdev->me_fw->data;
3543		WREG32(CP_ME_RAM_WADDR, 0);
3544		for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3545			WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3546		WREG32(CP_ME_RAM_WADDR, 0);
3547	}
3548
3549	WREG32(CP_PFP_UCODE_ADDR, 0);
3550	WREG32(CP_CE_UCODE_ADDR, 0);
3551	WREG32(CP_ME_RAM_WADDR, 0);
3552	WREG32(CP_ME_RAM_RADDR, 0);
3553	return 0;
3554}
3555
3556static int si_cp_start(struct radeon_device *rdev)
3557{
3558	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3559	int r, i;
3560
3561	r = radeon_ring_lock(rdev, ring, 7 + 4);
3562	if (r) {
3563		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3564		return r;
3565	}
3566	/* init the CP */
3567	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3568	radeon_ring_write(ring, 0x1);
3569	radeon_ring_write(ring, 0x0);
3570	radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3571	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3572	radeon_ring_write(ring, 0);
3573	radeon_ring_write(ring, 0);
3574
3575	/* init the CE partitions */
3576	radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3577	radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3578	radeon_ring_write(ring, 0xc000);
3579	radeon_ring_write(ring, 0xe000);
3580	radeon_ring_unlock_commit(rdev, ring, false);
3581
3582	si_cp_enable(rdev, true);
3583
3584	r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3585	if (r) {
3586		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3587		return r;
3588	}
3589
3590	/* setup clear context state */
3591	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3592	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3593
3594	for (i = 0; i < si_default_size; i++)
3595		radeon_ring_write(ring, si_default_state[i]);
3596
3597	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3598	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3599
3600	/* set clear context state */
3601	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3602	radeon_ring_write(ring, 0);
3603
3604	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3605	radeon_ring_write(ring, 0x00000316);
3606	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3607	radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3608
3609	radeon_ring_unlock_commit(rdev, ring, false);
3610
3611	for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3612		ring = &rdev->ring[i];
3613		r = radeon_ring_lock(rdev, ring, 2);
3614		if (r) {
3615			DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3616			return r;
3617		}
3618
3619		/* clear the compute context state */
3620		radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3621		radeon_ring_write(ring, 0);
3622
3623		radeon_ring_unlock_commit(rdev, ring, false);
3624	}
3625
3626	return 0;
3627}
3628
3629static void si_cp_fini(struct radeon_device *rdev)
3630{
3631	struct radeon_ring *ring;
3632	si_cp_enable(rdev, false);
3633
3634	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3635	radeon_ring_fini(rdev, ring);
3636	radeon_scratch_free(rdev, ring->rptr_save_reg);
3637
3638	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3639	radeon_ring_fini(rdev, ring);
3640	radeon_scratch_free(rdev, ring->rptr_save_reg);
3641
3642	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3643	radeon_ring_fini(rdev, ring);
3644	radeon_scratch_free(rdev, ring->rptr_save_reg);
3645}
3646
3647static int si_cp_resume(struct radeon_device *rdev)
3648{
3649	struct radeon_ring *ring;
3650	u32 tmp;
3651	u32 rb_bufsz;
3652	int r;
3653
3654	si_enable_gui_idle_interrupt(rdev, false);
3655
3656	WREG32(CP_SEM_WAIT_TIMER, 0x0);
3657	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3658
3659	/* Set the write pointer delay */
3660	WREG32(CP_RB_WPTR_DELAY, 0);
3661
3662	WREG32(CP_DEBUG, 0);
3663	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3664
3665	/* ring 0 - compute and gfx */
3666	/* Set ring buffer size */
3667	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3668	rb_bufsz = order_base_2(ring->ring_size / 8);
3669	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3670#ifdef __BIG_ENDIAN
3671	tmp |= BUF_SWAP_32BIT;
3672#endif
3673	WREG32(CP_RB0_CNTL, tmp);
3674
3675	/* Initialize the ring buffer's read and write pointers */
3676	WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3677	ring->wptr = 0;
3678	WREG32(CP_RB0_WPTR, ring->wptr);
3679
3680	/* set the wb address whether it's enabled or not */
3681	WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3682	WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3683
3684	if (rdev->wb.enabled)
3685		WREG32(SCRATCH_UMSK, 0xff);
3686	else {
3687		tmp |= RB_NO_UPDATE;
3688		WREG32(SCRATCH_UMSK, 0);
3689	}
3690
3691	mdelay(1);
3692	WREG32(CP_RB0_CNTL, tmp);
3693
3694	WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3695
3696	/* ring1  - compute only */
3697	/* Set ring buffer size */
3698	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3699	rb_bufsz = order_base_2(ring->ring_size / 8);
3700	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3701#ifdef __BIG_ENDIAN
3702	tmp |= BUF_SWAP_32BIT;
3703#endif
3704	WREG32(CP_RB1_CNTL, tmp);
3705
3706	/* Initialize the ring buffer's read and write pointers */
3707	WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3708	ring->wptr = 0;
3709	WREG32(CP_RB1_WPTR, ring->wptr);
3710
3711	/* set the wb address whether it's enabled or not */
3712	WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3713	WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3714
3715	mdelay(1);
3716	WREG32(CP_RB1_CNTL, tmp);
3717
3718	WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3719
3720	/* ring2 - compute only */
3721	/* Set ring buffer size */
3722	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3723	rb_bufsz = order_base_2(ring->ring_size / 8);
3724	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3725#ifdef __BIG_ENDIAN
3726	tmp |= BUF_SWAP_32BIT;
3727#endif
3728	WREG32(CP_RB2_CNTL, tmp);
3729
3730	/* Initialize the ring buffer's read and write pointers */
3731	WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3732	ring->wptr = 0;
3733	WREG32(CP_RB2_WPTR, ring->wptr);
3734
3735	/* set the wb address whether it's enabled or not */
3736	WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3737	WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3738
3739	mdelay(1);
3740	WREG32(CP_RB2_CNTL, tmp);
3741
3742	WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3743
3744	/* start the rings */
3745	si_cp_start(rdev);
3746	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3747	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3748	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3749	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3750	if (r) {
3751		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3752		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3753		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3754		return r;
3755	}
3756	r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3757	if (r) {
3758		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3759	}
3760	r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3761	if (r) {
3762		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3763	}
3764
3765	si_enable_gui_idle_interrupt(rdev, true);
3766
3767	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3768		radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3769
3770	return 0;
3771}
3772
3773u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3774{
3775	u32 reset_mask = 0;
3776	u32 tmp;
3777
3778	/* GRBM_STATUS */
3779	tmp = RREG32(GRBM_STATUS);
3780	if (tmp & (PA_BUSY | SC_BUSY |
3781		   BCI_BUSY | SX_BUSY |
3782		   TA_BUSY | VGT_BUSY |
3783		   DB_BUSY | CB_BUSY |
3784		   GDS_BUSY | SPI_BUSY |
3785		   IA_BUSY | IA_BUSY_NO_DMA))
3786		reset_mask |= RADEON_RESET_GFX;
3787
3788	if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3789		   CP_BUSY | CP_COHERENCY_BUSY))
3790		reset_mask |= RADEON_RESET_CP;
3791
3792	if (tmp & GRBM_EE_BUSY)
3793		reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3794
3795	/* GRBM_STATUS2 */
3796	tmp = RREG32(GRBM_STATUS2);
3797	if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3798		reset_mask |= RADEON_RESET_RLC;
3799
3800	/* DMA_STATUS_REG 0 */
3801	tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3802	if (!(tmp & DMA_IDLE))
3803		reset_mask |= RADEON_RESET_DMA;
3804
3805	/* DMA_STATUS_REG 1 */
3806	tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3807	if (!(tmp & DMA_IDLE))
3808		reset_mask |= RADEON_RESET_DMA1;
3809
3810	/* SRBM_STATUS2 */
3811	tmp = RREG32(SRBM_STATUS2);
3812	if (tmp & DMA_BUSY)
3813		reset_mask |= RADEON_RESET_DMA;
3814
3815	if (tmp & DMA1_BUSY)
3816		reset_mask |= RADEON_RESET_DMA1;
3817
3818	/* SRBM_STATUS */
3819	tmp = RREG32(SRBM_STATUS);
3820
3821	if (tmp & IH_BUSY)
3822		reset_mask |= RADEON_RESET_IH;
3823
3824	if (tmp & SEM_BUSY)
3825		reset_mask |= RADEON_RESET_SEM;
3826
3827	if (tmp & GRBM_RQ_PENDING)
3828		reset_mask |= RADEON_RESET_GRBM;
3829
3830	if (tmp & VMC_BUSY)
3831		reset_mask |= RADEON_RESET_VMC;
3832
3833	if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3834		   MCC_BUSY | MCD_BUSY))
3835		reset_mask |= RADEON_RESET_MC;
3836
3837	if (evergreen_is_display_hung(rdev))
3838		reset_mask |= RADEON_RESET_DISPLAY;
3839
3840	/* VM_L2_STATUS */
3841	tmp = RREG32(VM_L2_STATUS);
3842	if (tmp & L2_BUSY)
3843		reset_mask |= RADEON_RESET_VMC;
3844
3845	/* Skip MC reset as it's mostly likely not hung, just busy */
3846	if (reset_mask & RADEON_RESET_MC) {
3847		DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3848		reset_mask &= ~RADEON_RESET_MC;
3849	}
3850
3851	return reset_mask;
3852}
3853
3854static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3855{
3856	struct evergreen_mc_save save;
3857	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3858	u32 tmp;
3859
3860	if (reset_mask == 0)
3861		return;
3862
3863	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3864
3865	evergreen_print_gpu_status_regs(rdev);
3866	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
3867		 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3868	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3869		 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3870
3871	/* disable PG/CG */
3872	si_fini_pg(rdev);
3873	si_fini_cg(rdev);
3874
3875	/* stop the rlc */
3876	si_rlc_stop(rdev);
3877
3878	/* Disable CP parsing/prefetching */
3879	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3880
3881	if (reset_mask & RADEON_RESET_DMA) {
3882		/* dma0 */
3883		tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3884		tmp &= ~DMA_RB_ENABLE;
3885		WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3886	}
3887	if (reset_mask & RADEON_RESET_DMA1) {
3888		/* dma1 */
3889		tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3890		tmp &= ~DMA_RB_ENABLE;
3891		WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3892	}
3893
3894	udelay(50);
3895
3896	evergreen_mc_stop(rdev, &save);
3897	if (evergreen_mc_wait_for_idle(rdev)) {
3898		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3899	}
3900
3901	if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3902		grbm_soft_reset = SOFT_RESET_CB |
3903			SOFT_RESET_DB |
3904			SOFT_RESET_GDS |
3905			SOFT_RESET_PA |
3906			SOFT_RESET_SC |
3907			SOFT_RESET_BCI |
3908			SOFT_RESET_SPI |
3909			SOFT_RESET_SX |
3910			SOFT_RESET_TC |
3911			SOFT_RESET_TA |
3912			SOFT_RESET_VGT |
3913			SOFT_RESET_IA;
3914	}
3915
3916	if (reset_mask & RADEON_RESET_CP) {
3917		grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3918
3919		srbm_soft_reset |= SOFT_RESET_GRBM;
3920	}
3921
3922	if (reset_mask & RADEON_RESET_DMA)
3923		srbm_soft_reset |= SOFT_RESET_DMA;
3924
3925	if (reset_mask & RADEON_RESET_DMA1)
3926		srbm_soft_reset |= SOFT_RESET_DMA1;
3927
3928	if (reset_mask & RADEON_RESET_DISPLAY)
3929		srbm_soft_reset |= SOFT_RESET_DC;
3930
3931	if (reset_mask & RADEON_RESET_RLC)
3932		grbm_soft_reset |= SOFT_RESET_RLC;
3933
3934	if (reset_mask & RADEON_RESET_SEM)
3935		srbm_soft_reset |= SOFT_RESET_SEM;
3936
3937	if (reset_mask & RADEON_RESET_IH)
3938		srbm_soft_reset |= SOFT_RESET_IH;
3939
3940	if (reset_mask & RADEON_RESET_GRBM)
3941		srbm_soft_reset |= SOFT_RESET_GRBM;
3942
3943	if (reset_mask & RADEON_RESET_VMC)
3944		srbm_soft_reset |= SOFT_RESET_VMC;
3945
3946	if (reset_mask & RADEON_RESET_MC)
3947		srbm_soft_reset |= SOFT_RESET_MC;
3948
3949	if (grbm_soft_reset) {
3950		tmp = RREG32(GRBM_SOFT_RESET);
3951		tmp |= grbm_soft_reset;
3952		dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3953		WREG32(GRBM_SOFT_RESET, tmp);
3954		tmp = RREG32(GRBM_SOFT_RESET);
3955
3956		udelay(50);
3957
3958		tmp &= ~grbm_soft_reset;
3959		WREG32(GRBM_SOFT_RESET, tmp);
3960		tmp = RREG32(GRBM_SOFT_RESET);
3961	}
3962
3963	if (srbm_soft_reset) {
3964		tmp = RREG32(SRBM_SOFT_RESET);
3965		tmp |= srbm_soft_reset;
3966		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3967		WREG32(SRBM_SOFT_RESET, tmp);
3968		tmp = RREG32(SRBM_SOFT_RESET);
3969
3970		udelay(50);
3971
3972		tmp &= ~srbm_soft_reset;
3973		WREG32(SRBM_SOFT_RESET, tmp);
3974		tmp = RREG32(SRBM_SOFT_RESET);
3975	}
3976
3977	/* Wait a little for things to settle down */
3978	udelay(50);
3979
3980	evergreen_mc_resume(rdev, &save);
3981	udelay(50);
3982
3983	evergreen_print_gpu_status_regs(rdev);
3984}
3985
3986static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3987{
3988	u32 tmp, i;
3989
3990	tmp = RREG32(CG_SPLL_FUNC_CNTL);
3991	tmp |= SPLL_BYPASS_EN;
3992	WREG32(CG_SPLL_FUNC_CNTL, tmp);
3993
3994	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3995	tmp |= SPLL_CTLREQ_CHG;
3996	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3997
3998	for (i = 0; i < rdev->usec_timeout; i++) {
3999		if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
4000			break;
4001		udelay(1);
4002	}
4003
4004	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
4005	tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
4006	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
4007
4008	tmp = RREG32(MPLL_CNTL_MODE);
4009	tmp &= ~MPLL_MCLK_SEL;
4010	WREG32(MPLL_CNTL_MODE, tmp);
4011}
4012
4013static void si_spll_powerdown(struct radeon_device *rdev)
4014{
4015	u32 tmp;
4016
4017	tmp = RREG32(SPLL_CNTL_MODE);
4018	tmp |= SPLL_SW_DIR_CONTROL;
4019	WREG32(SPLL_CNTL_MODE, tmp);
4020
4021	tmp = RREG32(CG_SPLL_FUNC_CNTL);
4022	tmp |= SPLL_RESET;
4023	WREG32(CG_SPLL_FUNC_CNTL, tmp);
4024
4025	tmp = RREG32(CG_SPLL_FUNC_CNTL);
4026	tmp |= SPLL_SLEEP;
4027	WREG32(CG_SPLL_FUNC_CNTL, tmp);
4028
4029	tmp = RREG32(SPLL_CNTL_MODE);
4030	tmp &= ~SPLL_SW_DIR_CONTROL;
4031	WREG32(SPLL_CNTL_MODE, tmp);
4032}
4033
4034static void si_gpu_pci_config_reset(struct radeon_device *rdev)
4035{
4036	struct evergreen_mc_save save;
4037	u32 tmp, i;
4038
4039	dev_info(rdev->dev, "GPU pci config reset\n");
4040
4041	/* disable dpm? */
4042
4043	/* disable cg/pg */
4044	si_fini_pg(rdev);
4045	si_fini_cg(rdev);
4046
4047	/* Disable CP parsing/prefetching */
4048	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4049	/* dma0 */
4050	tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
4051	tmp &= ~DMA_RB_ENABLE;
4052	WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
4053	/* dma1 */
4054	tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
4055	tmp &= ~DMA_RB_ENABLE;
4056	WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
4057	/* XXX other engines? */
4058
4059	/* halt the rlc, disable cp internal ints */
4060	si_rlc_stop(rdev);
4061
4062	udelay(50);
4063
4064	/* disable mem access */
4065	evergreen_mc_stop(rdev, &save);
4066	if (evergreen_mc_wait_for_idle(rdev)) {
4067		dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4068	}
4069
4070	/* set mclk/sclk to bypass */
4071	si_set_clk_bypass_mode(rdev);
4072	/* powerdown spll */
4073	si_spll_powerdown(rdev);
4074	/* disable BM */
4075	pci_clear_master(rdev->pdev);
4076	/* reset */
4077	radeon_pci_config_reset(rdev);
4078	/* wait for asic to come out of reset */
4079	for (i = 0; i < rdev->usec_timeout; i++) {
4080		if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4081			break;
4082		udelay(1);
4083	}
4084}
4085
4086int si_asic_reset(struct radeon_device *rdev, bool hard)
4087{
4088	u32 reset_mask;
4089
4090	if (hard) {
4091		si_gpu_pci_config_reset(rdev);
4092		return 0;
4093	}
4094
4095	reset_mask = si_gpu_check_soft_reset(rdev);
4096
4097	if (reset_mask)
4098		r600_set_bios_scratch_engine_hung(rdev, true);
4099
4100	/* try soft reset */
4101	si_gpu_soft_reset(rdev, reset_mask);
4102
4103	reset_mask = si_gpu_check_soft_reset(rdev);
4104
4105	/* try pci config reset */
4106	if (reset_mask && radeon_hard_reset)
4107		si_gpu_pci_config_reset(rdev);
4108
4109	reset_mask = si_gpu_check_soft_reset(rdev);
4110
4111	if (!reset_mask)
4112		r600_set_bios_scratch_engine_hung(rdev, false);
4113
4114	return 0;
4115}
4116
4117/**
4118 * si_gfx_is_lockup - Check if the GFX engine is locked up
4119 *
4120 * @rdev: radeon_device pointer
4121 * @ring: radeon_ring structure holding ring information
4122 *
4123 * Check if the GFX engine is locked up.
4124 * Returns true if the engine appears to be locked up, false if not.
4125 */
4126bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4127{
4128	u32 reset_mask = si_gpu_check_soft_reset(rdev);
4129
4130	if (!(reset_mask & (RADEON_RESET_GFX |
4131			    RADEON_RESET_COMPUTE |
4132			    RADEON_RESET_CP))) {
4133		radeon_ring_lockup_update(rdev, ring);
4134		return false;
4135	}
4136	return radeon_ring_test_lockup(rdev, ring);
4137}
4138
4139/* MC */
4140static void si_mc_program(struct radeon_device *rdev)
4141{
4142	struct evergreen_mc_save save;
4143	u32 tmp;
4144	int i, j;
4145
4146	/* Initialize HDP */
4147	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4148		WREG32((0x2c14 + j), 0x00000000);
4149		WREG32((0x2c18 + j), 0x00000000);
4150		WREG32((0x2c1c + j), 0x00000000);
4151		WREG32((0x2c20 + j), 0x00000000);
4152		WREG32((0x2c24 + j), 0x00000000);
4153	}
4154	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4155
4156	evergreen_mc_stop(rdev, &save);
4157	if (radeon_mc_wait_for_idle(rdev)) {
4158		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4159	}
4160	if (!ASIC_IS_NODCE(rdev))
4161		/* Lockout access through VGA aperture*/
4162		WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4163	/* Update configuration */
4164	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4165	       rdev->mc.vram_start >> 12);
4166	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4167	       rdev->mc.vram_end >> 12);
4168	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4169	       rdev->vram_scratch.gpu_addr >> 12);
4170	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4171	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4172	WREG32(MC_VM_FB_LOCATION, tmp);
4173	/* XXX double check these! */
4174	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4175	WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4176	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4177	WREG32(MC_VM_AGP_BASE, 0);
4178	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4179	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4180	if (radeon_mc_wait_for_idle(rdev)) {
4181		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4182	}
4183	evergreen_mc_resume(rdev, &save);
4184	if (!ASIC_IS_NODCE(rdev)) {
4185		/* we need to own VRAM, so turn off the VGA renderer here
4186		 * to stop it overwriting our objects */
4187		rv515_vga_render_disable(rdev);
4188	}
4189}
4190
4191void si_vram_gtt_location(struct radeon_device *rdev,
4192			  struct radeon_mc *mc)
4193{
4194	if (mc->mc_vram_size > 0xFFC0000000ULL) {
4195		/* leave room for at least 1024M GTT */
4196		dev_warn(rdev->dev, "limiting VRAM\n");
4197		mc->real_vram_size = 0xFFC0000000ULL;
4198		mc->mc_vram_size = 0xFFC0000000ULL;
4199	}
4200	radeon_vram_location(rdev, &rdev->mc, 0);
4201	rdev->mc.gtt_base_align = 0;
4202	radeon_gtt_location(rdev, mc);
4203}
4204
4205static int si_mc_init(struct radeon_device *rdev)
4206{
4207	u32 tmp;
4208	int chansize, numchan;
4209
4210	/* Get VRAM informations */
4211	rdev->mc.vram_is_ddr = true;
4212	tmp = RREG32(MC_ARB_RAMCFG);
4213	if (tmp & CHANSIZE_OVERRIDE) {
4214		chansize = 16;
4215	} else if (tmp & CHANSIZE_MASK) {
4216		chansize = 64;
4217	} else {
4218		chansize = 32;
4219	}
4220	tmp = RREG32(MC_SHARED_CHMAP);
4221	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4222	case 0:
4223	default:
4224		numchan = 1;
4225		break;
4226	case 1:
4227		numchan = 2;
4228		break;
4229	case 2:
4230		numchan = 4;
4231		break;
4232	case 3:
4233		numchan = 8;
4234		break;
4235	case 4:
4236		numchan = 3;
4237		break;
4238	case 5:
4239		numchan = 6;
4240		break;
4241	case 6:
4242		numchan = 10;
4243		break;
4244	case 7:
4245		numchan = 12;
4246		break;
4247	case 8:
4248		numchan = 16;
4249		break;
4250	}
4251	rdev->mc.vram_width = numchan * chansize;
4252	/* Could aper size report 0 ? */
4253	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4254	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4255	/* size in MB on si */
4256	tmp = RREG32(CONFIG_MEMSIZE);
4257	/* some boards may have garbage in the upper 16 bits */
4258	if (tmp & 0xffff0000) {
4259		DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4260		if (tmp & 0xffff)
4261			tmp &= 0xffff;
4262	}
4263	rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4264	rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4265	rdev->mc.visible_vram_size = rdev->mc.aper_size;
4266	si_vram_gtt_location(rdev, &rdev->mc);
4267	radeon_update_bandwidth_info(rdev);
4268
4269	return 0;
4270}
4271
4272/*
4273 * GART
4274 */
4275void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4276{
4277	/* flush hdp cache */
4278	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4279
4280	/* bits 0-15 are the VM contexts0-15 */
4281	WREG32(VM_INVALIDATE_REQUEST, 1);
4282}
4283
4284static int si_pcie_gart_enable(struct radeon_device *rdev)
4285{
4286	int r, i;
4287
4288	if (rdev->gart.robj == NULL) {
4289		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4290		return -EINVAL;
4291	}
4292	r = radeon_gart_table_vram_pin(rdev);
4293	if (r)
4294		return r;
4295	/* Setup TLB control */
4296	WREG32(MC_VM_MX_L1_TLB_CNTL,
4297	       (0xA << 7) |
4298	       ENABLE_L1_TLB |
4299	       ENABLE_L1_FRAGMENT_PROCESSING |
4300	       SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4301	       ENABLE_ADVANCED_DRIVER_MODEL |
4302	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4303	/* Setup L2 cache */
4304	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4305	       ENABLE_L2_FRAGMENT_PROCESSING |
4306	       ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4307	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4308	       EFFECTIVE_L2_QUEUE_SIZE(7) |
4309	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
4310	WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4311	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4312	       BANK_SELECT(4) |
4313	       L2_CACHE_BIGK_FRAGMENT_SIZE(4));
4314	/* setup context0 */
4315	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4316	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4317	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4318	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4319			(u32)(rdev->dummy_page.addr >> 12));
4320	WREG32(VM_CONTEXT0_CNTL2, 0);
4321	WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4322				  RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4323
4324	WREG32(0x15D4, 0);
4325	WREG32(0x15D8, 0);
4326	WREG32(0x15DC, 0);
4327
4328	/* empty context1-15 */
4329	/* set vm size, must be a multiple of 4 */
4330	WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4331	WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
4332	/* Assign the pt base to something valid for now; the pts used for
4333	 * the VMs are determined by the application and setup and assigned
4334	 * on the fly in the vm part of radeon_gart.c
4335	 */
4336	for (i = 1; i < 16; i++) {
4337		if (i < 8)
4338			WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4339			       rdev->vm_manager.saved_table_addr[i]);
4340		else
4341			WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4342			       rdev->vm_manager.saved_table_addr[i]);
4343	}
4344
4345	/* enable context1-15 */
4346	WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4347	       (u32)(rdev->dummy_page.addr >> 12));
4348	WREG32(VM_CONTEXT1_CNTL2, 4);
4349	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4350				PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
4351				RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4352				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4353				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4354				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4355				PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4356				PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4357				VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4358				VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4359				READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4360				READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4361				WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4362				WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4363
4364	si_pcie_gart_tlb_flush(rdev);
4365	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4366		 (unsigned)(rdev->mc.gtt_size >> 20),
4367		 (unsigned long long)rdev->gart.table_addr);
4368	rdev->gart.ready = true;
4369	return 0;
4370}
4371
4372static void si_pcie_gart_disable(struct radeon_device *rdev)
4373{
4374	unsigned i;
4375
4376	for (i = 1; i < 16; ++i) {
4377		uint32_t reg;
4378		if (i < 8)
4379			reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4380		else
4381			reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4382		rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4383	}
4384
4385	/* Disable all tables */
4386	WREG32(VM_CONTEXT0_CNTL, 0);
4387	WREG32(VM_CONTEXT1_CNTL, 0);
4388	/* Setup TLB control */
4389	WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4390	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4391	/* Setup L2 cache */
4392	WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4393	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4394	       EFFECTIVE_L2_QUEUE_SIZE(7) |
4395	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
4396	WREG32(VM_L2_CNTL2, 0);
4397	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4398	       L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4399	radeon_gart_table_vram_unpin(rdev);
4400}
4401
4402static void si_pcie_gart_fini(struct radeon_device *rdev)
4403{
4404	si_pcie_gart_disable(rdev);
4405	radeon_gart_table_vram_free(rdev);
4406	radeon_gart_fini(rdev);
4407}
4408
4409/* vm parser */
4410static bool si_vm_reg_valid(u32 reg)
4411{
4412	/* context regs are fine */
4413	if (reg >= 0x28000)
4414		return true;
4415
4416	/* shader regs are also fine */
4417	if (reg >= 0xB000 && reg < 0xC000)
4418		return true;
4419
4420	/* check config regs */
4421	switch (reg) {
4422	case GRBM_GFX_INDEX:
4423	case CP_STRMOUT_CNTL:
4424	case VGT_VTX_VECT_EJECT_REG:
4425	case VGT_CACHE_INVALIDATION:
4426	case VGT_ESGS_RING_SIZE:
4427	case VGT_GSVS_RING_SIZE:
4428	case VGT_GS_VERTEX_REUSE:
4429	case VGT_PRIMITIVE_TYPE:
4430	case VGT_INDEX_TYPE:
4431	case VGT_NUM_INDICES:
4432	case VGT_NUM_INSTANCES:
4433	case VGT_TF_RING_SIZE:
4434	case VGT_HS_OFFCHIP_PARAM:
4435	case VGT_TF_MEMORY_BASE:
4436	case PA_CL_ENHANCE:
4437	case PA_SU_LINE_STIPPLE_VALUE:
4438	case PA_SC_LINE_STIPPLE_STATE:
4439	case PA_SC_ENHANCE:
4440	case SQC_CACHES:
4441	case SPI_STATIC_THREAD_MGMT_1:
4442	case SPI_STATIC_THREAD_MGMT_2:
4443	case SPI_STATIC_THREAD_MGMT_3:
4444	case SPI_PS_MAX_WAVE_ID:
4445	case SPI_CONFIG_CNTL:
4446	case SPI_CONFIG_CNTL_1:
4447	case TA_CNTL_AUX:
4448	case TA_CS_BC_BASE_ADDR:
4449		return true;
4450	default:
4451		DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4452		return false;
4453	}
4454}
4455
4456static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4457				  u32 *ib, struct radeon_cs_packet *pkt)
4458{
4459	switch (pkt->opcode) {
4460	case PACKET3_NOP:
4461	case PACKET3_SET_BASE:
4462	case PACKET3_SET_CE_DE_COUNTERS:
4463	case PACKET3_LOAD_CONST_RAM:
4464	case PACKET3_WRITE_CONST_RAM:
4465	case PACKET3_WRITE_CONST_RAM_OFFSET:
4466	case PACKET3_DUMP_CONST_RAM:
4467	case PACKET3_INCREMENT_CE_COUNTER:
4468	case PACKET3_WAIT_ON_DE_COUNTER:
4469	case PACKET3_CE_WRITE:
4470		break;
4471	default:
4472		DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4473		return -EINVAL;
4474	}
4475	return 0;
4476}
4477
4478static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4479{
4480	u32 start_reg, reg, i;
4481	u32 command = ib[idx + 4];
4482	u32 info = ib[idx + 1];
4483	u32 idx_value = ib[idx];
4484	if (command & PACKET3_CP_DMA_CMD_SAS) {
4485		/* src address space is register */
4486		if (((info & 0x60000000) >> 29) == 0) {
4487			start_reg = idx_value << 2;
4488			if (command & PACKET3_CP_DMA_CMD_SAIC) {
4489				reg = start_reg;
4490				if (!si_vm_reg_valid(reg)) {
4491					DRM_ERROR("CP DMA Bad SRC register\n");
4492					return -EINVAL;
4493				}
4494			} else {
4495				for (i = 0; i < (command & 0x1fffff); i++) {
4496					reg = start_reg + (4 * i);
4497					if (!si_vm_reg_valid(reg)) {
4498						DRM_ERROR("CP DMA Bad SRC register\n");
4499						return -EINVAL;
4500					}
4501				}
4502			}
4503		}
4504	}
4505	if (command & PACKET3_CP_DMA_CMD_DAS) {
4506		/* dst address space is register */
4507		if (((info & 0x00300000) >> 20) == 0) {
4508			start_reg = ib[idx + 2];
4509			if (command & PACKET3_CP_DMA_CMD_DAIC) {
4510				reg = start_reg;
4511				if (!si_vm_reg_valid(reg)) {
4512					DRM_ERROR("CP DMA Bad DST register\n");
4513					return -EINVAL;
4514				}
4515			} else {
4516				for (i = 0; i < (command & 0x1fffff); i++) {
4517					reg = start_reg + (4 * i);
4518					if (!si_vm_reg_valid(reg)) {
4519						DRM_ERROR("CP DMA Bad DST register\n");
4520						return -EINVAL;
4521					}
4522				}
4523			}
4524		}
4525	}
4526	return 0;
4527}
4528
4529static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4530				   u32 *ib, struct radeon_cs_packet *pkt)
4531{
4532	int r;
4533	u32 idx = pkt->idx + 1;
4534	u32 idx_value = ib[idx];
4535	u32 start_reg, end_reg, reg, i;
4536
4537	switch (pkt->opcode) {
4538	case PACKET3_NOP:
4539	case PACKET3_SET_BASE:
4540	case PACKET3_CLEAR_STATE:
4541	case PACKET3_INDEX_BUFFER_SIZE:
4542	case PACKET3_DISPATCH_DIRECT:
4543	case PACKET3_DISPATCH_INDIRECT:
4544	case PACKET3_ALLOC_GDS:
4545	case PACKET3_WRITE_GDS_RAM:
4546	case PACKET3_ATOMIC_GDS:
4547	case PACKET3_ATOMIC:
4548	case PACKET3_OCCLUSION_QUERY:
4549	case PACKET3_SET_PREDICATION:
4550	case PACKET3_COND_EXEC:
4551	case PACKET3_PRED_EXEC:
4552	case PACKET3_DRAW_INDIRECT:
4553	case PACKET3_DRAW_INDEX_INDIRECT:
4554	case PACKET3_INDEX_BASE:
4555	case PACKET3_DRAW_INDEX_2:
4556	case PACKET3_CONTEXT_CONTROL:
4557	case PACKET3_INDEX_TYPE:
4558	case PACKET3_DRAW_INDIRECT_MULTI:
4559	case PACKET3_DRAW_INDEX_AUTO:
4560	case PACKET3_DRAW_INDEX_IMMD:
4561	case PACKET3_NUM_INSTANCES:
4562	case PACKET3_DRAW_INDEX_MULTI_AUTO:
4563	case PACKET3_STRMOUT_BUFFER_UPDATE:
4564	case PACKET3_DRAW_INDEX_OFFSET_2:
4565	case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4566	case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4567	case PACKET3_MPEG_INDEX:
4568	case PACKET3_WAIT_REG_MEM:
4569	case PACKET3_MEM_WRITE:
4570	case PACKET3_PFP_SYNC_ME:
4571	case PACKET3_SURFACE_SYNC:
4572	case PACKET3_EVENT_WRITE:
4573	case PACKET3_EVENT_WRITE_EOP:
4574	case PACKET3_EVENT_WRITE_EOS:
4575	case PACKET3_SET_CONTEXT_REG:
4576	case PACKET3_SET_CONTEXT_REG_INDIRECT:
4577	case PACKET3_SET_SH_REG:
4578	case PACKET3_SET_SH_REG_OFFSET:
4579	case PACKET3_INCREMENT_DE_COUNTER:
4580	case PACKET3_WAIT_ON_CE_COUNTER:
4581	case PACKET3_WAIT_ON_AVAIL_BUFFER:
4582	case PACKET3_ME_WRITE:
4583		break;
4584	case PACKET3_COPY_DATA:
4585		if ((idx_value & 0xf00) == 0) {
4586			reg = ib[idx + 3] * 4;
4587			if (!si_vm_reg_valid(reg))
4588				return -EINVAL;
4589		}
4590		break;
4591	case PACKET3_WRITE_DATA:
4592		if ((idx_value & 0xf00) == 0) {
4593			start_reg = ib[idx + 1] * 4;
4594			if (idx_value & 0x10000) {
4595				if (!si_vm_reg_valid(start_reg))
4596					return -EINVAL;
4597			} else {
4598				for (i = 0; i < (pkt->count - 2); i++) {
4599					reg = start_reg + (4 * i);
4600					if (!si_vm_reg_valid(reg))
4601						return -EINVAL;
4602				}
4603			}
4604		}
4605		break;
4606	case PACKET3_COND_WRITE:
4607		if (idx_value & 0x100) {
4608			reg = ib[idx + 5] * 4;
4609			if (!si_vm_reg_valid(reg))
4610				return -EINVAL;
4611		}
4612		break;
4613	case PACKET3_COPY_DW:
4614		if (idx_value & 0x2) {
4615			reg = ib[idx + 3] * 4;
4616			if (!si_vm_reg_valid(reg))
4617				return -EINVAL;
4618		}
4619		break;
4620	case PACKET3_SET_CONFIG_REG:
4621		start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4622		end_reg = 4 * pkt->count + start_reg - 4;
4623		if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4624		    (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4625		    (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4626			DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4627			return -EINVAL;
4628		}
4629		for (i = 0; i < pkt->count; i++) {
4630			reg = start_reg + (4 * i);
4631			if (!si_vm_reg_valid(reg))
4632				return -EINVAL;
4633		}
4634		break;
4635	case PACKET3_CP_DMA:
4636		r = si_vm_packet3_cp_dma_check(ib, idx);
4637		if (r)
4638			return r;
4639		break;
4640	default:
4641		DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4642		return -EINVAL;
4643	}
4644	return 0;
4645}
4646
4647static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4648				       u32 *ib, struct radeon_cs_packet *pkt)
4649{
4650	int r;
4651	u32 idx = pkt->idx + 1;
4652	u32 idx_value = ib[idx];
4653	u32 start_reg, reg, i;
4654
4655	switch (pkt->opcode) {
4656	case PACKET3_NOP:
4657	case PACKET3_SET_BASE:
4658	case PACKET3_CLEAR_STATE:
4659	case PACKET3_DISPATCH_DIRECT:
4660	case PACKET3_DISPATCH_INDIRECT:
4661	case PACKET3_ALLOC_GDS:
4662	case PACKET3_WRITE_GDS_RAM:
4663	case PACKET3_ATOMIC_GDS:
4664	case PACKET3_ATOMIC:
4665	case PACKET3_OCCLUSION_QUERY:
4666	case PACKET3_SET_PREDICATION:
4667	case PACKET3_COND_EXEC:
4668	case PACKET3_PRED_EXEC:
4669	case PACKET3_CONTEXT_CONTROL:
4670	case PACKET3_STRMOUT_BUFFER_UPDATE:
4671	case PACKET3_WAIT_REG_MEM:
4672	case PACKET3_MEM_WRITE:
4673	case PACKET3_PFP_SYNC_ME:
4674	case PACKET3_SURFACE_SYNC:
4675	case PACKET3_EVENT_WRITE:
4676	case PACKET3_EVENT_WRITE_EOP:
4677	case PACKET3_EVENT_WRITE_EOS:
4678	case PACKET3_SET_CONTEXT_REG:
4679	case PACKET3_SET_CONTEXT_REG_INDIRECT:
4680	case PACKET3_SET_SH_REG:
4681	case PACKET3_SET_SH_REG_OFFSET:
4682	case PACKET3_INCREMENT_DE_COUNTER:
4683	case PACKET3_WAIT_ON_CE_COUNTER:
4684	case PACKET3_WAIT_ON_AVAIL_BUFFER:
4685	case PACKET3_ME_WRITE:
4686		break;
4687	case PACKET3_COPY_DATA:
4688		if ((idx_value & 0xf00) == 0) {
4689			reg = ib[idx + 3] * 4;
4690			if (!si_vm_reg_valid(reg))
4691				return -EINVAL;
4692		}
4693		break;
4694	case PACKET3_WRITE_DATA:
4695		if ((idx_value & 0xf00) == 0) {
4696			start_reg = ib[idx + 1] * 4;
4697			if (idx_value & 0x10000) {
4698				if (!si_vm_reg_valid(start_reg))
4699					return -EINVAL;
4700			} else {
4701				for (i = 0; i < (pkt->count - 2); i++) {
4702					reg = start_reg + (4 * i);
4703					if (!si_vm_reg_valid(reg))
4704						return -EINVAL;
4705				}
4706			}
4707		}
4708		break;
4709	case PACKET3_COND_WRITE:
4710		if (idx_value & 0x100) {
4711			reg = ib[idx + 5] * 4;
4712			if (!si_vm_reg_valid(reg))
4713				return -EINVAL;
4714		}
4715		break;
4716	case PACKET3_COPY_DW:
4717		if (idx_value & 0x2) {
4718			reg = ib[idx + 3] * 4;
4719			if (!si_vm_reg_valid(reg))
4720				return -EINVAL;
4721		}
4722		break;
4723	case PACKET3_CP_DMA:
4724		r = si_vm_packet3_cp_dma_check(ib, idx);
4725		if (r)
4726			return r;
4727		break;
4728	default:
4729		DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4730		return -EINVAL;
4731	}
4732	return 0;
4733}
4734
4735int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4736{
4737	int ret = 0;
4738	u32 idx = 0, i;
4739	struct radeon_cs_packet pkt;
4740
4741	do {
4742		pkt.idx = idx;
4743		pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4744		pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4745		pkt.one_reg_wr = 0;
4746		switch (pkt.type) {
4747		case RADEON_PACKET_TYPE0:
4748			dev_err(rdev->dev, "Packet0 not allowed!\n");
4749			ret = -EINVAL;
4750			break;
4751		case RADEON_PACKET_TYPE2:
4752			idx += 1;
4753			break;
4754		case RADEON_PACKET_TYPE3:
4755			pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4756			if (ib->is_const_ib)
4757				ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4758			else {
4759				switch (ib->ring) {
4760				case RADEON_RING_TYPE_GFX_INDEX:
4761					ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4762					break;
4763				case CAYMAN_RING_TYPE_CP1_INDEX:
4764				case CAYMAN_RING_TYPE_CP2_INDEX:
4765					ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4766					break;
4767				default:
4768					dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4769					ret = -EINVAL;
4770					break;
4771				}
4772			}
4773			idx += pkt.count + 2;
4774			break;
4775		default:
4776			dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4777			ret = -EINVAL;
4778			break;
4779		}
4780		if (ret) {
4781			for (i = 0; i < ib->length_dw; i++) {
4782				if (i == idx)
4783					printk("\t0x%08x <---\n", ib->ptr[i]);
4784				else
4785					printk("\t0x%08x\n", ib->ptr[i]);
4786			}
4787			break;
4788		}
4789	} while (idx < ib->length_dw);
4790
4791	return ret;
4792}
4793
4794/*
4795 * vm
4796 */
4797int si_vm_init(struct radeon_device *rdev)
4798{
4799	/* number of VMs */
4800	rdev->vm_manager.nvm = 16;
4801	/* base offset of vram pages */
4802	rdev->vm_manager.vram_base_offset = 0;
4803
4804	return 0;
4805}
4806
4807void si_vm_fini(struct radeon_device *rdev)
4808{
4809}
4810
4811/**
4812 * si_vm_decode_fault - print human readable fault info
4813 *
4814 * @rdev: radeon_device pointer
4815 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4816 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4817 *
4818 * Print human readable fault information (SI).
4819 */
4820static void si_vm_decode_fault(struct radeon_device *rdev,
4821			       u32 status, u32 addr)
4822{
4823	u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4824	u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4825	u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4826	char *block;
4827
4828	if (rdev->family == CHIP_TAHITI) {
4829		switch (mc_id) {
4830		case 160:
4831		case 144:
4832		case 96:
4833		case 80:
4834		case 224:
4835		case 208:
4836		case 32:
4837		case 16:
4838			block = "CB";
4839			break;
4840		case 161:
4841		case 145:
4842		case 97:
4843		case 81:
4844		case 225:
4845		case 209:
4846		case 33:
4847		case 17:
4848			block = "CB_FMASK";
4849			break;
4850		case 162:
4851		case 146:
4852		case 98:
4853		case 82:
4854		case 226:
4855		case 210:
4856		case 34:
4857		case 18:
4858			block = "CB_CMASK";
4859			break;
4860		case 163:
4861		case 147:
4862		case 99:
4863		case 83:
4864		case 227:
4865		case 211:
4866		case 35:
4867		case 19:
4868			block = "CB_IMMED";
4869			break;
4870		case 164:
4871		case 148:
4872		case 100:
4873		case 84:
4874		case 228:
4875		case 212:
4876		case 36:
4877		case 20:
4878			block = "DB";
4879			break;
4880		case 165:
4881		case 149:
4882		case 101:
4883		case 85:
4884		case 229:
4885		case 213:
4886		case 37:
4887		case 21:
4888			block = "DB_HTILE";
4889			break;
4890		case 167:
4891		case 151:
4892		case 103:
4893		case 87:
4894		case 231:
4895		case 215:
4896		case 39:
4897		case 23:
4898			block = "DB_STEN";
4899			break;
4900		case 72:
4901		case 68:
4902		case 64:
4903		case 8:
4904		case 4:
4905		case 0:
4906		case 136:
4907		case 132:
4908		case 128:
4909		case 200:
4910		case 196:
4911		case 192:
4912			block = "TC";
4913			break;
4914		case 112:
4915		case 48:
4916			block = "CP";
4917			break;
4918		case 49:
4919		case 177:
4920		case 50:
4921		case 178:
4922			block = "SH";
4923			break;
4924		case 53:
4925		case 190:
4926			block = "VGT";
4927			break;
4928		case 117:
4929			block = "IH";
4930			break;
4931		case 51:
4932		case 115:
4933			block = "RLC";
4934			break;
4935		case 119:
4936		case 183:
4937			block = "DMA0";
4938			break;
4939		case 61:
4940			block = "DMA1";
4941			break;
4942		case 248:
4943		case 120:
4944			block = "HDP";
4945			break;
4946		default:
4947			block = "unknown";
4948			break;
4949		}
4950	} else {
4951		switch (mc_id) {
4952		case 32:
4953		case 16:
4954		case 96:
4955		case 80:
4956		case 160:
4957		case 144:
4958		case 224:
4959		case 208:
4960			block = "CB";
4961			break;
4962		case 33:
4963		case 17:
4964		case 97:
4965		case 81:
4966		case 161:
4967		case 145:
4968		case 225:
4969		case 209:
4970			block = "CB_FMASK";
4971			break;
4972		case 34:
4973		case 18:
4974		case 98:
4975		case 82:
4976		case 162:
4977		case 146:
4978		case 226:
4979		case 210:
4980			block = "CB_CMASK";
4981			break;
4982		case 35:
4983		case 19:
4984		case 99:
4985		case 83:
4986		case 163:
4987		case 147:
4988		case 227:
4989		case 211:
4990			block = "CB_IMMED";
4991			break;
4992		case 36:
4993		case 20:
4994		case 100:
4995		case 84:
4996		case 164:
4997		case 148:
4998		case 228:
4999		case 212:
5000			block = "DB";
5001			break;
5002		case 37:
5003		case 21:
5004		case 101:
5005		case 85:
5006		case 165:
5007		case 149:
5008		case 229:
5009		case 213:
5010			block = "DB_HTILE";
5011			break;
5012		case 39:
5013		case 23:
5014		case 103:
5015		case 87:
5016		case 167:
5017		case 151:
5018		case 231:
5019		case 215:
5020			block = "DB_STEN";
5021			break;
5022		case 72:
5023		case 68:
5024		case 8:
5025		case 4:
5026		case 136:
5027		case 132:
5028		case 200:
5029		case 196:
5030			block = "TC";
5031			break;
5032		case 112:
5033		case 48:
5034			block = "CP";
5035			break;
5036		case 49:
5037		case 177:
5038		case 50:
5039		case 178:
5040			block = "SH";
5041			break;
5042		case 53:
5043			block = "VGT";
5044			break;
5045		case 117:
5046			block = "IH";
5047			break;
5048		case 51:
5049		case 115:
5050			block = "RLC";
5051			break;
5052		case 119:
5053		case 183:
5054			block = "DMA0";
5055			break;
5056		case 61:
5057			block = "DMA1";
5058			break;
5059		case 248:
5060		case 120:
5061			block = "HDP";
5062			break;
5063		default:
5064			block = "unknown";
5065			break;
5066		}
5067	}
5068
5069	printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
5070	       protections, vmid, addr,
5071	       (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5072	       block, mc_id);
5073}
5074
5075void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5076		 unsigned vm_id, uint64_t pd_addr)
5077{
5078	/* write new base address */
5079	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5080	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5081				 WRITE_DATA_DST_SEL(0)));
5082
5083	if (vm_id < 8) {
5084		radeon_ring_write(ring,
5085				  (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
5086	} else {
5087		radeon_ring_write(ring,
5088				  (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
5089	}
5090	radeon_ring_write(ring, 0);
5091	radeon_ring_write(ring, pd_addr >> 12);
5092
5093	/* flush hdp cache */
5094	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5095	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5096				 WRITE_DATA_DST_SEL(0)));
5097	radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5098	radeon_ring_write(ring, 0);
5099	radeon_ring_write(ring, 0x1);
5100
5101	/* bits 0-15 are the VM contexts0-15 */
5102	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5103	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5104				 WRITE_DATA_DST_SEL(0)));
5105	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5106	radeon_ring_write(ring, 0);
5107	radeon_ring_write(ring, 1 << vm_id);
5108
5109	/* wait for the invalidate to complete */
5110	radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5111	radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
5112				 WAIT_REG_MEM_ENGINE(0))); /* me */
5113	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5114	radeon_ring_write(ring, 0);
5115	radeon_ring_write(ring, 0); /* ref */
5116	radeon_ring_write(ring, 0); /* mask */
5117	radeon_ring_write(ring, 0x20); /* poll interval */
5118
5119	/* sync PFP to ME, otherwise we might get invalid PFP reads */
5120	radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5121	radeon_ring_write(ring, 0x0);
5122}
5123
5124/*
5125 *  Power and clock gating
5126 */
5127static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5128{
5129	int i;
5130
5131	for (i = 0; i < rdev->usec_timeout; i++) {
5132		if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5133			break;
5134		udelay(1);
5135	}
5136
5137	for (i = 0; i < rdev->usec_timeout; i++) {
5138		if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5139			break;
5140		udelay(1);
5141	}
5142}
5143
5144static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5145					 bool enable)
5146{
5147	u32 tmp = RREG32(CP_INT_CNTL_RING0);
5148	u32 mask;
5149	int i;
5150
5151	if (enable)
5152		tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5153	else
5154		tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5155	WREG32(CP_INT_CNTL_RING0, tmp);
5156
5157	if (!enable) {
5158		/* read a gfx register */
5159		tmp = RREG32(DB_DEPTH_INFO);
5160
5161		mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5162		for (i = 0; i < rdev->usec_timeout; i++) {
5163			if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5164				break;
5165			udelay(1);
5166		}
5167	}
5168}
5169
5170static void si_set_uvd_dcm(struct radeon_device *rdev,
5171			   bool sw_mode)
5172{
5173	u32 tmp, tmp2;
5174
5175	tmp = RREG32(UVD_CGC_CTRL);
5176	tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5177	tmp |= DCM | CG_DT(1) | CLK_OD(4);
5178
5179	if (sw_mode) {
5180		tmp &= ~0x7ffff800;
5181		tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5182	} else {
5183		tmp |= 0x7ffff800;
5184		tmp2 = 0;
5185	}
5186
5187	WREG32(UVD_CGC_CTRL, tmp);
5188	WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5189}
5190
5191void si_init_uvd_internal_cg(struct radeon_device *rdev)
5192{
5193	bool hw_mode = true;
5194
5195	if (hw_mode) {
5196		si_set_uvd_dcm(rdev, false);
5197	} else {
5198		u32 tmp = RREG32(UVD_CGC_CTRL);
5199		tmp &= ~DCM;
5200		WREG32(UVD_CGC_CTRL, tmp);
5201	}
5202}
5203
5204static u32 si_halt_rlc(struct radeon_device *rdev)
5205{
5206	u32 data, orig;
5207
5208	orig = data = RREG32(RLC_CNTL);
5209
5210	if (data & RLC_ENABLE) {
5211		data &= ~RLC_ENABLE;
5212		WREG32(RLC_CNTL, data);
5213
5214		si_wait_for_rlc_serdes(rdev);
5215	}
5216
5217	return orig;
5218}
5219
5220static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5221{
5222	u32 tmp;
5223
5224	tmp = RREG32(RLC_CNTL);
5225	if (tmp != rlc)
5226		WREG32(RLC_CNTL, rlc);
5227}
5228
5229static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5230{
5231	u32 data, orig;
5232
5233	orig = data = RREG32(DMA_PG);
5234	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
5235		data |= PG_CNTL_ENABLE;
5236	else
5237		data &= ~PG_CNTL_ENABLE;
5238	if (orig != data)
5239		WREG32(DMA_PG, data);
5240}
5241
5242static void si_init_dma_pg(struct radeon_device *rdev)
5243{
5244	u32 tmp;
5245
5246	WREG32(DMA_PGFSM_WRITE,  0x00002000);
5247	WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5248
5249	for (tmp = 0; tmp < 5; tmp++)
5250		WREG32(DMA_PGFSM_WRITE, 0);
5251}
5252
5253static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5254			       bool enable)
5255{
5256	u32 tmp;
5257
5258	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5259		tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5260		WREG32(RLC_TTOP_D, tmp);
5261
5262		tmp = RREG32(RLC_PG_CNTL);
5263		tmp |= GFX_PG_ENABLE;
5264		WREG32(RLC_PG_CNTL, tmp);
5265
5266		tmp = RREG32(RLC_AUTO_PG_CTRL);
5267		tmp |= AUTO_PG_EN;
5268		WREG32(RLC_AUTO_PG_CTRL, tmp);
5269	} else {
5270		tmp = RREG32(RLC_AUTO_PG_CTRL);
5271		tmp &= ~AUTO_PG_EN;
5272		WREG32(RLC_AUTO_PG_CTRL, tmp);
5273
5274		tmp = RREG32(DB_RENDER_CONTROL);
5275	}
5276}
5277
5278static void si_init_gfx_cgpg(struct radeon_device *rdev)
5279{
5280	u32 tmp;
5281
5282	WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5283
5284	tmp = RREG32(RLC_PG_CNTL);
5285	tmp |= GFX_PG_SRC;
5286	WREG32(RLC_PG_CNTL, tmp);
5287
5288	WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5289
5290	tmp = RREG32(RLC_AUTO_PG_CTRL);
5291
5292	tmp &= ~GRBM_REG_SGIT_MASK;
5293	tmp |= GRBM_REG_SGIT(0x700);
5294	tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5295	WREG32(RLC_AUTO_PG_CTRL, tmp);
5296}
5297
5298static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5299{
5300	u32 mask = 0, tmp, tmp1;
5301	int i;
5302
5303	si_select_se_sh(rdev, se, sh);
5304	tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5305	tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5306	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5307
5308	tmp &= 0xffff0000;
5309
5310	tmp |= tmp1;
5311	tmp >>= 16;
5312
5313	for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5314		mask <<= 1;
5315		mask |= 1;
5316	}
5317
5318	return (~tmp) & mask;
5319}
5320
5321static void si_init_ao_cu_mask(struct radeon_device *rdev)
5322{
5323	u32 i, j, k, active_cu_number = 0;
5324	u32 mask, counter, cu_bitmap;
5325	u32 tmp = 0;
5326
5327	for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5328		for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5329			mask = 1;
5330			cu_bitmap = 0;
5331			counter  = 0;
5332			for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5333				if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5334					if (counter < 2)
5335						cu_bitmap |= mask;
5336					counter++;
5337				}
5338				mask <<= 1;
5339			}
5340
5341			active_cu_number += counter;
5342			tmp |= (cu_bitmap << (i * 16 + j * 8));
5343		}
5344	}
5345
5346	WREG32(RLC_PG_AO_CU_MASK, tmp);
5347
5348	tmp = RREG32(RLC_MAX_PG_CU);
5349	tmp &= ~MAX_PU_CU_MASK;
5350	tmp |= MAX_PU_CU(active_cu_number);
5351	WREG32(RLC_MAX_PG_CU, tmp);
5352}
5353
5354static void si_enable_cgcg(struct radeon_device *rdev,
5355			   bool enable)
5356{
5357	u32 data, orig, tmp;
5358
5359	orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5360
5361	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5362		si_enable_gui_idle_interrupt(rdev, true);
5363
5364		WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5365
5366		tmp = si_halt_rlc(rdev);
5367
5368		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5369		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5370		WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5371
5372		si_wait_for_rlc_serdes(rdev);
5373
5374		si_update_rlc(rdev, tmp);
5375
5376		WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5377
5378		data |= CGCG_EN | CGLS_EN;
5379	} else {
5380		si_enable_gui_idle_interrupt(rdev, false);
5381
5382		RREG32(CB_CGTT_SCLK_CTRL);
5383		RREG32(CB_CGTT_SCLK_CTRL);
5384		RREG32(CB_CGTT_SCLK_CTRL);
5385		RREG32(CB_CGTT_SCLK_CTRL);
5386
5387		data &= ~(CGCG_EN | CGLS_EN);
5388	}
5389
5390	if (orig != data)
5391		WREG32(RLC_CGCG_CGLS_CTRL, data);
5392}
5393
5394static void si_enable_mgcg(struct radeon_device *rdev,
5395			   bool enable)
5396{
5397	u32 data, orig, tmp = 0;
5398
5399	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5400		orig = data = RREG32(CGTS_SM_CTRL_REG);
5401		data = 0x96940200;
5402		if (orig != data)
5403			WREG32(CGTS_SM_CTRL_REG, data);
5404
5405		if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5406			orig = data = RREG32(CP_MEM_SLP_CNTL);
5407			data |= CP_MEM_LS_EN;
5408			if (orig != data)
5409				WREG32(CP_MEM_SLP_CNTL, data);
5410		}
5411
5412		orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5413		data &= 0xffffffc0;
5414		if (orig != data)
5415			WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5416
5417		tmp = si_halt_rlc(rdev);
5418
5419		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5420		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5421		WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5422
5423		si_update_rlc(rdev, tmp);
5424	} else {
5425		orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5426		data |= 0x00000003;
5427		if (orig != data)
5428			WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5429
5430		data = RREG32(CP_MEM_SLP_CNTL);
5431		if (data & CP_MEM_LS_EN) {
5432			data &= ~CP_MEM_LS_EN;
5433			WREG32(CP_MEM_SLP_CNTL, data);
5434		}
5435		orig = data = RREG32(CGTS_SM_CTRL_REG);
5436		data |= LS_OVERRIDE | OVERRIDE;
5437		if (orig != data)
5438			WREG32(CGTS_SM_CTRL_REG, data);
5439
5440		tmp = si_halt_rlc(rdev);
5441
5442		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5443		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5444		WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5445
5446		si_update_rlc(rdev, tmp);
5447	}
5448}
5449
5450static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5451			       bool enable)
5452{
5453	u32 orig, data, tmp;
5454
5455	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5456		tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5457		tmp |= 0x3fff;
5458		WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5459
5460		orig = data = RREG32(UVD_CGC_CTRL);
5461		data |= DCM;
5462		if (orig != data)
5463			WREG32(UVD_CGC_CTRL, data);
5464
5465		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5466		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5467	} else {
5468		tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5469		tmp &= ~0x3fff;
5470		WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5471
5472		orig = data = RREG32(UVD_CGC_CTRL);
5473		data &= ~DCM;
5474		if (orig != data)
5475			WREG32(UVD_CGC_CTRL, data);
5476
5477		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5478		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5479	}
5480}
5481
5482static const u32 mc_cg_registers[] =
5483{
5484	MC_HUB_MISC_HUB_CG,
5485	MC_HUB_MISC_SIP_CG,
5486	MC_HUB_MISC_VM_CG,
5487	MC_XPB_CLK_GAT,
5488	ATC_MISC_CG,
5489	MC_CITF_MISC_WR_CG,
5490	MC_CITF_MISC_RD_CG,
5491	MC_CITF_MISC_VM_CG,
5492	VM_L2_CG,
5493};
5494
5495static void si_enable_mc_ls(struct radeon_device *rdev,
5496			    bool enable)
5497{
5498	int i;
5499	u32 orig, data;
5500
5501	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5502		orig = data = RREG32(mc_cg_registers[i]);
5503		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5504			data |= MC_LS_ENABLE;
5505		else
5506			data &= ~MC_LS_ENABLE;
5507		if (data != orig)
5508			WREG32(mc_cg_registers[i], data);
5509	}
5510}
5511
5512static void si_enable_mc_mgcg(struct radeon_device *rdev,
5513			       bool enable)
5514{
5515	int i;
5516	u32 orig, data;
5517
5518	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5519		orig = data = RREG32(mc_cg_registers[i]);
5520		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5521			data |= MC_CG_ENABLE;
5522		else
5523			data &= ~MC_CG_ENABLE;
5524		if (data != orig)
5525			WREG32(mc_cg_registers[i], data);
5526	}
5527}
5528
5529static void si_enable_dma_mgcg(struct radeon_device *rdev,
5530			       bool enable)
5531{
5532	u32 orig, data, offset;
5533	int i;
5534
5535	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5536		for (i = 0; i < 2; i++) {
5537			if (i == 0)
5538				offset = DMA0_REGISTER_OFFSET;
5539			else
5540				offset = DMA1_REGISTER_OFFSET;
5541			orig = data = RREG32(DMA_POWER_CNTL + offset);
5542			data &= ~MEM_POWER_OVERRIDE;
5543			if (data != orig)
5544				WREG32(DMA_POWER_CNTL + offset, data);
5545			WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5546		}
5547	} else {
5548		for (i = 0; i < 2; i++) {
5549			if (i == 0)
5550				offset = DMA0_REGISTER_OFFSET;
5551			else
5552				offset = DMA1_REGISTER_OFFSET;
5553			orig = data = RREG32(DMA_POWER_CNTL + offset);
5554			data |= MEM_POWER_OVERRIDE;
5555			if (data != orig)
5556				WREG32(DMA_POWER_CNTL + offset, data);
5557
5558			orig = data = RREG32(DMA_CLK_CTRL + offset);
5559			data = 0xff000000;
5560			if (data != orig)
5561				WREG32(DMA_CLK_CTRL + offset, data);
5562		}
5563	}
5564}
5565
5566static void si_enable_bif_mgls(struct radeon_device *rdev,
5567			       bool enable)
5568{
5569	u32 orig, data;
5570
5571	orig = data = RREG32_PCIE(PCIE_CNTL2);
5572
5573	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5574		data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5575			REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5576	else
5577		data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5578			  REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5579
5580	if (orig != data)
5581		WREG32_PCIE(PCIE_CNTL2, data);
5582}
5583
5584static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5585			       bool enable)
5586{
5587	u32 orig, data;
5588
5589	orig = data = RREG32(HDP_HOST_PATH_CNTL);
5590
5591	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5592		data &= ~CLOCK_GATING_DIS;
5593	else
5594		data |= CLOCK_GATING_DIS;
5595
5596	if (orig != data)
5597		WREG32(HDP_HOST_PATH_CNTL, data);
5598}
5599
5600static void si_enable_hdp_ls(struct radeon_device *rdev,
5601			     bool enable)
5602{
5603	u32 orig, data;
5604
5605	orig = data = RREG32(HDP_MEM_POWER_LS);
5606
5607	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5608		data |= HDP_LS_ENABLE;
5609	else
5610		data &= ~HDP_LS_ENABLE;
5611
5612	if (orig != data)
5613		WREG32(HDP_MEM_POWER_LS, data);
5614}
5615
5616static void si_update_cg(struct radeon_device *rdev,
5617			 u32 block, bool enable)
5618{
5619	if (block & RADEON_CG_BLOCK_GFX) {
5620		si_enable_gui_idle_interrupt(rdev, false);
5621		/* order matters! */
5622		if (enable) {
5623			si_enable_mgcg(rdev, true);
5624			si_enable_cgcg(rdev, true);
5625		} else {
5626			si_enable_cgcg(rdev, false);
5627			si_enable_mgcg(rdev, false);
5628		}
5629		si_enable_gui_idle_interrupt(rdev, true);
5630	}
5631
5632	if (block & RADEON_CG_BLOCK_MC) {
5633		si_enable_mc_mgcg(rdev, enable);
5634		si_enable_mc_ls(rdev, enable);
5635	}
5636
5637	if (block & RADEON_CG_BLOCK_SDMA) {
5638		si_enable_dma_mgcg(rdev, enable);
5639	}
5640
5641	if (block & RADEON_CG_BLOCK_BIF) {
5642		si_enable_bif_mgls(rdev, enable);
5643	}
5644
5645	if (block & RADEON_CG_BLOCK_UVD) {
5646		if (rdev->has_uvd) {
5647			si_enable_uvd_mgcg(rdev, enable);
5648		}
5649	}
5650
5651	if (block & RADEON_CG_BLOCK_HDP) {
5652		si_enable_hdp_mgcg(rdev, enable);
5653		si_enable_hdp_ls(rdev, enable);
5654	}
5655}
5656
5657static void si_init_cg(struct radeon_device *rdev)
5658{
5659	si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5660			    RADEON_CG_BLOCK_MC |
5661			    RADEON_CG_BLOCK_SDMA |
5662			    RADEON_CG_BLOCK_BIF |
5663			    RADEON_CG_BLOCK_HDP), true);
5664	if (rdev->has_uvd) {
5665		si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5666		si_init_uvd_internal_cg(rdev);
5667	}
5668}
5669
5670static void si_fini_cg(struct radeon_device *rdev)
5671{
5672	if (rdev->has_uvd) {
5673		si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5674	}
5675	si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5676			    RADEON_CG_BLOCK_MC |
5677			    RADEON_CG_BLOCK_SDMA |
5678			    RADEON_CG_BLOCK_BIF |
5679			    RADEON_CG_BLOCK_HDP), false);
5680}
5681
5682u32 si_get_csb_size(struct radeon_device *rdev)
5683{
5684	u32 count = 0;
5685	const struct cs_section_def *sect = NULL;
5686	const struct cs_extent_def *ext = NULL;
5687
5688	if (rdev->rlc.cs_data == NULL)
5689		return 0;
5690
5691	/* begin clear state */
5692	count += 2;
5693	/* context control state */
5694	count += 3;
5695
5696	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5697		for (ext = sect->section; ext->extent != NULL; ++ext) {
5698			if (sect->id == SECT_CONTEXT)
5699				count += 2 + ext->reg_count;
5700			else
5701				return 0;
5702		}
5703	}
5704	/* pa_sc_raster_config */
5705	count += 3;
5706	/* end clear state */
5707	count += 2;
5708	/* clear state */
5709	count += 2;
5710
5711	return count;
5712}
5713
5714void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5715{
5716	u32 count = 0, i;
5717	const struct cs_section_def *sect = NULL;
5718	const struct cs_extent_def *ext = NULL;
5719
5720	if (rdev->rlc.cs_data == NULL)
5721		return;
5722	if (buffer == NULL)
5723		return;
5724
5725	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5726	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5727
5728	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5729	buffer[count++] = cpu_to_le32(0x80000000);
5730	buffer[count++] = cpu_to_le32(0x80000000);
5731
5732	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5733		for (ext = sect->section; ext->extent != NULL; ++ext) {
5734			if (sect->id == SECT_CONTEXT) {
5735				buffer[count++] =
5736					cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5737				buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5738				for (i = 0; i < ext->reg_count; i++)
5739					buffer[count++] = cpu_to_le32(ext->extent[i]);
5740			} else {
5741				return;
5742			}
5743		}
5744	}
5745
5746	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5747	buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5748	switch (rdev->family) {
5749	case CHIP_TAHITI:
5750	case CHIP_PITCAIRN:
5751		buffer[count++] = cpu_to_le32(0x2a00126a);
5752		break;
5753	case CHIP_VERDE:
5754		buffer[count++] = cpu_to_le32(0x0000124a);
5755		break;
5756	case CHIP_OLAND:
5757		buffer[count++] = cpu_to_le32(0x00000082);
5758		break;
5759	case CHIP_HAINAN:
5760		buffer[count++] = cpu_to_le32(0x00000000);
5761		break;
5762	default:
5763		buffer[count++] = cpu_to_le32(0x00000000);
5764		break;
5765	}
5766
5767	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5768	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5769
5770	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5771	buffer[count++] = cpu_to_le32(0);
5772}
5773
5774static void si_init_pg(struct radeon_device *rdev)
5775{
5776	if (rdev->pg_flags) {
5777		if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5778			si_init_dma_pg(rdev);
5779		}
5780		si_init_ao_cu_mask(rdev);
5781		if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5782			si_init_gfx_cgpg(rdev);
5783		} else {
5784			WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5785			WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5786		}
5787		si_enable_dma_pg(rdev, true);
5788		si_enable_gfx_cgpg(rdev, true);
5789	} else {
5790		WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5791		WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5792	}
5793}
5794
5795static void si_fini_pg(struct radeon_device *rdev)
5796{
5797	if (rdev->pg_flags) {
5798		si_enable_dma_pg(rdev, false);
5799		si_enable_gfx_cgpg(rdev, false);
5800	}
5801}
5802
5803/*
5804 * RLC
5805 */
5806void si_rlc_reset(struct radeon_device *rdev)
5807{
5808	u32 tmp = RREG32(GRBM_SOFT_RESET);
5809
5810	tmp |= SOFT_RESET_RLC;
5811	WREG32(GRBM_SOFT_RESET, tmp);
5812	udelay(50);
5813	tmp &= ~SOFT_RESET_RLC;
5814	WREG32(GRBM_SOFT_RESET, tmp);
5815	udelay(50);
5816}
5817
5818static void si_rlc_stop(struct radeon_device *rdev)
5819{
5820	WREG32(RLC_CNTL, 0);
5821
5822	si_enable_gui_idle_interrupt(rdev, false);
5823
5824	si_wait_for_rlc_serdes(rdev);
5825}
5826
5827static void si_rlc_start(struct radeon_device *rdev)
5828{
5829	WREG32(RLC_CNTL, RLC_ENABLE);
5830
5831	si_enable_gui_idle_interrupt(rdev, true);
5832
5833	udelay(50);
5834}
5835
5836static bool si_lbpw_supported(struct radeon_device *rdev)
5837{
5838	u32 tmp;
5839
5840	/* Enable LBPW only for DDR3 */
5841	tmp = RREG32(MC_SEQ_MISC0);
5842	if ((tmp & 0xF0000000) == 0xB0000000)
5843		return true;
5844	return false;
5845}
5846
5847static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5848{
5849	u32 tmp;
5850
5851	tmp = RREG32(RLC_LB_CNTL);
5852	if (enable)
5853		tmp |= LOAD_BALANCE_ENABLE;
5854	else
5855		tmp &= ~LOAD_BALANCE_ENABLE;
5856	WREG32(RLC_LB_CNTL, tmp);
5857
5858	if (!enable) {
5859		si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5860		WREG32(SPI_LB_CU_MASK, 0x00ff);
5861	}
5862}
5863
5864static int si_rlc_resume(struct radeon_device *rdev)
5865{
5866	u32 i;
5867
5868	if (!rdev->rlc_fw)
5869		return -EINVAL;
5870
5871	si_rlc_stop(rdev);
5872
5873	si_rlc_reset(rdev);
5874
5875	si_init_pg(rdev);
5876
5877	si_init_cg(rdev);
5878
5879	WREG32(RLC_RL_BASE, 0);
5880	WREG32(RLC_RL_SIZE, 0);
5881	WREG32(RLC_LB_CNTL, 0);
5882	WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5883	WREG32(RLC_LB_CNTR_INIT, 0);
5884	WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5885
5886	WREG32(RLC_MC_CNTL, 0);
5887	WREG32(RLC_UCODE_CNTL, 0);
5888
5889	if (rdev->new_fw) {
5890		const struct rlc_firmware_header_v1_0 *hdr =
5891			(const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5892		u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5893		const __le32 *fw_data = (const __le32 *)
5894			(rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5895
5896		radeon_ucode_print_rlc_hdr(&hdr->header);
5897
5898		for (i = 0; i < fw_size; i++) {
5899			WREG32(RLC_UCODE_ADDR, i);
5900			WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5901		}
5902	} else {
5903		const __be32 *fw_data =
5904			(const __be32 *)rdev->rlc_fw->data;
5905		for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5906			WREG32(RLC_UCODE_ADDR, i);
5907			WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5908		}
5909	}
5910	WREG32(RLC_UCODE_ADDR, 0);
5911
5912	si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5913
5914	si_rlc_start(rdev);
5915
5916	return 0;
5917}
5918
5919static void si_enable_interrupts(struct radeon_device *rdev)
5920{
5921	u32 ih_cntl = RREG32(IH_CNTL);
5922	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5923
5924	ih_cntl |= ENABLE_INTR;
5925	ih_rb_cntl |= IH_RB_ENABLE;
5926	WREG32(IH_CNTL, ih_cntl);
5927	WREG32(IH_RB_CNTL, ih_rb_cntl);
5928	rdev->ih.enabled = true;
5929}
5930
5931static void si_disable_interrupts(struct radeon_device *rdev)
5932{
5933	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5934	u32 ih_cntl = RREG32(IH_CNTL);
5935
5936	ih_rb_cntl &= ~IH_RB_ENABLE;
5937	ih_cntl &= ~ENABLE_INTR;
5938	WREG32(IH_RB_CNTL, ih_rb_cntl);
5939	WREG32(IH_CNTL, ih_cntl);
5940	/* set rptr, wptr to 0 */
5941	WREG32(IH_RB_RPTR, 0);
5942	WREG32(IH_RB_WPTR, 0);
5943	rdev->ih.enabled = false;
5944	rdev->ih.rptr = 0;
5945}
5946
5947static void si_disable_interrupt_state(struct radeon_device *rdev)
5948{
5949	int i;
5950	u32 tmp;
5951
5952	tmp = RREG32(CP_INT_CNTL_RING0) &
5953		(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5954	WREG32(CP_INT_CNTL_RING0, tmp);
5955	WREG32(CP_INT_CNTL_RING1, 0);
5956	WREG32(CP_INT_CNTL_RING2, 0);
5957	tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5958	WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5959	tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5960	WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5961	WREG32(GRBM_INT_CNTL, 0);
5962	WREG32(SRBM_INT_CNTL, 0);
5963	for (i = 0; i < rdev->num_crtc; i++)
5964		WREG32(INT_MASK + crtc_offsets[i], 0);
5965	for (i = 0; i < rdev->num_crtc; i++)
5966		WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5967
5968	if (!ASIC_IS_NODCE(rdev)) {
5969		WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5970
5971		for (i = 0; i < 6; i++)
5972			WREG32_AND(DC_HPDx_INT_CONTROL(i),
5973				   DC_HPDx_INT_POLARITY);
 
 
 
 
 
 
 
 
 
5974	}
5975}
5976
5977static int si_irq_init(struct radeon_device *rdev)
5978{
5979	int ret = 0;
5980	int rb_bufsz;
5981	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5982
5983	/* allocate ring */
5984	ret = r600_ih_ring_alloc(rdev);
5985	if (ret)
5986		return ret;
5987
5988	/* disable irqs */
5989	si_disable_interrupts(rdev);
5990
5991	/* init rlc */
5992	ret = si_rlc_resume(rdev);
5993	if (ret) {
5994		r600_ih_ring_fini(rdev);
5995		return ret;
5996	}
5997
5998	/* setup interrupt control */
5999	/* set dummy read address to dummy page address */
6000	WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8);
6001	interrupt_cntl = RREG32(INTERRUPT_CNTL);
6002	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6003	 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6004	 */
6005	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6006	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6007	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6008	WREG32(INTERRUPT_CNTL, interrupt_cntl);
6009
6010	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6011	rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6012
6013	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6014		      IH_WPTR_OVERFLOW_CLEAR |
6015		      (rb_bufsz << 1));
6016
6017	if (rdev->wb.enabled)
6018		ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6019
6020	/* set the writeback address whether it's enabled or not */
6021	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6022	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6023
6024	WREG32(IH_RB_CNTL, ih_rb_cntl);
6025
6026	/* set rptr, wptr to 0 */
6027	WREG32(IH_RB_RPTR, 0);
6028	WREG32(IH_RB_WPTR, 0);
6029
6030	/* Default settings for IH_CNTL (disabled at first) */
6031	ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6032	/* RPTR_REARM only works if msi's are enabled */
6033	if (rdev->msi_enabled)
6034		ih_cntl |= RPTR_REARM;
6035	WREG32(IH_CNTL, ih_cntl);
6036
6037	/* force the active interrupt state to all disabled */
6038	si_disable_interrupt_state(rdev);
6039
6040	pci_set_master(rdev->pdev);
6041
6042	/* enable irqs */
6043	si_enable_interrupts(rdev);
6044
6045	return ret;
6046}
6047
6048/* The order we write back each register here is important */
6049int si_irq_set(struct radeon_device *rdev)
6050{
6051	int i;
6052	u32 cp_int_cntl;
6053	u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
 
 
6054	u32 grbm_int_cntl = 0;
6055	u32 dma_cntl, dma_cntl1;
6056	u32 thermal_int = 0;
6057
6058	if (!rdev->irq.installed) {
6059		WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6060		return -EINVAL;
6061	}
6062	/* don't enable anything if the ih is disabled */
6063	if (!rdev->ih.enabled) {
6064		si_disable_interrupts(rdev);
6065		/* force the active interrupt state to all disabled */
6066		si_disable_interrupt_state(rdev);
6067		return 0;
6068	}
6069
6070	cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6071		(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6072
 
 
 
 
 
 
 
 
 
6073	dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6074	dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6075
6076	thermal_int = RREG32(CG_THERMAL_INT) &
6077		~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6078
6079	/* enable CP interrupts on all rings */
6080	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6081		DRM_DEBUG("si_irq_set: sw int gfx\n");
6082		cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6083	}
6084	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6085		DRM_DEBUG("si_irq_set: sw int cp1\n");
6086		cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6087	}
6088	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6089		DRM_DEBUG("si_irq_set: sw int cp2\n");
6090		cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6091	}
6092	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6093		DRM_DEBUG("si_irq_set: sw int dma\n");
6094		dma_cntl |= TRAP_ENABLE;
6095	}
6096
6097	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6098		DRM_DEBUG("si_irq_set: sw int dma1\n");
6099		dma_cntl1 |= TRAP_ENABLE;
6100	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6101
6102	WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6103	WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6104	WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6105
6106	WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6107	WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6108
6109	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6110
6111	if (rdev->irq.dpm_thermal) {
6112		DRM_DEBUG("dpm thermal\n");
6113		thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6114	}
6115
6116	for (i = 0; i < rdev->num_crtc; i++) {
6117		radeon_irq_kms_set_irq_n_enabled(
6118		    rdev, INT_MASK + crtc_offsets[i], VBLANK_INT_MASK,
6119		    rdev->irq.crtc_vblank_int[i] ||
6120		    atomic_read(&rdev->irq.pflip[i]), "vblank", i);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6121	}
6122
6123	for (i = 0; i < rdev->num_crtc; i++)
6124		WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
6125
6126	if (!ASIC_IS_NODCE(rdev)) {
6127		for (i = 0; i < 6; i++) {
6128			radeon_irq_kms_set_irq_n_enabled(
6129			    rdev, DC_HPDx_INT_CONTROL(i),
6130			    DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
6131			    rdev->irq.hpd[i], "HPD", i);
6132		}
6133	}
6134
6135	WREG32(CG_THERMAL_INT, thermal_int);
6136
6137	/* posting read */
6138	RREG32(SRBM_STATUS);
6139
6140	return 0;
6141}
6142
6143/* The order we write back each register here is important */
6144static inline void si_irq_ack(struct radeon_device *rdev)
6145{
6146	int i, j;
6147	u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
6148	u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
6149
6150	if (ASIC_IS_NODCE(rdev))
6151		return;
6152
6153	for (i = 0; i < 6; i++) {
6154		disp_int[i] = RREG32(si_disp_int_status[i]);
6155		if (i < rdev->num_crtc)
6156			grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
6157	}
6158
6159	/* We write back each interrupt register in pairs of two */
6160	for (i = 0; i < rdev->num_crtc; i += 2) {
6161		for (j = i; j < (i + 2); j++) {
6162			if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
6163				WREG32(GRPH_INT_STATUS + crtc_offsets[j],
6164				       GRPH_PFLIP_INT_CLEAR);
6165		}
6166
6167		for (j = i; j < (i + 2); j++) {
6168			if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
6169				WREG32(VBLANK_STATUS + crtc_offsets[j],
6170				       VBLANK_ACK);
6171			if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
6172				WREG32(VLINE_STATUS + crtc_offsets[j],
6173				       VLINE_ACK);
6174		}
6175	}
6176
6177	for (i = 0; i < 6; i++) {
6178		if (disp_int[i] & DC_HPD1_INTERRUPT)
6179			WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
6180	}
6181
6182	for (i = 0; i < 6; i++) {
6183		if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
6184			WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6185	}
6186}
6187
6188static void si_irq_disable(struct radeon_device *rdev)
6189{
6190	si_disable_interrupts(rdev);
6191	/* Wait and acknowledge irq */
6192	mdelay(1);
6193	si_irq_ack(rdev);
6194	si_disable_interrupt_state(rdev);
6195}
6196
6197static void si_irq_suspend(struct radeon_device *rdev)
6198{
6199	si_irq_disable(rdev);
6200	si_rlc_stop(rdev);
6201}
6202
6203static void si_irq_fini(struct radeon_device *rdev)
6204{
6205	si_irq_suspend(rdev);
6206	r600_ih_ring_fini(rdev);
6207}
6208
6209static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6210{
6211	u32 wptr, tmp;
6212
6213	if (rdev->wb.enabled)
6214		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6215	else
6216		wptr = RREG32(IH_RB_WPTR);
6217
6218	if (wptr & RB_OVERFLOW) {
6219		wptr &= ~RB_OVERFLOW;
6220		/* When a ring buffer overflow happen start parsing interrupt
6221		 * from the last not overwritten vector (wptr + 16). Hopefully
6222		 * this should allow us to catchup.
6223		 */
6224		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6225			 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
6226		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6227		tmp = RREG32(IH_RB_CNTL);
6228		tmp |= IH_WPTR_OVERFLOW_CLEAR;
6229		WREG32(IH_RB_CNTL, tmp);
6230	}
6231	return (wptr & rdev->ih.ptr_mask);
6232}
6233
6234/*        SI IV Ring
6235 * Each IV ring entry is 128 bits:
6236 * [7:0]    - interrupt source id
6237 * [31:8]   - reserved
6238 * [59:32]  - interrupt source data
6239 * [63:60]  - reserved
6240 * [71:64]  - RINGID
6241 * [79:72]  - VMID
6242 * [127:80] - reserved
6243 */
6244int si_irq_process(struct radeon_device *rdev)
6245{
6246	u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
6247	u32 crtc_idx, hpd_idx;
6248	u32 mask;
6249	u32 wptr;
6250	u32 rptr;
6251	u32 src_id, src_data, ring_id;
6252	u32 ring_index;
6253	bool queue_hotplug = false;
6254	bool queue_dp = false;
6255	bool queue_thermal = false;
6256	u32 status, addr;
6257	const char *event_name;
6258
6259	if (!rdev->ih.enabled || rdev->shutdown)
6260		return IRQ_NONE;
6261
6262	wptr = si_get_ih_wptr(rdev);
6263
6264restart_ih:
6265	/* is somebody else already processing irqs? */
6266	if (atomic_xchg(&rdev->ih.lock, 1))
6267		return IRQ_NONE;
6268
6269	rptr = rdev->ih.rptr;
6270	DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6271
6272	/* Order reading of wptr vs. reading of IH ring data */
6273	rmb();
6274
6275	/* display interrupts */
6276	si_irq_ack(rdev);
6277
6278	while (rptr != wptr) {
6279		/* wptr/rptr are in bytes! */
6280		ring_index = rptr / 4;
6281		src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6282		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6283		ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6284
6285		switch (src_id) {
6286		case 1: /* D1 vblank/vline */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6287		case 2: /* D2 vblank/vline */
6288		case 3: /* D3 vblank/vline */
6289		case 4: /* D4 vblank/vline */
6290		case 5: /* D5 vblank/vline */
6291		case 6: /* D6 vblank/vline */
6292			crtc_idx = src_id - 1;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6293
6294			if (src_data == 0) { /* vblank */
6295				mask = LB_D1_VBLANK_INTERRUPT;
6296				event_name = "vblank";
 
 
 
 
 
 
 
 
 
 
 
6297
6298				if (rdev->irq.crtc_vblank_int[crtc_idx]) {
6299					drm_handle_vblank(rdev->ddev, crtc_idx);
6300					rdev->pm.vblank_sync = true;
6301					wake_up(&rdev->irq.vblank_queue);
6302				}
6303				if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
6304					radeon_crtc_handle_vblank(rdev,
6305								  crtc_idx);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6306				}
 
 
 
 
6307
6308			} else if (src_data == 1) { /* vline */
6309				mask = LB_D1_VLINE_INTERRUPT;
6310				event_name = "vline";
6311			} else {
6312				DRM_DEBUG("Unhandled interrupt: %d %d\n",
6313					  src_id, src_data);
 
 
 
 
 
6314				break;
6315			}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6316
6317			if (!(disp_int[crtc_idx] & mask)) {
6318				DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
6319					  crtc_idx + 1, event_name);
 
6320			}
 
 
 
 
 
 
6321
6322			disp_int[crtc_idx] &= ~mask;
6323			DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
 
 
 
 
 
 
 
 
 
 
 
 
6324
 
 
 
 
 
 
 
 
6325			break;
6326		case 8: /* D1 page flip */
6327		case 10: /* D2 page flip */
6328		case 12: /* D3 page flip */
6329		case 14: /* D4 page flip */
6330		case 16: /* D5 page flip */
6331		case 18: /* D6 page flip */
6332			DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
6333			if (radeon_use_pflipirq > 0)
6334				radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6335			break;
6336		case 42: /* HPD hotplug */
6337			if (src_data <= 5) {
6338				hpd_idx = src_data;
6339				mask = DC_HPD1_INTERRUPT;
 
 
 
6340				queue_hotplug = true;
6341				event_name = "HPD";
 
 
 
 
 
6342
6343			} else if (src_data <= 11) {
6344				hpd_idx = src_data - 6;
6345				mask = DC_HPD1_RX_INTERRUPT;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6346				queue_dp = true;
6347				event_name = "HPD_RX";
6348
6349			} else {
6350				DRM_DEBUG("Unhandled interrupt: %d %d\n",
6351					  src_id, src_data);
6352				break;
6353			}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6354
6355			if (!(disp_int[hpd_idx] & mask))
6356				DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
 
6357
6358			disp_int[hpd_idx] &= ~mask;
6359			DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
 
 
 
6360			break;
6361		case 96:
6362			DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
6363			WREG32(SRBM_INT_ACK, 0x1);
6364			break;
6365		case 124: /* UVD */
6366			DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6367			radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6368			break;
6369		case 146:
6370		case 147:
6371			addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6372			status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6373			/* reset addr and status */
6374			WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6375			if (addr == 0x0 && status == 0x0)
6376				break;
6377			dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6378			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
6379				addr);
6380			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6381				status);
6382			si_vm_decode_fault(rdev, status, addr);
6383			break;
6384		case 176: /* RINGID0 CP_INT */
6385			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6386			break;
6387		case 177: /* RINGID1 CP_INT */
6388			radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6389			break;
6390		case 178: /* RINGID2 CP_INT */
6391			radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6392			break;
6393		case 181: /* CP EOP event */
6394			DRM_DEBUG("IH: CP EOP\n");
6395			switch (ring_id) {
6396			case 0:
6397				radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6398				break;
6399			case 1:
6400				radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6401				break;
6402			case 2:
6403				radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6404				break;
6405			}
6406			break;
6407		case 224: /* DMA trap event */
6408			DRM_DEBUG("IH: DMA trap\n");
6409			radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6410			break;
6411		case 230: /* thermal low to high */
6412			DRM_DEBUG("IH: thermal low to high\n");
6413			rdev->pm.dpm.thermal.high_to_low = false;
6414			queue_thermal = true;
6415			break;
6416		case 231: /* thermal high to low */
6417			DRM_DEBUG("IH: thermal high to low\n");
6418			rdev->pm.dpm.thermal.high_to_low = true;
6419			queue_thermal = true;
6420			break;
6421		case 233: /* GUI IDLE */
6422			DRM_DEBUG("IH: GUI idle\n");
6423			break;
6424		case 244: /* DMA trap event */
6425			DRM_DEBUG("IH: DMA1 trap\n");
6426			radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6427			break;
6428		default:
6429			DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6430			break;
6431		}
6432
6433		/* wptr/rptr are in bytes! */
6434		rptr += 16;
6435		rptr &= rdev->ih.ptr_mask;
6436		WREG32(IH_RB_RPTR, rptr);
6437	}
6438	if (queue_dp)
6439		schedule_work(&rdev->dp_work);
6440	if (queue_hotplug)
6441		schedule_delayed_work(&rdev->hotplug_work, 0);
6442	if (queue_thermal && rdev->pm.dpm_enabled)
6443		schedule_work(&rdev->pm.dpm.thermal.work);
6444	rdev->ih.rptr = rptr;
6445	atomic_set(&rdev->ih.lock, 0);
6446
6447	/* make sure wptr hasn't changed while processing */
6448	wptr = si_get_ih_wptr(rdev);
6449	if (wptr != rptr)
6450		goto restart_ih;
6451
6452	return IRQ_HANDLED;
6453}
6454
6455/*
6456 * startup/shutdown callbacks
6457 */
6458static void si_uvd_init(struct radeon_device *rdev)
6459{
6460	int r;
6461
6462	if (!rdev->has_uvd)
6463		return;
6464
6465	r = radeon_uvd_init(rdev);
6466	if (r) {
6467		dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
6468		/*
6469		 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
6470		 * to early fails uvd_v2_2_resume() and thus nothing happens
6471		 * there. So it is pointless to try to go through that code
6472		 * hence why we disable uvd here.
6473		 */
6474		rdev->has_uvd = false;
6475		return;
6476	}
6477	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
6478	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
6479}
6480
6481static void si_uvd_start(struct radeon_device *rdev)
6482{
6483	int r;
6484
6485	if (!rdev->has_uvd)
6486		return;
6487
6488	r = uvd_v2_2_resume(rdev);
6489	if (r) {
6490		dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
6491		goto error;
6492	}
6493	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
6494	if (r) {
6495		dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
6496		goto error;
6497	}
6498	return;
6499
6500error:
6501	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6502}
6503
6504static void si_uvd_resume(struct radeon_device *rdev)
6505{
6506	struct radeon_ring *ring;
6507	int r;
6508
6509	if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
6510		return;
6511
6512	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6513	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
6514	if (r) {
6515		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
6516		return;
6517	}
6518	r = uvd_v1_0_init(rdev);
6519	if (r) {
6520		dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
6521		return;
6522	}
6523}
6524
6525static void si_vce_init(struct radeon_device *rdev)
6526{
6527	int r;
6528
6529	if (!rdev->has_vce)
6530		return;
6531
6532	r = radeon_vce_init(rdev);
6533	if (r) {
6534		dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
6535		/*
6536		 * At this point rdev->vce.vcpu_bo is NULL which trickles down
6537		 * to early fails si_vce_start() and thus nothing happens
6538		 * there. So it is pointless to try to go through that code
6539		 * hence why we disable vce here.
6540		 */
6541		rdev->has_vce = false;
6542		return;
6543	}
6544	rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
6545	r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
6546	rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
6547	r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
6548}
6549
6550static void si_vce_start(struct radeon_device *rdev)
6551{
6552	int r;
6553
6554	if (!rdev->has_vce)
6555		return;
6556
6557	r = radeon_vce_resume(rdev);
6558	if (r) {
6559		dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6560		goto error;
6561	}
6562	r = vce_v1_0_resume(rdev);
6563	if (r) {
6564		dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6565		goto error;
6566	}
6567	r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
6568	if (r) {
6569		dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
6570		goto error;
6571	}
6572	r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
6573	if (r) {
6574		dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
6575		goto error;
6576	}
6577	return;
6578
6579error:
6580	rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
6581	rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
6582}
6583
6584static void si_vce_resume(struct radeon_device *rdev)
6585{
6586	struct radeon_ring *ring;
6587	int r;
6588
6589	if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
6590		return;
6591
6592	ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
6593	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6594	if (r) {
6595		dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6596		return;
6597	}
6598	ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
6599	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6600	if (r) {
6601		dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6602		return;
6603	}
6604	r = vce_v1_0_init(rdev);
6605	if (r) {
6606		dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
6607		return;
6608	}
6609}
6610
6611static int si_startup(struct radeon_device *rdev)
6612{
6613	struct radeon_ring *ring;
6614	int r;
6615
6616	/* enable pcie gen2/3 link */
6617	si_pcie_gen3_enable(rdev);
6618	/* enable aspm */
6619	si_program_aspm(rdev);
6620
6621	/* scratch needs to be initialized before MC */
6622	r = r600_vram_scratch_init(rdev);
6623	if (r)
6624		return r;
6625
6626	si_mc_program(rdev);
6627
6628	if (!rdev->pm.dpm_enabled) {
6629		r = si_mc_load_microcode(rdev);
6630		if (r) {
6631			DRM_ERROR("Failed to load MC firmware!\n");
6632			return r;
6633		}
6634	}
6635
6636	r = si_pcie_gart_enable(rdev);
6637	if (r)
6638		return r;
6639	si_gpu_init(rdev);
6640
6641	/* allocate rlc buffers */
6642	if (rdev->family == CHIP_VERDE) {
6643		rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6644		rdev->rlc.reg_list_size =
6645			(u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6646	}
6647	rdev->rlc.cs_data = si_cs_data;
6648	r = sumo_rlc_init(rdev);
6649	if (r) {
6650		DRM_ERROR("Failed to init rlc BOs!\n");
6651		return r;
6652	}
6653
6654	/* allocate wb buffer */
6655	r = radeon_wb_init(rdev);
6656	if (r)
6657		return r;
6658
6659	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6660	if (r) {
6661		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6662		return r;
6663	}
6664
6665	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6666	if (r) {
6667		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6668		return r;
6669	}
6670
6671	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6672	if (r) {
6673		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6674		return r;
6675	}
6676
6677	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6678	if (r) {
6679		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6680		return r;
6681	}
6682
6683	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6684	if (r) {
6685		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6686		return r;
6687	}
6688
6689	si_uvd_start(rdev);
6690	si_vce_start(rdev);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6691
6692	/* Enable IRQ */
6693	if (!rdev->irq.installed) {
6694		r = radeon_irq_kms_init(rdev);
6695		if (r)
6696			return r;
6697	}
6698
6699	r = si_irq_init(rdev);
6700	if (r) {
6701		DRM_ERROR("radeon: IH init failed (%d).\n", r);
6702		radeon_irq_kms_fini(rdev);
6703		return r;
6704	}
6705	si_irq_set(rdev);
6706
6707	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6708	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6709			     RADEON_CP_PACKET2);
6710	if (r)
6711		return r;
6712
6713	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6714	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6715			     RADEON_CP_PACKET2);
6716	if (r)
6717		return r;
6718
6719	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6720	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6721			     RADEON_CP_PACKET2);
6722	if (r)
6723		return r;
6724
6725	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6726	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6727			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6728	if (r)
6729		return r;
6730
6731	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6732	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6733			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6734	if (r)
6735		return r;
6736
6737	r = si_cp_load_microcode(rdev);
6738	if (r)
6739		return r;
6740	r = si_cp_resume(rdev);
6741	if (r)
6742		return r;
6743
6744	r = cayman_dma_resume(rdev);
6745	if (r)
6746		return r;
6747
6748	si_uvd_resume(rdev);
6749	si_vce_resume(rdev);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6750
6751	r = radeon_ib_pool_init(rdev);
6752	if (r) {
6753		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6754		return r;
6755	}
6756
6757	r = radeon_vm_manager_init(rdev);
6758	if (r) {
6759		dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6760		return r;
6761	}
6762
6763	r = radeon_audio_init(rdev);
6764	if (r)
6765		return r;
6766
6767	return 0;
6768}
6769
6770int si_resume(struct radeon_device *rdev)
6771{
6772	int r;
6773
6774	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6775	 * posting will perform necessary task to bring back GPU into good
6776	 * shape.
6777	 */
6778	/* post card */
6779	atom_asic_init(rdev->mode_info.atom_context);
6780
6781	/* init golden registers */
6782	si_init_golden_registers(rdev);
6783
6784	if (rdev->pm.pm_method == PM_METHOD_DPM)
6785		radeon_pm_resume(rdev);
6786
6787	rdev->accel_working = true;
6788	r = si_startup(rdev);
6789	if (r) {
6790		DRM_ERROR("si startup failed on resume\n");
6791		rdev->accel_working = false;
6792		return r;
6793	}
6794
6795	return r;
6796
6797}
6798
6799int si_suspend(struct radeon_device *rdev)
6800{
6801	radeon_pm_suspend(rdev);
6802	radeon_audio_fini(rdev);
6803	radeon_vm_manager_fini(rdev);
6804	si_cp_enable(rdev, false);
6805	cayman_dma_stop(rdev);
6806	if (rdev->has_uvd) {
6807		radeon_uvd_suspend(rdev);
6808		uvd_v1_0_fini(rdev);
6809	}
6810	if (rdev->has_vce)
6811		radeon_vce_suspend(rdev);
 
6812	si_fini_pg(rdev);
6813	si_fini_cg(rdev);
6814	si_irq_suspend(rdev);
6815	radeon_wb_disable(rdev);
6816	si_pcie_gart_disable(rdev);
6817	return 0;
6818}
6819
6820/* Plan is to move initialization in that function and use
6821 * helper function so that radeon_device_init pretty much
6822 * do nothing more than calling asic specific function. This
6823 * should also allow to remove a bunch of callback function
6824 * like vram_info.
6825 */
6826int si_init(struct radeon_device *rdev)
6827{
6828	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6829	int r;
6830
6831	/* Read BIOS */
6832	if (!radeon_get_bios(rdev)) {
6833		if (ASIC_IS_AVIVO(rdev))
6834			return -EINVAL;
6835	}
6836	/* Must be an ATOMBIOS */
6837	if (!rdev->is_atom_bios) {
6838		dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6839		return -EINVAL;
6840	}
6841	r = radeon_atombios_init(rdev);
6842	if (r)
6843		return r;
6844
6845	/* Post card if necessary */
6846	if (!radeon_card_posted(rdev)) {
6847		if (!rdev->bios) {
6848			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6849			return -EINVAL;
6850		}
6851		DRM_INFO("GPU not posted. posting now...\n");
6852		atom_asic_init(rdev->mode_info.atom_context);
6853	}
6854	/* init golden registers */
6855	si_init_golden_registers(rdev);
6856	/* Initialize scratch registers */
6857	si_scratch_init(rdev);
6858	/* Initialize surface registers */
6859	radeon_surface_init(rdev);
6860	/* Initialize clocks */
6861	radeon_get_clock_info(rdev->ddev);
6862
6863	/* Fence driver */
6864	radeon_fence_driver_init(rdev);
 
 
6865
6866	/* initialize memory controller */
6867	r = si_mc_init(rdev);
6868	if (r)
6869		return r;
6870	/* Memory manager */
6871	r = radeon_bo_init(rdev);
6872	if (r)
6873		return r;
6874
6875	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6876	    !rdev->rlc_fw || !rdev->mc_fw) {
6877		r = si_init_microcode(rdev);
6878		if (r) {
6879			DRM_ERROR("Failed to load firmware!\n");
6880			return r;
6881		}
6882	}
6883
6884	/* Initialize power management */
6885	radeon_pm_init(rdev);
6886
6887	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6888	ring->ring_obj = NULL;
6889	r600_ring_init(rdev, ring, 1024 * 1024);
6890
6891	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6892	ring->ring_obj = NULL;
6893	r600_ring_init(rdev, ring, 1024 * 1024);
6894
6895	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6896	ring->ring_obj = NULL;
6897	r600_ring_init(rdev, ring, 1024 * 1024);
6898
6899	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6900	ring->ring_obj = NULL;
6901	r600_ring_init(rdev, ring, 64 * 1024);
6902
6903	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6904	ring->ring_obj = NULL;
6905	r600_ring_init(rdev, ring, 64 * 1024);
6906
6907	si_uvd_init(rdev);
6908	si_vce_init(rdev);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6909
6910	rdev->ih.ring_obj = NULL;
6911	r600_ih_ring_init(rdev, 64 * 1024);
6912
6913	r = r600_pcie_gart_init(rdev);
6914	if (r)
6915		return r;
6916
6917	rdev->accel_working = true;
6918	r = si_startup(rdev);
6919	if (r) {
6920		dev_err(rdev->dev, "disabling GPU acceleration\n");
6921		si_cp_fini(rdev);
6922		cayman_dma_fini(rdev);
6923		si_irq_fini(rdev);
6924		sumo_rlc_fini(rdev);
6925		radeon_wb_fini(rdev);
6926		radeon_ib_pool_fini(rdev);
6927		radeon_vm_manager_fini(rdev);
6928		radeon_irq_kms_fini(rdev);
6929		si_pcie_gart_fini(rdev);
6930		rdev->accel_working = false;
6931	}
6932
6933	/* Don't start up if the MC ucode is missing.
6934	 * The default clocks and voltages before the MC ucode
6935	 * is loaded are not suffient for advanced operations.
6936	 */
6937	if (!rdev->mc_fw) {
6938		DRM_ERROR("radeon: MC ucode required for NI+.\n");
6939		return -EINVAL;
6940	}
6941
6942	return 0;
6943}
6944
6945void si_fini(struct radeon_device *rdev)
6946{
6947	radeon_pm_fini(rdev);
6948	si_cp_fini(rdev);
6949	cayman_dma_fini(rdev);
6950	si_fini_pg(rdev);
6951	si_fini_cg(rdev);
6952	si_irq_fini(rdev);
6953	sumo_rlc_fini(rdev);
6954	radeon_wb_fini(rdev);
6955	radeon_vm_manager_fini(rdev);
6956	radeon_ib_pool_fini(rdev);
6957	radeon_irq_kms_fini(rdev);
6958	if (rdev->has_uvd) {
6959		uvd_v1_0_fini(rdev);
6960		radeon_uvd_fini(rdev);
6961	}
6962	if (rdev->has_vce)
6963		radeon_vce_fini(rdev);
 
6964	si_pcie_gart_fini(rdev);
6965	r600_vram_scratch_fini(rdev);
6966	radeon_gem_fini(rdev);
6967	radeon_fence_driver_fini(rdev);
6968	radeon_bo_fini(rdev);
6969	radeon_atombios_fini(rdev);
6970	kfree(rdev->bios);
6971	rdev->bios = NULL;
6972}
6973
6974/**
6975 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6976 *
6977 * @rdev: radeon_device pointer
6978 *
6979 * Fetches a GPU clock counter snapshot (SI).
6980 * Returns the 64 bit clock counter snapshot.
6981 */
6982uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6983{
6984	uint64_t clock;
6985
6986	mutex_lock(&rdev->gpu_clock_mutex);
6987	WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6988	clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6989		((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6990	mutex_unlock(&rdev->gpu_clock_mutex);
6991	return clock;
6992}
6993
6994int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6995{
6996	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
6997	int r;
6998
6999	/* bypass vclk and dclk with bclk */
7000	WREG32_P(CG_UPLL_FUNC_CNTL_2,
7001		VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
7002		~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7003
7004	/* put PLL in bypass mode */
7005	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
7006
7007	if (!vclk || !dclk) {
7008		/* keep the Bypass mode */
7009		return 0;
7010	}
7011
7012	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
7013					  16384, 0x03FFFFFF, 0, 128, 5,
7014					  &fb_div, &vclk_div, &dclk_div);
7015	if (r)
7016		return r;
7017
7018	/* set RESET_ANTI_MUX to 0 */
7019	WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7020
7021	/* set VCO_MODE to 1 */
7022	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7023
7024	/* disable sleep mode */
7025	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7026
7027	/* deassert UPLL_RESET */
7028	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7029
7030	mdelay(1);
7031
7032	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7033	if (r)
7034		return r;
7035
7036	/* assert UPLL_RESET again */
7037	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7038
7039	/* disable spread spectrum. */
7040	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7041
7042	/* set feedback divider */
7043	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
7044
7045	/* set ref divider to 0 */
7046	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7047
7048	if (fb_div < 307200)
7049		WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7050	else
7051		WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7052
7053	/* set PDIV_A and PDIV_B */
7054	WREG32_P(CG_UPLL_FUNC_CNTL_2,
7055		UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
7056		~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7057
7058	/* give the PLL some time to settle */
7059	mdelay(15);
7060
7061	/* deassert PLL_RESET */
7062	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7063
7064	mdelay(15);
7065
7066	/* switch from bypass mode to normal mode */
7067	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7068
7069	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7070	if (r)
7071		return r;
7072
7073	/* switch VCLK and DCLK selection */
7074	WREG32_P(CG_UPLL_FUNC_CNTL_2,
7075		VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7076		~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7077
7078	mdelay(100);
7079
7080	return 0;
7081}
7082
7083static void si_pcie_gen3_enable(struct radeon_device *rdev)
7084{
7085	struct pci_dev *root = rdev->pdev->bus->self;
7086	enum pci_bus_speed speed_cap;
7087	u32 speed_cntl, current_data_rate;
7088	int i;
7089	u16 tmp16;
7090
7091	if (pci_is_root_bus(rdev->pdev->bus))
7092		return;
7093
7094	if (radeon_pcie_gen2 == 0)
7095		return;
7096
7097	if (rdev->flags & RADEON_IS_IGP)
7098		return;
7099
7100	if (!(rdev->flags & RADEON_IS_PCIE))
7101		return;
7102
7103	speed_cap = pcie_get_speed_cap(root);
7104	if (speed_cap == PCI_SPEED_UNKNOWN)
7105		return;
7106
7107	if ((speed_cap != PCIE_SPEED_8_0GT) &&
7108	    (speed_cap != PCIE_SPEED_5_0GT))
7109		return;
7110
7111	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7112	current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7113		LC_CURRENT_DATA_RATE_SHIFT;
7114	if (speed_cap == PCIE_SPEED_8_0GT) {
7115		if (current_data_rate == 2) {
7116			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7117			return;
7118		}
7119		DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7120	} else if (speed_cap == PCIE_SPEED_5_0GT) {
7121		if (current_data_rate == 1) {
7122			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7123			return;
7124		}
7125		DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7126	}
7127
7128	if (!pci_is_pcie(root) || !pci_is_pcie(rdev->pdev))
 
7129		return;
7130
7131	if (speed_cap == PCIE_SPEED_8_0GT) {
 
 
 
 
7132		/* re-try equalization if gen3 is not already enabled */
7133		if (current_data_rate != 2) {
7134			u16 bridge_cfg, gpu_cfg;
7135			u16 bridge_cfg2, gpu_cfg2;
7136			u32 max_lw, current_lw, tmp;
7137
7138			pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
7139			pcie_capability_set_word(rdev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
 
 
 
 
 
 
7140
7141			tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7142			max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7143			current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7144
7145			if (current_lw < max_lw) {
7146				tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7147				if (tmp & LC_RENEGOTIATION_SUPPORT) {
7148					tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7149					tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7150					tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7151					WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7152				}
7153			}
7154
7155			for (i = 0; i < 10; i++) {
7156				/* check status */
7157				pcie_capability_read_word(rdev->pdev,
7158							  PCI_EXP_DEVSTA,
7159							  &tmp16);
7160				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7161					break;
7162
7163				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
7164							  &bridge_cfg);
7165				pcie_capability_read_word(rdev->pdev,
7166							  PCI_EXP_LNKCTL,
7167							  &gpu_cfg);
7168
7169				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
7170							  &bridge_cfg2);
7171				pcie_capability_read_word(rdev->pdev,
7172							  PCI_EXP_LNKCTL2,
7173							  &gpu_cfg2);
7174
7175				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7176				tmp |= LC_SET_QUIESCE;
7177				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7178
7179				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7180				tmp |= LC_REDO_EQ;
7181				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7182
7183				msleep(100);
7184
7185				/* linkctl */
7186				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,
7187								   PCI_EXP_LNKCTL_HAWD,
7188								   bridge_cfg &
7189								   PCI_EXP_LNKCTL_HAWD);
7190				pcie_capability_clear_and_set_word(rdev->pdev, PCI_EXP_LNKCTL,
7191								   PCI_EXP_LNKCTL_HAWD,
7192								   gpu_cfg &
7193								   PCI_EXP_LNKCTL_HAWD);
 
7194
7195				/* linkctl2 */
7196				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
7197							  &tmp16);
7198				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
7199					   PCI_EXP_LNKCTL2_TX_MARGIN);
7200				tmp16 |= (bridge_cfg2 &
7201					  (PCI_EXP_LNKCTL2_ENTER_COMP |
7202					   PCI_EXP_LNKCTL2_TX_MARGIN));
7203				pcie_capability_write_word(root,
7204							   PCI_EXP_LNKCTL2,
7205							   tmp16);
7206
7207				pcie_capability_read_word(rdev->pdev,
7208							  PCI_EXP_LNKCTL2,
7209							  &tmp16);
7210				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
7211					   PCI_EXP_LNKCTL2_TX_MARGIN);
7212				tmp16 |= (gpu_cfg2 &
7213					  (PCI_EXP_LNKCTL2_ENTER_COMP |
7214					   PCI_EXP_LNKCTL2_TX_MARGIN));
7215				pcie_capability_write_word(rdev->pdev,
7216							   PCI_EXP_LNKCTL2,
7217							   tmp16);
7218
7219				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7220				tmp &= ~LC_SET_QUIESCE;
7221				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7222			}
7223		}
7224	}
7225
7226	/* set the link speed */
7227	speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7228	speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7229	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7230
7231	pcie_capability_read_word(rdev->pdev, PCI_EXP_LNKCTL2, &tmp16);
7232	tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
7233	if (speed_cap == PCIE_SPEED_8_0GT)
7234		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
7235	else if (speed_cap == PCIE_SPEED_5_0GT)
7236		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
7237	else
7238		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
7239	pcie_capability_write_word(rdev->pdev, PCI_EXP_LNKCTL2, tmp16);
7240
7241	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7242	speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7243	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7244
7245	for (i = 0; i < rdev->usec_timeout; i++) {
7246		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7247		if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7248			break;
7249		udelay(1);
7250	}
7251}
7252
7253static void si_program_aspm(struct radeon_device *rdev)
7254{
7255	u32 data, orig;
7256	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7257	bool disable_clkreq = false;
7258
7259	if (radeon_aspm == 0)
7260		return;
7261
7262	if (!(rdev->flags & RADEON_IS_PCIE))
7263		return;
7264
7265	orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7266	data &= ~LC_XMIT_N_FTS_MASK;
7267	data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7268	if (orig != data)
7269		WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7270
7271	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7272	data |= LC_GO_TO_RECOVERY;
7273	if (orig != data)
7274		WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7275
7276	orig = data = RREG32_PCIE(PCIE_P_CNTL);
7277	data |= P_IGNORE_EDB_ERR;
7278	if (orig != data)
7279		WREG32_PCIE(PCIE_P_CNTL, data);
7280
7281	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7282	data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7283	data |= LC_PMI_TO_L1_DIS;
7284	if (!disable_l0s)
7285		data |= LC_L0S_INACTIVITY(7);
7286
7287	if (!disable_l1) {
7288		data |= LC_L1_INACTIVITY(7);
7289		data &= ~LC_PMI_TO_L1_DIS;
7290		if (orig != data)
7291			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7292
7293		if (!disable_plloff_in_l1) {
7294			bool clk_req_support;
7295
7296			orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7297			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7298			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7299			if (orig != data)
7300				WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7301
7302			orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7303			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7304			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7305			if (orig != data)
7306				WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7307
7308			orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7309			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7310			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7311			if (orig != data)
7312				WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7313
7314			orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7315			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7316			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7317			if (orig != data)
7318				WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7319
7320			if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7321				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7322				data &= ~PLL_RAMP_UP_TIME_0_MASK;
7323				if (orig != data)
7324					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7325
7326				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7327				data &= ~PLL_RAMP_UP_TIME_1_MASK;
7328				if (orig != data)
7329					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7330
7331				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7332				data &= ~PLL_RAMP_UP_TIME_2_MASK;
7333				if (orig != data)
7334					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7335
7336				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7337				data &= ~PLL_RAMP_UP_TIME_3_MASK;
7338				if (orig != data)
7339					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7340
7341				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7342				data &= ~PLL_RAMP_UP_TIME_0_MASK;
7343				if (orig != data)
7344					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7345
7346				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7347				data &= ~PLL_RAMP_UP_TIME_1_MASK;
7348				if (orig != data)
7349					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7350
7351				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7352				data &= ~PLL_RAMP_UP_TIME_2_MASK;
7353				if (orig != data)
7354					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7355
7356				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7357				data &= ~PLL_RAMP_UP_TIME_3_MASK;
7358				if (orig != data)
7359					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7360			}
7361			orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7362			data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7363			data |= LC_DYN_LANES_PWR_STATE(3);
7364			if (orig != data)
7365				WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7366
7367			orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7368			data &= ~LS2_EXIT_TIME_MASK;
7369			if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7370				data |= LS2_EXIT_TIME(5);
7371			if (orig != data)
7372				WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7373
7374			orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7375			data &= ~LS2_EXIT_TIME_MASK;
7376			if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7377				data |= LS2_EXIT_TIME(5);
7378			if (orig != data)
7379				WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7380
7381			if (!disable_clkreq &&
7382			    !pci_is_root_bus(rdev->pdev->bus)) {
7383				struct pci_dev *root = rdev->pdev->bus->self;
7384				u32 lnkcap;
7385
7386				clk_req_support = false;
7387				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7388				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7389					clk_req_support = true;
7390			} else {
7391				clk_req_support = false;
7392			}
7393
7394			if (clk_req_support) {
7395				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7396				data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7397				if (orig != data)
7398					WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7399
7400				orig = data = RREG32(THM_CLK_CNTL);
7401				data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7402				data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7403				if (orig != data)
7404					WREG32(THM_CLK_CNTL, data);
7405
7406				orig = data = RREG32(MISC_CLK_CNTL);
7407				data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7408				data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7409				if (orig != data)
7410					WREG32(MISC_CLK_CNTL, data);
7411
7412				orig = data = RREG32(CG_CLKPIN_CNTL);
7413				data &= ~BCLK_AS_XCLK;
7414				if (orig != data)
7415					WREG32(CG_CLKPIN_CNTL, data);
7416
7417				orig = data = RREG32(CG_CLKPIN_CNTL_2);
7418				data &= ~FORCE_BIF_REFCLK_EN;
7419				if (orig != data)
7420					WREG32(CG_CLKPIN_CNTL_2, data);
7421
7422				orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7423				data &= ~MPLL_CLKOUT_SEL_MASK;
7424				data |= MPLL_CLKOUT_SEL(4);
7425				if (orig != data)
7426					WREG32(MPLL_BYPASSCLK_SEL, data);
7427
7428				orig = data = RREG32(SPLL_CNTL_MODE);
7429				data &= ~SPLL_REFCLK_SEL_MASK;
7430				if (orig != data)
7431					WREG32(SPLL_CNTL_MODE, data);
7432			}
7433		}
7434	} else {
7435		if (orig != data)
7436			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7437	}
7438
7439	orig = data = RREG32_PCIE(PCIE_CNTL2);
7440	data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7441	if (orig != data)
7442		WREG32_PCIE(PCIE_CNTL2, data);
7443
7444	if (!disable_l0s) {
7445		data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7446		if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7447			data = RREG32_PCIE(PCIE_LC_STATUS1);
7448			if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7449				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7450				data &= ~LC_L0S_INACTIVITY_MASK;
7451				if (orig != data)
7452					WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7453			}
7454		}
7455	}
7456}
7457
7458static int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
7459{
7460	unsigned i;
7461
7462	/* make sure VCEPLL_CTLREQ is deasserted */
7463	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7464
7465	mdelay(10);
7466
7467	/* assert UPLL_CTLREQ */
7468	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
7469
7470	/* wait for CTLACK and CTLACK2 to get asserted */
7471	for (i = 0; i < 100; ++i) {
7472		uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
7473		if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
7474			break;
7475		mdelay(10);
7476	}
7477
7478	/* deassert UPLL_CTLREQ */
7479	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7480
7481	if (i == 100) {
7482		DRM_ERROR("Timeout setting UVD clocks!\n");
7483		return -ETIMEDOUT;
7484	}
7485
7486	return 0;
7487}
7488
7489int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
7490{
7491	unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
7492	int r;
7493
7494	/* bypass evclk and ecclk with bclk */
7495	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7496		     EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
7497		     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7498
7499	/* put PLL in bypass mode */
7500	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
7501		     ~VCEPLL_BYPASS_EN_MASK);
7502
7503	if (!evclk || !ecclk) {
7504		/* keep the Bypass mode, put PLL to sleep */
7505		WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7506			     ~VCEPLL_SLEEP_MASK);
7507		return 0;
7508	}
7509
7510	r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
7511					  16384, 0x03FFFFFF, 0, 128, 5,
7512					  &fb_div, &evclk_div, &ecclk_div);
7513	if (r)
7514		return r;
7515
7516	/* set RESET_ANTI_MUX to 0 */
7517	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7518
7519	/* set VCO_MODE to 1 */
7520	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
7521		     ~VCEPLL_VCO_MODE_MASK);
7522
7523	/* toggle VCEPLL_SLEEP to 1 then back to 0 */
7524	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7525		     ~VCEPLL_SLEEP_MASK);
7526	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
7527
7528	/* deassert VCEPLL_RESET */
7529	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7530
7531	mdelay(1);
7532
7533	r = si_vce_send_vcepll_ctlreq(rdev);
7534	if (r)
7535		return r;
7536
7537	/* assert VCEPLL_RESET again */
7538	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
7539
7540	/* disable spread spectrum. */
7541	WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7542
7543	/* set feedback divider */
7544	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK);
7545
7546	/* set ref divider to 0 */
7547	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
7548
7549	/* set PDIV_A and PDIV_B */
7550	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7551		     VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
7552		     ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
7553
7554	/* give the PLL some time to settle */
7555	mdelay(15);
7556
7557	/* deassert PLL_RESET */
7558	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7559
7560	mdelay(15);
7561
7562	/* switch from bypass mode to normal mode */
7563	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
7564
7565	r = si_vce_send_vcepll_ctlreq(rdev);
7566	if (r)
7567		return r;
7568
7569	/* switch VCLK and DCLK selection */
7570	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7571		     EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
7572		     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7573
7574	mdelay(100);
7575
7576	return 0;
7577}
v4.6
   1/*
   2 * Copyright 2011 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
 
  24#include <linux/firmware.h>
 
 
  25#include <linux/slab.h>
  26#include <linux/module.h>
  27#include <drm/drmP.h>
 
 
 
 
 
 
  28#include "radeon.h"
  29#include "radeon_asic.h"
  30#include "radeon_audio.h"
  31#include <drm/radeon_drm.h>
 
 
  32#include "sid.h"
  33#include "atom.h"
  34#include "si_blit_shaders.h"
  35#include "clearstate_si.h"
  36#include "radeon_ucode.h"
  37
  38
  39MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
  40MODULE_FIRMWARE("radeon/TAHITI_me.bin");
  41MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
  42MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
  43MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
  44MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
  45MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
  46
  47MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
  48MODULE_FIRMWARE("radeon/tahiti_me.bin");
  49MODULE_FIRMWARE("radeon/tahiti_ce.bin");
  50MODULE_FIRMWARE("radeon/tahiti_mc.bin");
  51MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
  52MODULE_FIRMWARE("radeon/tahiti_smc.bin");
  53
  54MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
  55MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
  56MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
  57MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
  58MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
  59MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
  60MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
  61
  62MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
  63MODULE_FIRMWARE("radeon/pitcairn_me.bin");
  64MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
  65MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
  66MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
  67MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
 
  68
  69MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
  70MODULE_FIRMWARE("radeon/VERDE_me.bin");
  71MODULE_FIRMWARE("radeon/VERDE_ce.bin");
  72MODULE_FIRMWARE("radeon/VERDE_mc.bin");
  73MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
  74MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
  75MODULE_FIRMWARE("radeon/VERDE_smc.bin");
  76
  77MODULE_FIRMWARE("radeon/verde_pfp.bin");
  78MODULE_FIRMWARE("radeon/verde_me.bin");
  79MODULE_FIRMWARE("radeon/verde_ce.bin");
  80MODULE_FIRMWARE("radeon/verde_mc.bin");
  81MODULE_FIRMWARE("radeon/verde_rlc.bin");
  82MODULE_FIRMWARE("radeon/verde_smc.bin");
 
  83
  84MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
  85MODULE_FIRMWARE("radeon/OLAND_me.bin");
  86MODULE_FIRMWARE("radeon/OLAND_ce.bin");
  87MODULE_FIRMWARE("radeon/OLAND_mc.bin");
  88MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
  89MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
  90MODULE_FIRMWARE("radeon/OLAND_smc.bin");
  91
  92MODULE_FIRMWARE("radeon/oland_pfp.bin");
  93MODULE_FIRMWARE("radeon/oland_me.bin");
  94MODULE_FIRMWARE("radeon/oland_ce.bin");
  95MODULE_FIRMWARE("radeon/oland_mc.bin");
  96MODULE_FIRMWARE("radeon/oland_rlc.bin");
  97MODULE_FIRMWARE("radeon/oland_smc.bin");
 
  98
  99MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
 100MODULE_FIRMWARE("radeon/HAINAN_me.bin");
 101MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
 102MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
 103MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
 104MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
 105MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
 106
 107MODULE_FIRMWARE("radeon/hainan_pfp.bin");
 108MODULE_FIRMWARE("radeon/hainan_me.bin");
 109MODULE_FIRMWARE("radeon/hainan_ce.bin");
 110MODULE_FIRMWARE("radeon/hainan_mc.bin");
 111MODULE_FIRMWARE("radeon/hainan_rlc.bin");
 112MODULE_FIRMWARE("radeon/hainan_smc.bin");
 
 
 
 
 113
 114static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
 115static void si_pcie_gen3_enable(struct radeon_device *rdev);
 116static void si_program_aspm(struct radeon_device *rdev);
 117extern void sumo_rlc_fini(struct radeon_device *rdev);
 118extern int sumo_rlc_init(struct radeon_device *rdev);
 119extern int r600_ih_ring_alloc(struct radeon_device *rdev);
 120extern void r600_ih_ring_fini(struct radeon_device *rdev);
 121extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 122extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 123extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 124extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
 125extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 126extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 127static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
 128					 bool enable);
 129static void si_init_pg(struct radeon_device *rdev);
 130static void si_init_cg(struct radeon_device *rdev);
 131static void si_fini_pg(struct radeon_device *rdev);
 132static void si_fini_cg(struct radeon_device *rdev);
 133static void si_rlc_stop(struct radeon_device *rdev);
 134
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 135static const u32 verde_rlc_save_restore_register_list[] =
 136{
 137	(0x8000 << 16) | (0x98f4 >> 2),
 138	0x00000000,
 139	(0x8040 << 16) | (0x98f4 >> 2),
 140	0x00000000,
 141	(0x8000 << 16) | (0xe80 >> 2),
 142	0x00000000,
 143	(0x8040 << 16) | (0xe80 >> 2),
 144	0x00000000,
 145	(0x8000 << 16) | (0x89bc >> 2),
 146	0x00000000,
 147	(0x8040 << 16) | (0x89bc >> 2),
 148	0x00000000,
 149	(0x8000 << 16) | (0x8c1c >> 2),
 150	0x00000000,
 151	(0x8040 << 16) | (0x8c1c >> 2),
 152	0x00000000,
 153	(0x9c00 << 16) | (0x98f0 >> 2),
 154	0x00000000,
 155	(0x9c00 << 16) | (0xe7c >> 2),
 156	0x00000000,
 157	(0x8000 << 16) | (0x9148 >> 2),
 158	0x00000000,
 159	(0x8040 << 16) | (0x9148 >> 2),
 160	0x00000000,
 161	(0x9c00 << 16) | (0x9150 >> 2),
 162	0x00000000,
 163	(0x9c00 << 16) | (0x897c >> 2),
 164	0x00000000,
 165	(0x9c00 << 16) | (0x8d8c >> 2),
 166	0x00000000,
 167	(0x9c00 << 16) | (0xac54 >> 2),
 168	0X00000000,
 169	0x3,
 170	(0x9c00 << 16) | (0x98f8 >> 2),
 171	0x00000000,
 172	(0x9c00 << 16) | (0x9910 >> 2),
 173	0x00000000,
 174	(0x9c00 << 16) | (0x9914 >> 2),
 175	0x00000000,
 176	(0x9c00 << 16) | (0x9918 >> 2),
 177	0x00000000,
 178	(0x9c00 << 16) | (0x991c >> 2),
 179	0x00000000,
 180	(0x9c00 << 16) | (0x9920 >> 2),
 181	0x00000000,
 182	(0x9c00 << 16) | (0x9924 >> 2),
 183	0x00000000,
 184	(0x9c00 << 16) | (0x9928 >> 2),
 185	0x00000000,
 186	(0x9c00 << 16) | (0x992c >> 2),
 187	0x00000000,
 188	(0x9c00 << 16) | (0x9930 >> 2),
 189	0x00000000,
 190	(0x9c00 << 16) | (0x9934 >> 2),
 191	0x00000000,
 192	(0x9c00 << 16) | (0x9938 >> 2),
 193	0x00000000,
 194	(0x9c00 << 16) | (0x993c >> 2),
 195	0x00000000,
 196	(0x9c00 << 16) | (0x9940 >> 2),
 197	0x00000000,
 198	(0x9c00 << 16) | (0x9944 >> 2),
 199	0x00000000,
 200	(0x9c00 << 16) | (0x9948 >> 2),
 201	0x00000000,
 202	(0x9c00 << 16) | (0x994c >> 2),
 203	0x00000000,
 204	(0x9c00 << 16) | (0x9950 >> 2),
 205	0x00000000,
 206	(0x9c00 << 16) | (0x9954 >> 2),
 207	0x00000000,
 208	(0x9c00 << 16) | (0x9958 >> 2),
 209	0x00000000,
 210	(0x9c00 << 16) | (0x995c >> 2),
 211	0x00000000,
 212	(0x9c00 << 16) | (0x9960 >> 2),
 213	0x00000000,
 214	(0x9c00 << 16) | (0x9964 >> 2),
 215	0x00000000,
 216	(0x9c00 << 16) | (0x9968 >> 2),
 217	0x00000000,
 218	(0x9c00 << 16) | (0x996c >> 2),
 219	0x00000000,
 220	(0x9c00 << 16) | (0x9970 >> 2),
 221	0x00000000,
 222	(0x9c00 << 16) | (0x9974 >> 2),
 223	0x00000000,
 224	(0x9c00 << 16) | (0x9978 >> 2),
 225	0x00000000,
 226	(0x9c00 << 16) | (0x997c >> 2),
 227	0x00000000,
 228	(0x9c00 << 16) | (0x9980 >> 2),
 229	0x00000000,
 230	(0x9c00 << 16) | (0x9984 >> 2),
 231	0x00000000,
 232	(0x9c00 << 16) | (0x9988 >> 2),
 233	0x00000000,
 234	(0x9c00 << 16) | (0x998c >> 2),
 235	0x00000000,
 236	(0x9c00 << 16) | (0x8c00 >> 2),
 237	0x00000000,
 238	(0x9c00 << 16) | (0x8c14 >> 2),
 239	0x00000000,
 240	(0x9c00 << 16) | (0x8c04 >> 2),
 241	0x00000000,
 242	(0x9c00 << 16) | (0x8c08 >> 2),
 243	0x00000000,
 244	(0x8000 << 16) | (0x9b7c >> 2),
 245	0x00000000,
 246	(0x8040 << 16) | (0x9b7c >> 2),
 247	0x00000000,
 248	(0x8000 << 16) | (0xe84 >> 2),
 249	0x00000000,
 250	(0x8040 << 16) | (0xe84 >> 2),
 251	0x00000000,
 252	(0x8000 << 16) | (0x89c0 >> 2),
 253	0x00000000,
 254	(0x8040 << 16) | (0x89c0 >> 2),
 255	0x00000000,
 256	(0x8000 << 16) | (0x914c >> 2),
 257	0x00000000,
 258	(0x8040 << 16) | (0x914c >> 2),
 259	0x00000000,
 260	(0x8000 << 16) | (0x8c20 >> 2),
 261	0x00000000,
 262	(0x8040 << 16) | (0x8c20 >> 2),
 263	0x00000000,
 264	(0x8000 << 16) | (0x9354 >> 2),
 265	0x00000000,
 266	(0x8040 << 16) | (0x9354 >> 2),
 267	0x00000000,
 268	(0x9c00 << 16) | (0x9060 >> 2),
 269	0x00000000,
 270	(0x9c00 << 16) | (0x9364 >> 2),
 271	0x00000000,
 272	(0x9c00 << 16) | (0x9100 >> 2),
 273	0x00000000,
 274	(0x9c00 << 16) | (0x913c >> 2),
 275	0x00000000,
 276	(0x8000 << 16) | (0x90e0 >> 2),
 277	0x00000000,
 278	(0x8000 << 16) | (0x90e4 >> 2),
 279	0x00000000,
 280	(0x8000 << 16) | (0x90e8 >> 2),
 281	0x00000000,
 282	(0x8040 << 16) | (0x90e0 >> 2),
 283	0x00000000,
 284	(0x8040 << 16) | (0x90e4 >> 2),
 285	0x00000000,
 286	(0x8040 << 16) | (0x90e8 >> 2),
 287	0x00000000,
 288	(0x9c00 << 16) | (0x8bcc >> 2),
 289	0x00000000,
 290	(0x9c00 << 16) | (0x8b24 >> 2),
 291	0x00000000,
 292	(0x9c00 << 16) | (0x88c4 >> 2),
 293	0x00000000,
 294	(0x9c00 << 16) | (0x8e50 >> 2),
 295	0x00000000,
 296	(0x9c00 << 16) | (0x8c0c >> 2),
 297	0x00000000,
 298	(0x9c00 << 16) | (0x8e58 >> 2),
 299	0x00000000,
 300	(0x9c00 << 16) | (0x8e5c >> 2),
 301	0x00000000,
 302	(0x9c00 << 16) | (0x9508 >> 2),
 303	0x00000000,
 304	(0x9c00 << 16) | (0x950c >> 2),
 305	0x00000000,
 306	(0x9c00 << 16) | (0x9494 >> 2),
 307	0x00000000,
 308	(0x9c00 << 16) | (0xac0c >> 2),
 309	0x00000000,
 310	(0x9c00 << 16) | (0xac10 >> 2),
 311	0x00000000,
 312	(0x9c00 << 16) | (0xac14 >> 2),
 313	0x00000000,
 314	(0x9c00 << 16) | (0xae00 >> 2),
 315	0x00000000,
 316	(0x9c00 << 16) | (0xac08 >> 2),
 317	0x00000000,
 318	(0x9c00 << 16) | (0x88d4 >> 2),
 319	0x00000000,
 320	(0x9c00 << 16) | (0x88c8 >> 2),
 321	0x00000000,
 322	(0x9c00 << 16) | (0x88cc >> 2),
 323	0x00000000,
 324	(0x9c00 << 16) | (0x89b0 >> 2),
 325	0x00000000,
 326	(0x9c00 << 16) | (0x8b10 >> 2),
 327	0x00000000,
 328	(0x9c00 << 16) | (0x8a14 >> 2),
 329	0x00000000,
 330	(0x9c00 << 16) | (0x9830 >> 2),
 331	0x00000000,
 332	(0x9c00 << 16) | (0x9834 >> 2),
 333	0x00000000,
 334	(0x9c00 << 16) | (0x9838 >> 2),
 335	0x00000000,
 336	(0x9c00 << 16) | (0x9a10 >> 2),
 337	0x00000000,
 338	(0x8000 << 16) | (0x9870 >> 2),
 339	0x00000000,
 340	(0x8000 << 16) | (0x9874 >> 2),
 341	0x00000000,
 342	(0x8001 << 16) | (0x9870 >> 2),
 343	0x00000000,
 344	(0x8001 << 16) | (0x9874 >> 2),
 345	0x00000000,
 346	(0x8040 << 16) | (0x9870 >> 2),
 347	0x00000000,
 348	(0x8040 << 16) | (0x9874 >> 2),
 349	0x00000000,
 350	(0x8041 << 16) | (0x9870 >> 2),
 351	0x00000000,
 352	(0x8041 << 16) | (0x9874 >> 2),
 353	0x00000000,
 354	0x00000000
 355};
 356
 357static const u32 tahiti_golden_rlc_registers[] =
 358{
 359	0xc424, 0xffffffff, 0x00601005,
 360	0xc47c, 0xffffffff, 0x10104040,
 361	0xc488, 0xffffffff, 0x0100000a,
 362	0xc314, 0xffffffff, 0x00000800,
 363	0xc30c, 0xffffffff, 0x800000f4,
 364	0xf4a8, 0xffffffff, 0x00000000
 365};
 366
 367static const u32 tahiti_golden_registers[] =
 368{
 369	0x9a10, 0x00010000, 0x00018208,
 370	0x9830, 0xffffffff, 0x00000000,
 371	0x9834, 0xf00fffff, 0x00000400,
 372	0x9838, 0x0002021c, 0x00020200,
 373	0xc78, 0x00000080, 0x00000000,
 374	0xd030, 0x000300c0, 0x00800040,
 375	0xd830, 0x000300c0, 0x00800040,
 376	0x5bb0, 0x000000f0, 0x00000070,
 377	0x5bc0, 0x00200000, 0x50100000,
 378	0x7030, 0x31000311, 0x00000011,
 379	0x277c, 0x00000003, 0x000007ff,
 380	0x240c, 0x000007ff, 0x00000000,
 381	0x8a14, 0xf000001f, 0x00000007,
 382	0x8b24, 0xffffffff, 0x00ffffff,
 383	0x8b10, 0x0000ff0f, 0x00000000,
 384	0x28a4c, 0x07ffffff, 0x4e000000,
 385	0x28350, 0x3f3f3fff, 0x2a00126a,
 386	0x30, 0x000000ff, 0x0040,
 387	0x34, 0x00000040, 0x00004040,
 388	0x9100, 0x07ffffff, 0x03000000,
 389	0x8e88, 0x01ff1f3f, 0x00000000,
 390	0x8e84, 0x01ff1f3f, 0x00000000,
 391	0x9060, 0x0000007f, 0x00000020,
 392	0x9508, 0x00010000, 0x00010000,
 393	0xac14, 0x00000200, 0x000002fb,
 394	0xac10, 0xffffffff, 0x0000543b,
 395	0xac0c, 0xffffffff, 0xa9210876,
 396	0x88d0, 0xffffffff, 0x000fff40,
 397	0x88d4, 0x0000001f, 0x00000010,
 398	0x1410, 0x20000000, 0x20fffed8,
 399	0x15c0, 0x000c0fc0, 0x000c0400
 400};
 401
 402static const u32 tahiti_golden_registers2[] =
 403{
 404	0xc64, 0x00000001, 0x00000001
 405};
 406
 407static const u32 pitcairn_golden_rlc_registers[] =
 408{
 409	0xc424, 0xffffffff, 0x00601004,
 410	0xc47c, 0xffffffff, 0x10102020,
 411	0xc488, 0xffffffff, 0x01000020,
 412	0xc314, 0xffffffff, 0x00000800,
 413	0xc30c, 0xffffffff, 0x800000a4
 414};
 415
 416static const u32 pitcairn_golden_registers[] =
 417{
 418	0x9a10, 0x00010000, 0x00018208,
 419	0x9830, 0xffffffff, 0x00000000,
 420	0x9834, 0xf00fffff, 0x00000400,
 421	0x9838, 0x0002021c, 0x00020200,
 422	0xc78, 0x00000080, 0x00000000,
 423	0xd030, 0x000300c0, 0x00800040,
 424	0xd830, 0x000300c0, 0x00800040,
 425	0x5bb0, 0x000000f0, 0x00000070,
 426	0x5bc0, 0x00200000, 0x50100000,
 427	0x7030, 0x31000311, 0x00000011,
 428	0x2ae4, 0x00073ffe, 0x000022a2,
 429	0x240c, 0x000007ff, 0x00000000,
 430	0x8a14, 0xf000001f, 0x00000007,
 431	0x8b24, 0xffffffff, 0x00ffffff,
 432	0x8b10, 0x0000ff0f, 0x00000000,
 433	0x28a4c, 0x07ffffff, 0x4e000000,
 434	0x28350, 0x3f3f3fff, 0x2a00126a,
 435	0x30, 0x000000ff, 0x0040,
 436	0x34, 0x00000040, 0x00004040,
 437	0x9100, 0x07ffffff, 0x03000000,
 438	0x9060, 0x0000007f, 0x00000020,
 439	0x9508, 0x00010000, 0x00010000,
 440	0xac14, 0x000003ff, 0x000000f7,
 441	0xac10, 0xffffffff, 0x00000000,
 442	0xac0c, 0xffffffff, 0x32761054,
 443	0x88d4, 0x0000001f, 0x00000010,
 444	0x15c0, 0x000c0fc0, 0x000c0400
 445};
 446
 447static const u32 verde_golden_rlc_registers[] =
 448{
 449	0xc424, 0xffffffff, 0x033f1005,
 450	0xc47c, 0xffffffff, 0x10808020,
 451	0xc488, 0xffffffff, 0x00800008,
 452	0xc314, 0xffffffff, 0x00001000,
 453	0xc30c, 0xffffffff, 0x80010014
 454};
 455
 456static const u32 verde_golden_registers[] =
 457{
 458	0x9a10, 0x00010000, 0x00018208,
 459	0x9830, 0xffffffff, 0x00000000,
 460	0x9834, 0xf00fffff, 0x00000400,
 461	0x9838, 0x0002021c, 0x00020200,
 462	0xc78, 0x00000080, 0x00000000,
 463	0xd030, 0x000300c0, 0x00800040,
 464	0xd030, 0x000300c0, 0x00800040,
 465	0xd830, 0x000300c0, 0x00800040,
 466	0xd830, 0x000300c0, 0x00800040,
 467	0x5bb0, 0x000000f0, 0x00000070,
 468	0x5bc0, 0x00200000, 0x50100000,
 469	0x7030, 0x31000311, 0x00000011,
 470	0x2ae4, 0x00073ffe, 0x000022a2,
 471	0x2ae4, 0x00073ffe, 0x000022a2,
 472	0x2ae4, 0x00073ffe, 0x000022a2,
 473	0x240c, 0x000007ff, 0x00000000,
 474	0x240c, 0x000007ff, 0x00000000,
 475	0x240c, 0x000007ff, 0x00000000,
 476	0x8a14, 0xf000001f, 0x00000007,
 477	0x8a14, 0xf000001f, 0x00000007,
 478	0x8a14, 0xf000001f, 0x00000007,
 479	0x8b24, 0xffffffff, 0x00ffffff,
 480	0x8b10, 0x0000ff0f, 0x00000000,
 481	0x28a4c, 0x07ffffff, 0x4e000000,
 482	0x28350, 0x3f3f3fff, 0x0000124a,
 483	0x28350, 0x3f3f3fff, 0x0000124a,
 484	0x28350, 0x3f3f3fff, 0x0000124a,
 485	0x30, 0x000000ff, 0x0040,
 486	0x34, 0x00000040, 0x00004040,
 487	0x9100, 0x07ffffff, 0x03000000,
 488	0x9100, 0x07ffffff, 0x03000000,
 489	0x8e88, 0x01ff1f3f, 0x00000000,
 490	0x8e88, 0x01ff1f3f, 0x00000000,
 491	0x8e88, 0x01ff1f3f, 0x00000000,
 492	0x8e84, 0x01ff1f3f, 0x00000000,
 493	0x8e84, 0x01ff1f3f, 0x00000000,
 494	0x8e84, 0x01ff1f3f, 0x00000000,
 495	0x9060, 0x0000007f, 0x00000020,
 496	0x9508, 0x00010000, 0x00010000,
 497	0xac14, 0x000003ff, 0x00000003,
 498	0xac14, 0x000003ff, 0x00000003,
 499	0xac14, 0x000003ff, 0x00000003,
 500	0xac10, 0xffffffff, 0x00000000,
 501	0xac10, 0xffffffff, 0x00000000,
 502	0xac10, 0xffffffff, 0x00000000,
 503	0xac0c, 0xffffffff, 0x00001032,
 504	0xac0c, 0xffffffff, 0x00001032,
 505	0xac0c, 0xffffffff, 0x00001032,
 506	0x88d4, 0x0000001f, 0x00000010,
 507	0x88d4, 0x0000001f, 0x00000010,
 508	0x88d4, 0x0000001f, 0x00000010,
 509	0x15c0, 0x000c0fc0, 0x000c0400
 510};
 511
 512static const u32 oland_golden_rlc_registers[] =
 513{
 514	0xc424, 0xffffffff, 0x00601005,
 515	0xc47c, 0xffffffff, 0x10104040,
 516	0xc488, 0xffffffff, 0x0100000a,
 517	0xc314, 0xffffffff, 0x00000800,
 518	0xc30c, 0xffffffff, 0x800000f4
 519};
 520
 521static const u32 oland_golden_registers[] =
 522{
 523	0x9a10, 0x00010000, 0x00018208,
 524	0x9830, 0xffffffff, 0x00000000,
 525	0x9834, 0xf00fffff, 0x00000400,
 526	0x9838, 0x0002021c, 0x00020200,
 527	0xc78, 0x00000080, 0x00000000,
 528	0xd030, 0x000300c0, 0x00800040,
 529	0xd830, 0x000300c0, 0x00800040,
 530	0x5bb0, 0x000000f0, 0x00000070,
 531	0x5bc0, 0x00200000, 0x50100000,
 532	0x7030, 0x31000311, 0x00000011,
 533	0x2ae4, 0x00073ffe, 0x000022a2,
 534	0x240c, 0x000007ff, 0x00000000,
 535	0x8a14, 0xf000001f, 0x00000007,
 536	0x8b24, 0xffffffff, 0x00ffffff,
 537	0x8b10, 0x0000ff0f, 0x00000000,
 538	0x28a4c, 0x07ffffff, 0x4e000000,
 539	0x28350, 0x3f3f3fff, 0x00000082,
 540	0x30, 0x000000ff, 0x0040,
 541	0x34, 0x00000040, 0x00004040,
 542	0x9100, 0x07ffffff, 0x03000000,
 543	0x9060, 0x0000007f, 0x00000020,
 544	0x9508, 0x00010000, 0x00010000,
 545	0xac14, 0x000003ff, 0x000000f3,
 546	0xac10, 0xffffffff, 0x00000000,
 547	0xac0c, 0xffffffff, 0x00003210,
 548	0x88d4, 0x0000001f, 0x00000010,
 549	0x15c0, 0x000c0fc0, 0x000c0400
 550};
 551
 552static const u32 hainan_golden_registers[] =
 553{
 554	0x9a10, 0x00010000, 0x00018208,
 555	0x9830, 0xffffffff, 0x00000000,
 556	0x9834, 0xf00fffff, 0x00000400,
 557	0x9838, 0x0002021c, 0x00020200,
 558	0xd0c0, 0xff000fff, 0x00000100,
 559	0xd030, 0x000300c0, 0x00800040,
 560	0xd8c0, 0xff000fff, 0x00000100,
 561	0xd830, 0x000300c0, 0x00800040,
 562	0x2ae4, 0x00073ffe, 0x000022a2,
 563	0x240c, 0x000007ff, 0x00000000,
 564	0x8a14, 0xf000001f, 0x00000007,
 565	0x8b24, 0xffffffff, 0x00ffffff,
 566	0x8b10, 0x0000ff0f, 0x00000000,
 567	0x28a4c, 0x07ffffff, 0x4e000000,
 568	0x28350, 0x3f3f3fff, 0x00000000,
 569	0x30, 0x000000ff, 0x0040,
 570	0x34, 0x00000040, 0x00004040,
 571	0x9100, 0x03e00000, 0x03600000,
 572	0x9060, 0x0000007f, 0x00000020,
 573	0x9508, 0x00010000, 0x00010000,
 574	0xac14, 0x000003ff, 0x000000f1,
 575	0xac10, 0xffffffff, 0x00000000,
 576	0xac0c, 0xffffffff, 0x00003210,
 577	0x88d4, 0x0000001f, 0x00000010,
 578	0x15c0, 0x000c0fc0, 0x000c0400
 579};
 580
 581static const u32 hainan_golden_registers2[] =
 582{
 583	0x98f8, 0xffffffff, 0x02010001
 584};
 585
 586static const u32 tahiti_mgcg_cgcg_init[] =
 587{
 588	0xc400, 0xffffffff, 0xfffffffc,
 589	0x802c, 0xffffffff, 0xe0000000,
 590	0x9a60, 0xffffffff, 0x00000100,
 591	0x92a4, 0xffffffff, 0x00000100,
 592	0xc164, 0xffffffff, 0x00000100,
 593	0x9774, 0xffffffff, 0x00000100,
 594	0x8984, 0xffffffff, 0x06000100,
 595	0x8a18, 0xffffffff, 0x00000100,
 596	0x92a0, 0xffffffff, 0x00000100,
 597	0xc380, 0xffffffff, 0x00000100,
 598	0x8b28, 0xffffffff, 0x00000100,
 599	0x9144, 0xffffffff, 0x00000100,
 600	0x8d88, 0xffffffff, 0x00000100,
 601	0x8d8c, 0xffffffff, 0x00000100,
 602	0x9030, 0xffffffff, 0x00000100,
 603	0x9034, 0xffffffff, 0x00000100,
 604	0x9038, 0xffffffff, 0x00000100,
 605	0x903c, 0xffffffff, 0x00000100,
 606	0xad80, 0xffffffff, 0x00000100,
 607	0xac54, 0xffffffff, 0x00000100,
 608	0x897c, 0xffffffff, 0x06000100,
 609	0x9868, 0xffffffff, 0x00000100,
 610	0x9510, 0xffffffff, 0x00000100,
 611	0xaf04, 0xffffffff, 0x00000100,
 612	0xae04, 0xffffffff, 0x00000100,
 613	0x949c, 0xffffffff, 0x00000100,
 614	0x802c, 0xffffffff, 0xe0000000,
 615	0x9160, 0xffffffff, 0x00010000,
 616	0x9164, 0xffffffff, 0x00030002,
 617	0x9168, 0xffffffff, 0x00040007,
 618	0x916c, 0xffffffff, 0x00060005,
 619	0x9170, 0xffffffff, 0x00090008,
 620	0x9174, 0xffffffff, 0x00020001,
 621	0x9178, 0xffffffff, 0x00040003,
 622	0x917c, 0xffffffff, 0x00000007,
 623	0x9180, 0xffffffff, 0x00060005,
 624	0x9184, 0xffffffff, 0x00090008,
 625	0x9188, 0xffffffff, 0x00030002,
 626	0x918c, 0xffffffff, 0x00050004,
 627	0x9190, 0xffffffff, 0x00000008,
 628	0x9194, 0xffffffff, 0x00070006,
 629	0x9198, 0xffffffff, 0x000a0009,
 630	0x919c, 0xffffffff, 0x00040003,
 631	0x91a0, 0xffffffff, 0x00060005,
 632	0x91a4, 0xffffffff, 0x00000009,
 633	0x91a8, 0xffffffff, 0x00080007,
 634	0x91ac, 0xffffffff, 0x000b000a,
 635	0x91b0, 0xffffffff, 0x00050004,
 636	0x91b4, 0xffffffff, 0x00070006,
 637	0x91b8, 0xffffffff, 0x0008000b,
 638	0x91bc, 0xffffffff, 0x000a0009,
 639	0x91c0, 0xffffffff, 0x000d000c,
 640	0x91c4, 0xffffffff, 0x00060005,
 641	0x91c8, 0xffffffff, 0x00080007,
 642	0x91cc, 0xffffffff, 0x0000000b,
 643	0x91d0, 0xffffffff, 0x000a0009,
 644	0x91d4, 0xffffffff, 0x000d000c,
 645	0x91d8, 0xffffffff, 0x00070006,
 646	0x91dc, 0xffffffff, 0x00090008,
 647	0x91e0, 0xffffffff, 0x0000000c,
 648	0x91e4, 0xffffffff, 0x000b000a,
 649	0x91e8, 0xffffffff, 0x000e000d,
 650	0x91ec, 0xffffffff, 0x00080007,
 651	0x91f0, 0xffffffff, 0x000a0009,
 652	0x91f4, 0xffffffff, 0x0000000d,
 653	0x91f8, 0xffffffff, 0x000c000b,
 654	0x91fc, 0xffffffff, 0x000f000e,
 655	0x9200, 0xffffffff, 0x00090008,
 656	0x9204, 0xffffffff, 0x000b000a,
 657	0x9208, 0xffffffff, 0x000c000f,
 658	0x920c, 0xffffffff, 0x000e000d,
 659	0x9210, 0xffffffff, 0x00110010,
 660	0x9214, 0xffffffff, 0x000a0009,
 661	0x9218, 0xffffffff, 0x000c000b,
 662	0x921c, 0xffffffff, 0x0000000f,
 663	0x9220, 0xffffffff, 0x000e000d,
 664	0x9224, 0xffffffff, 0x00110010,
 665	0x9228, 0xffffffff, 0x000b000a,
 666	0x922c, 0xffffffff, 0x000d000c,
 667	0x9230, 0xffffffff, 0x00000010,
 668	0x9234, 0xffffffff, 0x000f000e,
 669	0x9238, 0xffffffff, 0x00120011,
 670	0x923c, 0xffffffff, 0x000c000b,
 671	0x9240, 0xffffffff, 0x000e000d,
 672	0x9244, 0xffffffff, 0x00000011,
 673	0x9248, 0xffffffff, 0x0010000f,
 674	0x924c, 0xffffffff, 0x00130012,
 675	0x9250, 0xffffffff, 0x000d000c,
 676	0x9254, 0xffffffff, 0x000f000e,
 677	0x9258, 0xffffffff, 0x00100013,
 678	0x925c, 0xffffffff, 0x00120011,
 679	0x9260, 0xffffffff, 0x00150014,
 680	0x9264, 0xffffffff, 0x000e000d,
 681	0x9268, 0xffffffff, 0x0010000f,
 682	0x926c, 0xffffffff, 0x00000013,
 683	0x9270, 0xffffffff, 0x00120011,
 684	0x9274, 0xffffffff, 0x00150014,
 685	0x9278, 0xffffffff, 0x000f000e,
 686	0x927c, 0xffffffff, 0x00110010,
 687	0x9280, 0xffffffff, 0x00000014,
 688	0x9284, 0xffffffff, 0x00130012,
 689	0x9288, 0xffffffff, 0x00160015,
 690	0x928c, 0xffffffff, 0x0010000f,
 691	0x9290, 0xffffffff, 0x00120011,
 692	0x9294, 0xffffffff, 0x00000015,
 693	0x9298, 0xffffffff, 0x00140013,
 694	0x929c, 0xffffffff, 0x00170016,
 695	0x9150, 0xffffffff, 0x96940200,
 696	0x8708, 0xffffffff, 0x00900100,
 697	0xc478, 0xffffffff, 0x00000080,
 698	0xc404, 0xffffffff, 0x0020003f,
 699	0x30, 0xffffffff, 0x0000001c,
 700	0x34, 0x000f0000, 0x000f0000,
 701	0x160c, 0xffffffff, 0x00000100,
 702	0x1024, 0xffffffff, 0x00000100,
 703	0x102c, 0x00000101, 0x00000000,
 704	0x20a8, 0xffffffff, 0x00000104,
 705	0x264c, 0x000c0000, 0x000c0000,
 706	0x2648, 0x000c0000, 0x000c0000,
 707	0x55e4, 0xff000fff, 0x00000100,
 708	0x55e8, 0x00000001, 0x00000001,
 709	0x2f50, 0x00000001, 0x00000001,
 710	0x30cc, 0xc0000fff, 0x00000104,
 711	0xc1e4, 0x00000001, 0x00000001,
 712	0xd0c0, 0xfffffff0, 0x00000100,
 713	0xd8c0, 0xfffffff0, 0x00000100
 714};
 715
 716static const u32 pitcairn_mgcg_cgcg_init[] =
 717{
 718	0xc400, 0xffffffff, 0xfffffffc,
 719	0x802c, 0xffffffff, 0xe0000000,
 720	0x9a60, 0xffffffff, 0x00000100,
 721	0x92a4, 0xffffffff, 0x00000100,
 722	0xc164, 0xffffffff, 0x00000100,
 723	0x9774, 0xffffffff, 0x00000100,
 724	0x8984, 0xffffffff, 0x06000100,
 725	0x8a18, 0xffffffff, 0x00000100,
 726	0x92a0, 0xffffffff, 0x00000100,
 727	0xc380, 0xffffffff, 0x00000100,
 728	0x8b28, 0xffffffff, 0x00000100,
 729	0x9144, 0xffffffff, 0x00000100,
 730	0x8d88, 0xffffffff, 0x00000100,
 731	0x8d8c, 0xffffffff, 0x00000100,
 732	0x9030, 0xffffffff, 0x00000100,
 733	0x9034, 0xffffffff, 0x00000100,
 734	0x9038, 0xffffffff, 0x00000100,
 735	0x903c, 0xffffffff, 0x00000100,
 736	0xad80, 0xffffffff, 0x00000100,
 737	0xac54, 0xffffffff, 0x00000100,
 738	0x897c, 0xffffffff, 0x06000100,
 739	0x9868, 0xffffffff, 0x00000100,
 740	0x9510, 0xffffffff, 0x00000100,
 741	0xaf04, 0xffffffff, 0x00000100,
 742	0xae04, 0xffffffff, 0x00000100,
 743	0x949c, 0xffffffff, 0x00000100,
 744	0x802c, 0xffffffff, 0xe0000000,
 745	0x9160, 0xffffffff, 0x00010000,
 746	0x9164, 0xffffffff, 0x00030002,
 747	0x9168, 0xffffffff, 0x00040007,
 748	0x916c, 0xffffffff, 0x00060005,
 749	0x9170, 0xffffffff, 0x00090008,
 750	0x9174, 0xffffffff, 0x00020001,
 751	0x9178, 0xffffffff, 0x00040003,
 752	0x917c, 0xffffffff, 0x00000007,
 753	0x9180, 0xffffffff, 0x00060005,
 754	0x9184, 0xffffffff, 0x00090008,
 755	0x9188, 0xffffffff, 0x00030002,
 756	0x918c, 0xffffffff, 0x00050004,
 757	0x9190, 0xffffffff, 0x00000008,
 758	0x9194, 0xffffffff, 0x00070006,
 759	0x9198, 0xffffffff, 0x000a0009,
 760	0x919c, 0xffffffff, 0x00040003,
 761	0x91a0, 0xffffffff, 0x00060005,
 762	0x91a4, 0xffffffff, 0x00000009,
 763	0x91a8, 0xffffffff, 0x00080007,
 764	0x91ac, 0xffffffff, 0x000b000a,
 765	0x91b0, 0xffffffff, 0x00050004,
 766	0x91b4, 0xffffffff, 0x00070006,
 767	0x91b8, 0xffffffff, 0x0008000b,
 768	0x91bc, 0xffffffff, 0x000a0009,
 769	0x91c0, 0xffffffff, 0x000d000c,
 770	0x9200, 0xffffffff, 0x00090008,
 771	0x9204, 0xffffffff, 0x000b000a,
 772	0x9208, 0xffffffff, 0x000c000f,
 773	0x920c, 0xffffffff, 0x000e000d,
 774	0x9210, 0xffffffff, 0x00110010,
 775	0x9214, 0xffffffff, 0x000a0009,
 776	0x9218, 0xffffffff, 0x000c000b,
 777	0x921c, 0xffffffff, 0x0000000f,
 778	0x9220, 0xffffffff, 0x000e000d,
 779	0x9224, 0xffffffff, 0x00110010,
 780	0x9228, 0xffffffff, 0x000b000a,
 781	0x922c, 0xffffffff, 0x000d000c,
 782	0x9230, 0xffffffff, 0x00000010,
 783	0x9234, 0xffffffff, 0x000f000e,
 784	0x9238, 0xffffffff, 0x00120011,
 785	0x923c, 0xffffffff, 0x000c000b,
 786	0x9240, 0xffffffff, 0x000e000d,
 787	0x9244, 0xffffffff, 0x00000011,
 788	0x9248, 0xffffffff, 0x0010000f,
 789	0x924c, 0xffffffff, 0x00130012,
 790	0x9250, 0xffffffff, 0x000d000c,
 791	0x9254, 0xffffffff, 0x000f000e,
 792	0x9258, 0xffffffff, 0x00100013,
 793	0x925c, 0xffffffff, 0x00120011,
 794	0x9260, 0xffffffff, 0x00150014,
 795	0x9150, 0xffffffff, 0x96940200,
 796	0x8708, 0xffffffff, 0x00900100,
 797	0xc478, 0xffffffff, 0x00000080,
 798	0xc404, 0xffffffff, 0x0020003f,
 799	0x30, 0xffffffff, 0x0000001c,
 800	0x34, 0x000f0000, 0x000f0000,
 801	0x160c, 0xffffffff, 0x00000100,
 802	0x1024, 0xffffffff, 0x00000100,
 803	0x102c, 0x00000101, 0x00000000,
 804	0x20a8, 0xffffffff, 0x00000104,
 805	0x55e4, 0xff000fff, 0x00000100,
 806	0x55e8, 0x00000001, 0x00000001,
 807	0x2f50, 0x00000001, 0x00000001,
 808	0x30cc, 0xc0000fff, 0x00000104,
 809	0xc1e4, 0x00000001, 0x00000001,
 810	0xd0c0, 0xfffffff0, 0x00000100,
 811	0xd8c0, 0xfffffff0, 0x00000100
 812};
 813
 814static const u32 verde_mgcg_cgcg_init[] =
 815{
 816	0xc400, 0xffffffff, 0xfffffffc,
 817	0x802c, 0xffffffff, 0xe0000000,
 818	0x9a60, 0xffffffff, 0x00000100,
 819	0x92a4, 0xffffffff, 0x00000100,
 820	0xc164, 0xffffffff, 0x00000100,
 821	0x9774, 0xffffffff, 0x00000100,
 822	0x8984, 0xffffffff, 0x06000100,
 823	0x8a18, 0xffffffff, 0x00000100,
 824	0x92a0, 0xffffffff, 0x00000100,
 825	0xc380, 0xffffffff, 0x00000100,
 826	0x8b28, 0xffffffff, 0x00000100,
 827	0x9144, 0xffffffff, 0x00000100,
 828	0x8d88, 0xffffffff, 0x00000100,
 829	0x8d8c, 0xffffffff, 0x00000100,
 830	0x9030, 0xffffffff, 0x00000100,
 831	0x9034, 0xffffffff, 0x00000100,
 832	0x9038, 0xffffffff, 0x00000100,
 833	0x903c, 0xffffffff, 0x00000100,
 834	0xad80, 0xffffffff, 0x00000100,
 835	0xac54, 0xffffffff, 0x00000100,
 836	0x897c, 0xffffffff, 0x06000100,
 837	0x9868, 0xffffffff, 0x00000100,
 838	0x9510, 0xffffffff, 0x00000100,
 839	0xaf04, 0xffffffff, 0x00000100,
 840	0xae04, 0xffffffff, 0x00000100,
 841	0x949c, 0xffffffff, 0x00000100,
 842	0x802c, 0xffffffff, 0xe0000000,
 843	0x9160, 0xffffffff, 0x00010000,
 844	0x9164, 0xffffffff, 0x00030002,
 845	0x9168, 0xffffffff, 0x00040007,
 846	0x916c, 0xffffffff, 0x00060005,
 847	0x9170, 0xffffffff, 0x00090008,
 848	0x9174, 0xffffffff, 0x00020001,
 849	0x9178, 0xffffffff, 0x00040003,
 850	0x917c, 0xffffffff, 0x00000007,
 851	0x9180, 0xffffffff, 0x00060005,
 852	0x9184, 0xffffffff, 0x00090008,
 853	0x9188, 0xffffffff, 0x00030002,
 854	0x918c, 0xffffffff, 0x00050004,
 855	0x9190, 0xffffffff, 0x00000008,
 856	0x9194, 0xffffffff, 0x00070006,
 857	0x9198, 0xffffffff, 0x000a0009,
 858	0x919c, 0xffffffff, 0x00040003,
 859	0x91a0, 0xffffffff, 0x00060005,
 860	0x91a4, 0xffffffff, 0x00000009,
 861	0x91a8, 0xffffffff, 0x00080007,
 862	0x91ac, 0xffffffff, 0x000b000a,
 863	0x91b0, 0xffffffff, 0x00050004,
 864	0x91b4, 0xffffffff, 0x00070006,
 865	0x91b8, 0xffffffff, 0x0008000b,
 866	0x91bc, 0xffffffff, 0x000a0009,
 867	0x91c0, 0xffffffff, 0x000d000c,
 868	0x9200, 0xffffffff, 0x00090008,
 869	0x9204, 0xffffffff, 0x000b000a,
 870	0x9208, 0xffffffff, 0x000c000f,
 871	0x920c, 0xffffffff, 0x000e000d,
 872	0x9210, 0xffffffff, 0x00110010,
 873	0x9214, 0xffffffff, 0x000a0009,
 874	0x9218, 0xffffffff, 0x000c000b,
 875	0x921c, 0xffffffff, 0x0000000f,
 876	0x9220, 0xffffffff, 0x000e000d,
 877	0x9224, 0xffffffff, 0x00110010,
 878	0x9228, 0xffffffff, 0x000b000a,
 879	0x922c, 0xffffffff, 0x000d000c,
 880	0x9230, 0xffffffff, 0x00000010,
 881	0x9234, 0xffffffff, 0x000f000e,
 882	0x9238, 0xffffffff, 0x00120011,
 883	0x923c, 0xffffffff, 0x000c000b,
 884	0x9240, 0xffffffff, 0x000e000d,
 885	0x9244, 0xffffffff, 0x00000011,
 886	0x9248, 0xffffffff, 0x0010000f,
 887	0x924c, 0xffffffff, 0x00130012,
 888	0x9250, 0xffffffff, 0x000d000c,
 889	0x9254, 0xffffffff, 0x000f000e,
 890	0x9258, 0xffffffff, 0x00100013,
 891	0x925c, 0xffffffff, 0x00120011,
 892	0x9260, 0xffffffff, 0x00150014,
 893	0x9150, 0xffffffff, 0x96940200,
 894	0x8708, 0xffffffff, 0x00900100,
 895	0xc478, 0xffffffff, 0x00000080,
 896	0xc404, 0xffffffff, 0x0020003f,
 897	0x30, 0xffffffff, 0x0000001c,
 898	0x34, 0x000f0000, 0x000f0000,
 899	0x160c, 0xffffffff, 0x00000100,
 900	0x1024, 0xffffffff, 0x00000100,
 901	0x102c, 0x00000101, 0x00000000,
 902	0x20a8, 0xffffffff, 0x00000104,
 903	0x264c, 0x000c0000, 0x000c0000,
 904	0x2648, 0x000c0000, 0x000c0000,
 905	0x55e4, 0xff000fff, 0x00000100,
 906	0x55e8, 0x00000001, 0x00000001,
 907	0x2f50, 0x00000001, 0x00000001,
 908	0x30cc, 0xc0000fff, 0x00000104,
 909	0xc1e4, 0x00000001, 0x00000001,
 910	0xd0c0, 0xfffffff0, 0x00000100,
 911	0xd8c0, 0xfffffff0, 0x00000100
 912};
 913
 914static const u32 oland_mgcg_cgcg_init[] =
 915{
 916	0xc400, 0xffffffff, 0xfffffffc,
 917	0x802c, 0xffffffff, 0xe0000000,
 918	0x9a60, 0xffffffff, 0x00000100,
 919	0x92a4, 0xffffffff, 0x00000100,
 920	0xc164, 0xffffffff, 0x00000100,
 921	0x9774, 0xffffffff, 0x00000100,
 922	0x8984, 0xffffffff, 0x06000100,
 923	0x8a18, 0xffffffff, 0x00000100,
 924	0x92a0, 0xffffffff, 0x00000100,
 925	0xc380, 0xffffffff, 0x00000100,
 926	0x8b28, 0xffffffff, 0x00000100,
 927	0x9144, 0xffffffff, 0x00000100,
 928	0x8d88, 0xffffffff, 0x00000100,
 929	0x8d8c, 0xffffffff, 0x00000100,
 930	0x9030, 0xffffffff, 0x00000100,
 931	0x9034, 0xffffffff, 0x00000100,
 932	0x9038, 0xffffffff, 0x00000100,
 933	0x903c, 0xffffffff, 0x00000100,
 934	0xad80, 0xffffffff, 0x00000100,
 935	0xac54, 0xffffffff, 0x00000100,
 936	0x897c, 0xffffffff, 0x06000100,
 937	0x9868, 0xffffffff, 0x00000100,
 938	0x9510, 0xffffffff, 0x00000100,
 939	0xaf04, 0xffffffff, 0x00000100,
 940	0xae04, 0xffffffff, 0x00000100,
 941	0x949c, 0xffffffff, 0x00000100,
 942	0x802c, 0xffffffff, 0xe0000000,
 943	0x9160, 0xffffffff, 0x00010000,
 944	0x9164, 0xffffffff, 0x00030002,
 945	0x9168, 0xffffffff, 0x00040007,
 946	0x916c, 0xffffffff, 0x00060005,
 947	0x9170, 0xffffffff, 0x00090008,
 948	0x9174, 0xffffffff, 0x00020001,
 949	0x9178, 0xffffffff, 0x00040003,
 950	0x917c, 0xffffffff, 0x00000007,
 951	0x9180, 0xffffffff, 0x00060005,
 952	0x9184, 0xffffffff, 0x00090008,
 953	0x9188, 0xffffffff, 0x00030002,
 954	0x918c, 0xffffffff, 0x00050004,
 955	0x9190, 0xffffffff, 0x00000008,
 956	0x9194, 0xffffffff, 0x00070006,
 957	0x9198, 0xffffffff, 0x000a0009,
 958	0x919c, 0xffffffff, 0x00040003,
 959	0x91a0, 0xffffffff, 0x00060005,
 960	0x91a4, 0xffffffff, 0x00000009,
 961	0x91a8, 0xffffffff, 0x00080007,
 962	0x91ac, 0xffffffff, 0x000b000a,
 963	0x91b0, 0xffffffff, 0x00050004,
 964	0x91b4, 0xffffffff, 0x00070006,
 965	0x91b8, 0xffffffff, 0x0008000b,
 966	0x91bc, 0xffffffff, 0x000a0009,
 967	0x91c0, 0xffffffff, 0x000d000c,
 968	0x91c4, 0xffffffff, 0x00060005,
 969	0x91c8, 0xffffffff, 0x00080007,
 970	0x91cc, 0xffffffff, 0x0000000b,
 971	0x91d0, 0xffffffff, 0x000a0009,
 972	0x91d4, 0xffffffff, 0x000d000c,
 973	0x9150, 0xffffffff, 0x96940200,
 974	0x8708, 0xffffffff, 0x00900100,
 975	0xc478, 0xffffffff, 0x00000080,
 976	0xc404, 0xffffffff, 0x0020003f,
 977	0x30, 0xffffffff, 0x0000001c,
 978	0x34, 0x000f0000, 0x000f0000,
 979	0x160c, 0xffffffff, 0x00000100,
 980	0x1024, 0xffffffff, 0x00000100,
 981	0x102c, 0x00000101, 0x00000000,
 982	0x20a8, 0xffffffff, 0x00000104,
 983	0x264c, 0x000c0000, 0x000c0000,
 984	0x2648, 0x000c0000, 0x000c0000,
 985	0x55e4, 0xff000fff, 0x00000100,
 986	0x55e8, 0x00000001, 0x00000001,
 987	0x2f50, 0x00000001, 0x00000001,
 988	0x30cc, 0xc0000fff, 0x00000104,
 989	0xc1e4, 0x00000001, 0x00000001,
 990	0xd0c0, 0xfffffff0, 0x00000100,
 991	0xd8c0, 0xfffffff0, 0x00000100
 992};
 993
 994static const u32 hainan_mgcg_cgcg_init[] =
 995{
 996	0xc400, 0xffffffff, 0xfffffffc,
 997	0x802c, 0xffffffff, 0xe0000000,
 998	0x9a60, 0xffffffff, 0x00000100,
 999	0x92a4, 0xffffffff, 0x00000100,
1000	0xc164, 0xffffffff, 0x00000100,
1001	0x9774, 0xffffffff, 0x00000100,
1002	0x8984, 0xffffffff, 0x06000100,
1003	0x8a18, 0xffffffff, 0x00000100,
1004	0x92a0, 0xffffffff, 0x00000100,
1005	0xc380, 0xffffffff, 0x00000100,
1006	0x8b28, 0xffffffff, 0x00000100,
1007	0x9144, 0xffffffff, 0x00000100,
1008	0x8d88, 0xffffffff, 0x00000100,
1009	0x8d8c, 0xffffffff, 0x00000100,
1010	0x9030, 0xffffffff, 0x00000100,
1011	0x9034, 0xffffffff, 0x00000100,
1012	0x9038, 0xffffffff, 0x00000100,
1013	0x903c, 0xffffffff, 0x00000100,
1014	0xad80, 0xffffffff, 0x00000100,
1015	0xac54, 0xffffffff, 0x00000100,
1016	0x897c, 0xffffffff, 0x06000100,
1017	0x9868, 0xffffffff, 0x00000100,
1018	0x9510, 0xffffffff, 0x00000100,
1019	0xaf04, 0xffffffff, 0x00000100,
1020	0xae04, 0xffffffff, 0x00000100,
1021	0x949c, 0xffffffff, 0x00000100,
1022	0x802c, 0xffffffff, 0xe0000000,
1023	0x9160, 0xffffffff, 0x00010000,
1024	0x9164, 0xffffffff, 0x00030002,
1025	0x9168, 0xffffffff, 0x00040007,
1026	0x916c, 0xffffffff, 0x00060005,
1027	0x9170, 0xffffffff, 0x00090008,
1028	0x9174, 0xffffffff, 0x00020001,
1029	0x9178, 0xffffffff, 0x00040003,
1030	0x917c, 0xffffffff, 0x00000007,
1031	0x9180, 0xffffffff, 0x00060005,
1032	0x9184, 0xffffffff, 0x00090008,
1033	0x9188, 0xffffffff, 0x00030002,
1034	0x918c, 0xffffffff, 0x00050004,
1035	0x9190, 0xffffffff, 0x00000008,
1036	0x9194, 0xffffffff, 0x00070006,
1037	0x9198, 0xffffffff, 0x000a0009,
1038	0x919c, 0xffffffff, 0x00040003,
1039	0x91a0, 0xffffffff, 0x00060005,
1040	0x91a4, 0xffffffff, 0x00000009,
1041	0x91a8, 0xffffffff, 0x00080007,
1042	0x91ac, 0xffffffff, 0x000b000a,
1043	0x91b0, 0xffffffff, 0x00050004,
1044	0x91b4, 0xffffffff, 0x00070006,
1045	0x91b8, 0xffffffff, 0x0008000b,
1046	0x91bc, 0xffffffff, 0x000a0009,
1047	0x91c0, 0xffffffff, 0x000d000c,
1048	0x91c4, 0xffffffff, 0x00060005,
1049	0x91c8, 0xffffffff, 0x00080007,
1050	0x91cc, 0xffffffff, 0x0000000b,
1051	0x91d0, 0xffffffff, 0x000a0009,
1052	0x91d4, 0xffffffff, 0x000d000c,
1053	0x9150, 0xffffffff, 0x96940200,
1054	0x8708, 0xffffffff, 0x00900100,
1055	0xc478, 0xffffffff, 0x00000080,
1056	0xc404, 0xffffffff, 0x0020003f,
1057	0x30, 0xffffffff, 0x0000001c,
1058	0x34, 0x000f0000, 0x000f0000,
1059	0x160c, 0xffffffff, 0x00000100,
1060	0x1024, 0xffffffff, 0x00000100,
1061	0x20a8, 0xffffffff, 0x00000104,
1062	0x264c, 0x000c0000, 0x000c0000,
1063	0x2648, 0x000c0000, 0x000c0000,
1064	0x2f50, 0x00000001, 0x00000001,
1065	0x30cc, 0xc0000fff, 0x00000104,
1066	0xc1e4, 0x00000001, 0x00000001,
1067	0xd0c0, 0xfffffff0, 0x00000100,
1068	0xd8c0, 0xfffffff0, 0x00000100
1069};
1070
1071static u32 verde_pg_init[] =
1072{
1073	0x353c, 0xffffffff, 0x40000,
1074	0x3538, 0xffffffff, 0x200010ff,
1075	0x353c, 0xffffffff, 0x0,
1076	0x353c, 0xffffffff, 0x0,
1077	0x353c, 0xffffffff, 0x0,
1078	0x353c, 0xffffffff, 0x0,
1079	0x353c, 0xffffffff, 0x0,
1080	0x353c, 0xffffffff, 0x7007,
1081	0x3538, 0xffffffff, 0x300010ff,
1082	0x353c, 0xffffffff, 0x0,
1083	0x353c, 0xffffffff, 0x0,
1084	0x353c, 0xffffffff, 0x0,
1085	0x353c, 0xffffffff, 0x0,
1086	0x353c, 0xffffffff, 0x0,
1087	0x353c, 0xffffffff, 0x400000,
1088	0x3538, 0xffffffff, 0x100010ff,
1089	0x353c, 0xffffffff, 0x0,
1090	0x353c, 0xffffffff, 0x0,
1091	0x353c, 0xffffffff, 0x0,
1092	0x353c, 0xffffffff, 0x0,
1093	0x353c, 0xffffffff, 0x0,
1094	0x353c, 0xffffffff, 0x120200,
1095	0x3538, 0xffffffff, 0x500010ff,
1096	0x353c, 0xffffffff, 0x0,
1097	0x353c, 0xffffffff, 0x0,
1098	0x353c, 0xffffffff, 0x0,
1099	0x353c, 0xffffffff, 0x0,
1100	0x353c, 0xffffffff, 0x0,
1101	0x353c, 0xffffffff, 0x1e1e16,
1102	0x3538, 0xffffffff, 0x600010ff,
1103	0x353c, 0xffffffff, 0x0,
1104	0x353c, 0xffffffff, 0x0,
1105	0x353c, 0xffffffff, 0x0,
1106	0x353c, 0xffffffff, 0x0,
1107	0x353c, 0xffffffff, 0x0,
1108	0x353c, 0xffffffff, 0x171f1e,
1109	0x3538, 0xffffffff, 0x700010ff,
1110	0x353c, 0xffffffff, 0x0,
1111	0x353c, 0xffffffff, 0x0,
1112	0x353c, 0xffffffff, 0x0,
1113	0x353c, 0xffffffff, 0x0,
1114	0x353c, 0xffffffff, 0x0,
1115	0x353c, 0xffffffff, 0x0,
1116	0x3538, 0xffffffff, 0x9ff,
1117	0x3500, 0xffffffff, 0x0,
1118	0x3504, 0xffffffff, 0x10000800,
1119	0x3504, 0xffffffff, 0xf,
1120	0x3504, 0xffffffff, 0xf,
1121	0x3500, 0xffffffff, 0x4,
1122	0x3504, 0xffffffff, 0x1000051e,
1123	0x3504, 0xffffffff, 0xffff,
1124	0x3504, 0xffffffff, 0xffff,
1125	0x3500, 0xffffffff, 0x8,
1126	0x3504, 0xffffffff, 0x80500,
1127	0x3500, 0xffffffff, 0x12,
1128	0x3504, 0xffffffff, 0x9050c,
1129	0x3500, 0xffffffff, 0x1d,
1130	0x3504, 0xffffffff, 0xb052c,
1131	0x3500, 0xffffffff, 0x2a,
1132	0x3504, 0xffffffff, 0x1053e,
1133	0x3500, 0xffffffff, 0x2d,
1134	0x3504, 0xffffffff, 0x10546,
1135	0x3500, 0xffffffff, 0x30,
1136	0x3504, 0xffffffff, 0xa054e,
1137	0x3500, 0xffffffff, 0x3c,
1138	0x3504, 0xffffffff, 0x1055f,
1139	0x3500, 0xffffffff, 0x3f,
1140	0x3504, 0xffffffff, 0x10567,
1141	0x3500, 0xffffffff, 0x42,
1142	0x3504, 0xffffffff, 0x1056f,
1143	0x3500, 0xffffffff, 0x45,
1144	0x3504, 0xffffffff, 0x10572,
1145	0x3500, 0xffffffff, 0x48,
1146	0x3504, 0xffffffff, 0x20575,
1147	0x3500, 0xffffffff, 0x4c,
1148	0x3504, 0xffffffff, 0x190801,
1149	0x3500, 0xffffffff, 0x67,
1150	0x3504, 0xffffffff, 0x1082a,
1151	0x3500, 0xffffffff, 0x6a,
1152	0x3504, 0xffffffff, 0x1b082d,
1153	0x3500, 0xffffffff, 0x87,
1154	0x3504, 0xffffffff, 0x310851,
1155	0x3500, 0xffffffff, 0xba,
1156	0x3504, 0xffffffff, 0x891,
1157	0x3500, 0xffffffff, 0xbc,
1158	0x3504, 0xffffffff, 0x893,
1159	0x3500, 0xffffffff, 0xbe,
1160	0x3504, 0xffffffff, 0x20895,
1161	0x3500, 0xffffffff, 0xc2,
1162	0x3504, 0xffffffff, 0x20899,
1163	0x3500, 0xffffffff, 0xc6,
1164	0x3504, 0xffffffff, 0x2089d,
1165	0x3500, 0xffffffff, 0xca,
1166	0x3504, 0xffffffff, 0x8a1,
1167	0x3500, 0xffffffff, 0xcc,
1168	0x3504, 0xffffffff, 0x8a3,
1169	0x3500, 0xffffffff, 0xce,
1170	0x3504, 0xffffffff, 0x308a5,
1171	0x3500, 0xffffffff, 0xd3,
1172	0x3504, 0xffffffff, 0x6d08cd,
1173	0x3500, 0xffffffff, 0x142,
1174	0x3504, 0xffffffff, 0x2000095a,
1175	0x3504, 0xffffffff, 0x1,
1176	0x3500, 0xffffffff, 0x144,
1177	0x3504, 0xffffffff, 0x301f095b,
1178	0x3500, 0xffffffff, 0x165,
1179	0x3504, 0xffffffff, 0xc094d,
1180	0x3500, 0xffffffff, 0x173,
1181	0x3504, 0xffffffff, 0xf096d,
1182	0x3500, 0xffffffff, 0x184,
1183	0x3504, 0xffffffff, 0x15097f,
1184	0x3500, 0xffffffff, 0x19b,
1185	0x3504, 0xffffffff, 0xc0998,
1186	0x3500, 0xffffffff, 0x1a9,
1187	0x3504, 0xffffffff, 0x409a7,
1188	0x3500, 0xffffffff, 0x1af,
1189	0x3504, 0xffffffff, 0xcdc,
1190	0x3500, 0xffffffff, 0x1b1,
1191	0x3504, 0xffffffff, 0x800,
1192	0x3508, 0xffffffff, 0x6c9b2000,
1193	0x3510, 0xfc00, 0x2000,
1194	0x3544, 0xffffffff, 0xfc0,
1195	0x28d4, 0x00000100, 0x100
1196};
1197
1198static void si_init_golden_registers(struct radeon_device *rdev)
1199{
1200	switch (rdev->family) {
1201	case CHIP_TAHITI:
1202		radeon_program_register_sequence(rdev,
1203						 tahiti_golden_registers,
1204						 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1205		radeon_program_register_sequence(rdev,
1206						 tahiti_golden_rlc_registers,
1207						 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1208		radeon_program_register_sequence(rdev,
1209						 tahiti_mgcg_cgcg_init,
1210						 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1211		radeon_program_register_sequence(rdev,
1212						 tahiti_golden_registers2,
1213						 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1214		break;
1215	case CHIP_PITCAIRN:
1216		radeon_program_register_sequence(rdev,
1217						 pitcairn_golden_registers,
1218						 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1219		radeon_program_register_sequence(rdev,
1220						 pitcairn_golden_rlc_registers,
1221						 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1222		radeon_program_register_sequence(rdev,
1223						 pitcairn_mgcg_cgcg_init,
1224						 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1225		break;
1226	case CHIP_VERDE:
1227		radeon_program_register_sequence(rdev,
1228						 verde_golden_registers,
1229						 (const u32)ARRAY_SIZE(verde_golden_registers));
1230		radeon_program_register_sequence(rdev,
1231						 verde_golden_rlc_registers,
1232						 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1233		radeon_program_register_sequence(rdev,
1234						 verde_mgcg_cgcg_init,
1235						 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1236		radeon_program_register_sequence(rdev,
1237						 verde_pg_init,
1238						 (const u32)ARRAY_SIZE(verde_pg_init));
1239		break;
1240	case CHIP_OLAND:
1241		radeon_program_register_sequence(rdev,
1242						 oland_golden_registers,
1243						 (const u32)ARRAY_SIZE(oland_golden_registers));
1244		radeon_program_register_sequence(rdev,
1245						 oland_golden_rlc_registers,
1246						 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1247		radeon_program_register_sequence(rdev,
1248						 oland_mgcg_cgcg_init,
1249						 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1250		break;
1251	case CHIP_HAINAN:
1252		radeon_program_register_sequence(rdev,
1253						 hainan_golden_registers,
1254						 (const u32)ARRAY_SIZE(hainan_golden_registers));
1255		radeon_program_register_sequence(rdev,
1256						 hainan_golden_registers2,
1257						 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1258		radeon_program_register_sequence(rdev,
1259						 hainan_mgcg_cgcg_init,
1260						 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1261		break;
1262	default:
1263		break;
1264	}
1265}
1266
1267/**
1268 * si_get_allowed_info_register - fetch the register for the info ioctl
1269 *
1270 * @rdev: radeon_device pointer
1271 * @reg: register offset in bytes
1272 * @val: register value
1273 *
1274 * Returns 0 for success or -EINVAL for an invalid register
1275 *
1276 */
1277int si_get_allowed_info_register(struct radeon_device *rdev,
1278				 u32 reg, u32 *val)
1279{
1280	switch (reg) {
1281	case GRBM_STATUS:
1282	case GRBM_STATUS2:
1283	case GRBM_STATUS_SE0:
1284	case GRBM_STATUS_SE1:
1285	case SRBM_STATUS:
1286	case SRBM_STATUS2:
1287	case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
1288	case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
1289	case UVD_STATUS:
1290		*val = RREG32(reg);
1291		return 0;
1292	default:
1293		return -EINVAL;
1294	}
1295}
1296
1297#define PCIE_BUS_CLK                10000
1298#define TCLK                        (PCIE_BUS_CLK / 10)
1299
1300/**
1301 * si_get_xclk - get the xclk
1302 *
1303 * @rdev: radeon_device pointer
1304 *
1305 * Returns the reference clock used by the gfx engine
1306 * (SI).
1307 */
1308u32 si_get_xclk(struct radeon_device *rdev)
1309{
1310	u32 reference_clock = rdev->clock.spll.reference_freq;
1311	u32 tmp;
1312
1313	tmp = RREG32(CG_CLKPIN_CNTL_2);
1314	if (tmp & MUX_TCLK_TO_XCLK)
1315		return TCLK;
1316
1317	tmp = RREG32(CG_CLKPIN_CNTL);
1318	if (tmp & XTALIN_DIVIDE)
1319		return reference_clock / 4;
1320
1321	return reference_clock;
1322}
1323
1324/* get temperature in millidegrees */
1325int si_get_temp(struct radeon_device *rdev)
1326{
1327	u32 temp;
1328	int actual_temp = 0;
1329
1330	temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1331		CTF_TEMP_SHIFT;
1332
1333	if (temp & 0x200)
1334		actual_temp = 255;
1335	else
1336		actual_temp = temp & 0x1ff;
1337
1338	actual_temp = (actual_temp * 1000);
1339
1340	return actual_temp;
1341}
1342
1343#define TAHITI_IO_MC_REGS_SIZE 36
1344
1345static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1346	{0x0000006f, 0x03044000},
1347	{0x00000070, 0x0480c018},
1348	{0x00000071, 0x00000040},
1349	{0x00000072, 0x01000000},
1350	{0x00000074, 0x000000ff},
1351	{0x00000075, 0x00143400},
1352	{0x00000076, 0x08ec0800},
1353	{0x00000077, 0x040000cc},
1354	{0x00000079, 0x00000000},
1355	{0x0000007a, 0x21000409},
1356	{0x0000007c, 0x00000000},
1357	{0x0000007d, 0xe8000000},
1358	{0x0000007e, 0x044408a8},
1359	{0x0000007f, 0x00000003},
1360	{0x00000080, 0x00000000},
1361	{0x00000081, 0x01000000},
1362	{0x00000082, 0x02000000},
1363	{0x00000083, 0x00000000},
1364	{0x00000084, 0xe3f3e4f4},
1365	{0x00000085, 0x00052024},
1366	{0x00000087, 0x00000000},
1367	{0x00000088, 0x66036603},
1368	{0x00000089, 0x01000000},
1369	{0x0000008b, 0x1c0a0000},
1370	{0x0000008c, 0xff010000},
1371	{0x0000008e, 0xffffefff},
1372	{0x0000008f, 0xfff3efff},
1373	{0x00000090, 0xfff3efbf},
1374	{0x00000094, 0x00101101},
1375	{0x00000095, 0x00000fff},
1376	{0x00000096, 0x00116fff},
1377	{0x00000097, 0x60010000},
1378	{0x00000098, 0x10010000},
1379	{0x00000099, 0x00006000},
1380	{0x0000009a, 0x00001000},
1381	{0x0000009f, 0x00a77400}
1382};
1383
1384static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1385	{0x0000006f, 0x03044000},
1386	{0x00000070, 0x0480c018},
1387	{0x00000071, 0x00000040},
1388	{0x00000072, 0x01000000},
1389	{0x00000074, 0x000000ff},
1390	{0x00000075, 0x00143400},
1391	{0x00000076, 0x08ec0800},
1392	{0x00000077, 0x040000cc},
1393	{0x00000079, 0x00000000},
1394	{0x0000007a, 0x21000409},
1395	{0x0000007c, 0x00000000},
1396	{0x0000007d, 0xe8000000},
1397	{0x0000007e, 0x044408a8},
1398	{0x0000007f, 0x00000003},
1399	{0x00000080, 0x00000000},
1400	{0x00000081, 0x01000000},
1401	{0x00000082, 0x02000000},
1402	{0x00000083, 0x00000000},
1403	{0x00000084, 0xe3f3e4f4},
1404	{0x00000085, 0x00052024},
1405	{0x00000087, 0x00000000},
1406	{0x00000088, 0x66036603},
1407	{0x00000089, 0x01000000},
1408	{0x0000008b, 0x1c0a0000},
1409	{0x0000008c, 0xff010000},
1410	{0x0000008e, 0xffffefff},
1411	{0x0000008f, 0xfff3efff},
1412	{0x00000090, 0xfff3efbf},
1413	{0x00000094, 0x00101101},
1414	{0x00000095, 0x00000fff},
1415	{0x00000096, 0x00116fff},
1416	{0x00000097, 0x60010000},
1417	{0x00000098, 0x10010000},
1418	{0x00000099, 0x00006000},
1419	{0x0000009a, 0x00001000},
1420	{0x0000009f, 0x00a47400}
1421};
1422
1423static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1424	{0x0000006f, 0x03044000},
1425	{0x00000070, 0x0480c018},
1426	{0x00000071, 0x00000040},
1427	{0x00000072, 0x01000000},
1428	{0x00000074, 0x000000ff},
1429	{0x00000075, 0x00143400},
1430	{0x00000076, 0x08ec0800},
1431	{0x00000077, 0x040000cc},
1432	{0x00000079, 0x00000000},
1433	{0x0000007a, 0x21000409},
1434	{0x0000007c, 0x00000000},
1435	{0x0000007d, 0xe8000000},
1436	{0x0000007e, 0x044408a8},
1437	{0x0000007f, 0x00000003},
1438	{0x00000080, 0x00000000},
1439	{0x00000081, 0x01000000},
1440	{0x00000082, 0x02000000},
1441	{0x00000083, 0x00000000},
1442	{0x00000084, 0xe3f3e4f4},
1443	{0x00000085, 0x00052024},
1444	{0x00000087, 0x00000000},
1445	{0x00000088, 0x66036603},
1446	{0x00000089, 0x01000000},
1447	{0x0000008b, 0x1c0a0000},
1448	{0x0000008c, 0xff010000},
1449	{0x0000008e, 0xffffefff},
1450	{0x0000008f, 0xfff3efff},
1451	{0x00000090, 0xfff3efbf},
1452	{0x00000094, 0x00101101},
1453	{0x00000095, 0x00000fff},
1454	{0x00000096, 0x00116fff},
1455	{0x00000097, 0x60010000},
1456	{0x00000098, 0x10010000},
1457	{0x00000099, 0x00006000},
1458	{0x0000009a, 0x00001000},
1459	{0x0000009f, 0x00a37400}
1460};
1461
1462static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1463	{0x0000006f, 0x03044000},
1464	{0x00000070, 0x0480c018},
1465	{0x00000071, 0x00000040},
1466	{0x00000072, 0x01000000},
1467	{0x00000074, 0x000000ff},
1468	{0x00000075, 0x00143400},
1469	{0x00000076, 0x08ec0800},
1470	{0x00000077, 0x040000cc},
1471	{0x00000079, 0x00000000},
1472	{0x0000007a, 0x21000409},
1473	{0x0000007c, 0x00000000},
1474	{0x0000007d, 0xe8000000},
1475	{0x0000007e, 0x044408a8},
1476	{0x0000007f, 0x00000003},
1477	{0x00000080, 0x00000000},
1478	{0x00000081, 0x01000000},
1479	{0x00000082, 0x02000000},
1480	{0x00000083, 0x00000000},
1481	{0x00000084, 0xe3f3e4f4},
1482	{0x00000085, 0x00052024},
1483	{0x00000087, 0x00000000},
1484	{0x00000088, 0x66036603},
1485	{0x00000089, 0x01000000},
1486	{0x0000008b, 0x1c0a0000},
1487	{0x0000008c, 0xff010000},
1488	{0x0000008e, 0xffffefff},
1489	{0x0000008f, 0xfff3efff},
1490	{0x00000090, 0xfff3efbf},
1491	{0x00000094, 0x00101101},
1492	{0x00000095, 0x00000fff},
1493	{0x00000096, 0x00116fff},
1494	{0x00000097, 0x60010000},
1495	{0x00000098, 0x10010000},
1496	{0x00000099, 0x00006000},
1497	{0x0000009a, 0x00001000},
1498	{0x0000009f, 0x00a17730}
1499};
1500
1501static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1502	{0x0000006f, 0x03044000},
1503	{0x00000070, 0x0480c018},
1504	{0x00000071, 0x00000040},
1505	{0x00000072, 0x01000000},
1506	{0x00000074, 0x000000ff},
1507	{0x00000075, 0x00143400},
1508	{0x00000076, 0x08ec0800},
1509	{0x00000077, 0x040000cc},
1510	{0x00000079, 0x00000000},
1511	{0x0000007a, 0x21000409},
1512	{0x0000007c, 0x00000000},
1513	{0x0000007d, 0xe8000000},
1514	{0x0000007e, 0x044408a8},
1515	{0x0000007f, 0x00000003},
1516	{0x00000080, 0x00000000},
1517	{0x00000081, 0x01000000},
1518	{0x00000082, 0x02000000},
1519	{0x00000083, 0x00000000},
1520	{0x00000084, 0xe3f3e4f4},
1521	{0x00000085, 0x00052024},
1522	{0x00000087, 0x00000000},
1523	{0x00000088, 0x66036603},
1524	{0x00000089, 0x01000000},
1525	{0x0000008b, 0x1c0a0000},
1526	{0x0000008c, 0xff010000},
1527	{0x0000008e, 0xffffefff},
1528	{0x0000008f, 0xfff3efff},
1529	{0x00000090, 0xfff3efbf},
1530	{0x00000094, 0x00101101},
1531	{0x00000095, 0x00000fff},
1532	{0x00000096, 0x00116fff},
1533	{0x00000097, 0x60010000},
1534	{0x00000098, 0x10010000},
1535	{0x00000099, 0x00006000},
1536	{0x0000009a, 0x00001000},
1537	{0x0000009f, 0x00a07730}
1538};
1539
1540/* ucode loading */
1541int si_mc_load_microcode(struct radeon_device *rdev)
1542{
1543	const __be32 *fw_data = NULL;
1544	const __le32 *new_fw_data = NULL;
1545	u32 running, blackout = 0;
1546	u32 *io_mc_regs = NULL;
1547	const __le32 *new_io_mc_regs = NULL;
1548	int i, regs_size, ucode_size;
1549
1550	if (!rdev->mc_fw)
1551		return -EINVAL;
1552
1553	if (rdev->new_fw) {
1554		const struct mc_firmware_header_v1_0 *hdr =
1555			(const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1556
1557		radeon_ucode_print_mc_hdr(&hdr->header);
1558		regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1559		new_io_mc_regs = (const __le32 *)
1560			(rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1561		ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1562		new_fw_data = (const __le32 *)
1563			(rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1564	} else {
1565		ucode_size = rdev->mc_fw->size / 4;
1566
1567		switch (rdev->family) {
1568		case CHIP_TAHITI:
1569			io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1570			regs_size = TAHITI_IO_MC_REGS_SIZE;
1571			break;
1572		case CHIP_PITCAIRN:
1573			io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1574			regs_size = TAHITI_IO_MC_REGS_SIZE;
1575			break;
1576		case CHIP_VERDE:
1577		default:
1578			io_mc_regs = (u32 *)&verde_io_mc_regs;
1579			regs_size = TAHITI_IO_MC_REGS_SIZE;
1580			break;
1581		case CHIP_OLAND:
1582			io_mc_regs = (u32 *)&oland_io_mc_regs;
1583			regs_size = TAHITI_IO_MC_REGS_SIZE;
1584			break;
1585		case CHIP_HAINAN:
1586			io_mc_regs = (u32 *)&hainan_io_mc_regs;
1587			regs_size = TAHITI_IO_MC_REGS_SIZE;
1588			break;
1589		}
1590		fw_data = (const __be32 *)rdev->mc_fw->data;
1591	}
1592
1593	running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1594
1595	if (running == 0) {
1596		if (running) {
1597			blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1598			WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1599		}
1600
1601		/* reset the engine and set to writable */
1602		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1603		WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1604
1605		/* load mc io regs */
1606		for (i = 0; i < regs_size; i++) {
1607			if (rdev->new_fw) {
1608				WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1609				WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1610			} else {
1611				WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1612				WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1613			}
1614		}
1615		/* load the MC ucode */
1616		for (i = 0; i < ucode_size; i++) {
1617			if (rdev->new_fw)
1618				WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1619			else
1620				WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1621		}
1622
1623		/* put the engine back into the active state */
1624		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1625		WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1626		WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1627
1628		/* wait for training to complete */
1629		for (i = 0; i < rdev->usec_timeout; i++) {
1630			if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1631				break;
1632			udelay(1);
1633		}
1634		for (i = 0; i < rdev->usec_timeout; i++) {
1635			if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1636				break;
1637			udelay(1);
1638		}
1639
1640		if (running)
1641			WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1642	}
1643
1644	return 0;
1645}
1646
1647static int si_init_microcode(struct radeon_device *rdev)
1648{
1649	const char *chip_name;
1650	const char *new_chip_name;
1651	size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1652	size_t smc_req_size, mc2_req_size;
1653	char fw_name[30];
1654	int err;
1655	int new_fw = 0;
 
 
 
1656
1657	DRM_DEBUG("\n");
1658
1659	switch (rdev->family) {
1660	case CHIP_TAHITI:
1661		chip_name = "TAHITI";
1662		new_chip_name = "tahiti";
1663		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1664		me_req_size = SI_PM4_UCODE_SIZE * 4;
1665		ce_req_size = SI_CE_UCODE_SIZE * 4;
1666		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1667		mc_req_size = SI_MC_UCODE_SIZE * 4;
1668		mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1669		smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1670		break;
1671	case CHIP_PITCAIRN:
1672		chip_name = "PITCAIRN";
 
 
 
 
1673		new_chip_name = "pitcairn";
1674		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1675		me_req_size = SI_PM4_UCODE_SIZE * 4;
1676		ce_req_size = SI_CE_UCODE_SIZE * 4;
1677		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1678		mc_req_size = SI_MC_UCODE_SIZE * 4;
1679		mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1680		smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1681		break;
1682	case CHIP_VERDE:
1683		chip_name = "VERDE";
 
 
 
 
 
 
 
 
 
 
1684		new_chip_name = "verde";
1685		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1686		me_req_size = SI_PM4_UCODE_SIZE * 4;
1687		ce_req_size = SI_CE_UCODE_SIZE * 4;
1688		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1689		mc_req_size = SI_MC_UCODE_SIZE * 4;
1690		mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1691		smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1692		break;
1693	case CHIP_OLAND:
1694		chip_name = "OLAND";
 
 
 
 
 
 
 
 
1695		new_chip_name = "oland";
1696		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1697		me_req_size = SI_PM4_UCODE_SIZE * 4;
1698		ce_req_size = SI_CE_UCODE_SIZE * 4;
1699		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1700		mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1701		smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1702		break;
1703	case CHIP_HAINAN:
1704		chip_name = "HAINAN";
 
 
 
 
 
 
 
 
 
 
 
1705		new_chip_name = "hainan";
1706		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1707		me_req_size = SI_PM4_UCODE_SIZE * 4;
1708		ce_req_size = SI_CE_UCODE_SIZE * 4;
1709		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1710		mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1711		smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1712		break;
1713	default: BUG();
1714	}
1715
 
 
 
 
1716	DRM_INFO("Loading %s Microcode\n", new_chip_name);
1717
1718	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1719	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1720	if (err) {
1721		snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1722		err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1723		if (err)
1724			goto out;
1725		if (rdev->pfp_fw->size != pfp_req_size) {
1726			printk(KERN_ERR
1727			       "si_cp: Bogus length %zu in firmware \"%s\"\n",
1728			       rdev->pfp_fw->size, fw_name);
1729			err = -EINVAL;
1730			goto out;
1731		}
1732	} else {
1733		err = radeon_ucode_validate(rdev->pfp_fw);
1734		if (err) {
1735			printk(KERN_ERR
1736			       "si_cp: validation failed for firmware \"%s\"\n",
1737			       fw_name);
1738			goto out;
1739		} else {
1740			new_fw++;
1741		}
1742	}
1743
1744	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
1745	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1746	if (err) {
1747		snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1748		err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1749		if (err)
1750			goto out;
1751		if (rdev->me_fw->size != me_req_size) {
1752			printk(KERN_ERR
1753			       "si_cp: Bogus length %zu in firmware \"%s\"\n",
1754			       rdev->me_fw->size, fw_name);
1755			err = -EINVAL;
1756		}
1757	} else {
1758		err = radeon_ucode_validate(rdev->me_fw);
1759		if (err) {
1760			printk(KERN_ERR
1761			       "si_cp: validation failed for firmware \"%s\"\n",
1762			       fw_name);
1763			goto out;
1764		} else {
1765			new_fw++;
1766		}
1767	}
1768
1769	snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
1770	err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1771	if (err) {
1772		snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1773		err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1774		if (err)
1775			goto out;
1776		if (rdev->ce_fw->size != ce_req_size) {
1777			printk(KERN_ERR
1778			       "si_cp: Bogus length %zu in firmware \"%s\"\n",
1779			       rdev->ce_fw->size, fw_name);
1780			err = -EINVAL;
1781		}
1782	} else {
1783		err = radeon_ucode_validate(rdev->ce_fw);
1784		if (err) {
1785			printk(KERN_ERR
1786			       "si_cp: validation failed for firmware \"%s\"\n",
1787			       fw_name);
1788			goto out;
1789		} else {
1790			new_fw++;
1791		}
1792	}
1793
1794	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
1795	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1796	if (err) {
1797		snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1798		err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1799		if (err)
1800			goto out;
1801		if (rdev->rlc_fw->size != rlc_req_size) {
1802			printk(KERN_ERR
1803			       "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1804			       rdev->rlc_fw->size, fw_name);
1805			err = -EINVAL;
1806		}
1807	} else {
1808		err = radeon_ucode_validate(rdev->rlc_fw);
1809		if (err) {
1810			printk(KERN_ERR
1811			       "si_cp: validation failed for firmware \"%s\"\n",
1812			       fw_name);
1813			goto out;
1814		} else {
1815			new_fw++;
1816		}
1817	}
1818
1819	snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
 
 
 
1820	err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1821	if (err) {
1822		snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1823		err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1824		if (err) {
1825			snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1826			err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1827			if (err)
1828				goto out;
1829		}
1830		if ((rdev->mc_fw->size != mc_req_size) &&
1831		    (rdev->mc_fw->size != mc2_req_size)) {
1832			printk(KERN_ERR
1833			       "si_mc: Bogus length %zu in firmware \"%s\"\n",
1834			       rdev->mc_fw->size, fw_name);
1835			err = -EINVAL;
1836		}
1837		DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1838	} else {
1839		err = radeon_ucode_validate(rdev->mc_fw);
1840		if (err) {
1841			printk(KERN_ERR
1842			       "si_cp: validation failed for firmware \"%s\"\n",
1843			       fw_name);
1844			goto out;
1845		} else {
1846			new_fw++;
1847		}
1848	}
1849
1850	snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
 
 
 
 
 
1851	err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1852	if (err) {
1853		snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1854		err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1855		if (err) {
1856			printk(KERN_ERR
1857			       "smc: error loading firmware \"%s\"\n",
1858			       fw_name);
1859			release_firmware(rdev->smc_fw);
1860			rdev->smc_fw = NULL;
1861			err = 0;
1862		} else if (rdev->smc_fw->size != smc_req_size) {
1863			printk(KERN_ERR
1864			       "si_smc: Bogus length %zu in firmware \"%s\"\n",
1865			       rdev->smc_fw->size, fw_name);
1866			err = -EINVAL;
1867		}
1868	} else {
1869		err = radeon_ucode_validate(rdev->smc_fw);
1870		if (err) {
1871			printk(KERN_ERR
1872			       "si_cp: validation failed for firmware \"%s\"\n",
1873			       fw_name);
1874			goto out;
1875		} else {
1876			new_fw++;
1877		}
1878	}
1879
1880	if (new_fw == 0) {
1881		rdev->new_fw = false;
1882	} else if (new_fw < 6) {
1883		printk(KERN_ERR "si_fw: mixing new and old firmware!\n");
1884		err = -EINVAL;
1885	} else {
1886		rdev->new_fw = true;
1887	}
1888out:
1889	if (err) {
1890		if (err != -EINVAL)
1891			printk(KERN_ERR
1892			       "si_cp: Failed to load firmware \"%s\"\n",
1893			       fw_name);
1894		release_firmware(rdev->pfp_fw);
1895		rdev->pfp_fw = NULL;
1896		release_firmware(rdev->me_fw);
1897		rdev->me_fw = NULL;
1898		release_firmware(rdev->ce_fw);
1899		rdev->ce_fw = NULL;
1900		release_firmware(rdev->rlc_fw);
1901		rdev->rlc_fw = NULL;
1902		release_firmware(rdev->mc_fw);
1903		rdev->mc_fw = NULL;
1904		release_firmware(rdev->smc_fw);
1905		rdev->smc_fw = NULL;
1906	}
1907	return err;
1908}
1909
1910/* watermark setup */
1911static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1912				   struct radeon_crtc *radeon_crtc,
1913				   struct drm_display_mode *mode,
1914				   struct drm_display_mode *other_mode)
1915{
1916	u32 tmp, buffer_alloc, i;
1917	u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1918	/*
1919	 * Line Buffer Setup
1920	 * There are 3 line buffers, each one shared by 2 display controllers.
1921	 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1922	 * the display controllers.  The paritioning is done via one of four
1923	 * preset allocations specified in bits 21:20:
1924	 *  0 - half lb
1925	 *  2 - whole lb, other crtc must be disabled
1926	 */
1927	/* this can get tricky if we have two large displays on a paired group
1928	 * of crtcs.  Ideally for multiple large displays we'd assign them to
1929	 * non-linked crtcs for maximum line buffer allocation.
1930	 */
1931	if (radeon_crtc->base.enabled && mode) {
1932		if (other_mode) {
1933			tmp = 0; /* 1/2 */
1934			buffer_alloc = 1;
1935		} else {
1936			tmp = 2; /* whole */
1937			buffer_alloc = 2;
1938		}
1939	} else {
1940		tmp = 0;
1941		buffer_alloc = 0;
1942	}
1943
1944	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1945	       DC_LB_MEMORY_CONFIG(tmp));
1946
1947	WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1948	       DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1949	for (i = 0; i < rdev->usec_timeout; i++) {
1950		if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1951		    DMIF_BUFFERS_ALLOCATED_COMPLETED)
1952			break;
1953		udelay(1);
1954	}
1955
1956	if (radeon_crtc->base.enabled && mode) {
1957		switch (tmp) {
1958		case 0:
1959		default:
1960			return 4096 * 2;
1961		case 2:
1962			return 8192 * 2;
1963		}
1964	}
1965
1966	/* controller not enabled, so no lb used */
1967	return 0;
1968}
1969
1970static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1971{
1972	u32 tmp = RREG32(MC_SHARED_CHMAP);
1973
1974	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1975	case 0:
1976	default:
1977		return 1;
1978	case 1:
1979		return 2;
1980	case 2:
1981		return 4;
1982	case 3:
1983		return 8;
1984	case 4:
1985		return 3;
1986	case 5:
1987		return 6;
1988	case 6:
1989		return 10;
1990	case 7:
1991		return 12;
1992	case 8:
1993		return 16;
1994	}
1995}
1996
1997struct dce6_wm_params {
1998	u32 dram_channels; /* number of dram channels */
1999	u32 yclk;          /* bandwidth per dram data pin in kHz */
2000	u32 sclk;          /* engine clock in kHz */
2001	u32 disp_clk;      /* display clock in kHz */
2002	u32 src_width;     /* viewport width */
2003	u32 active_time;   /* active display time in ns */
2004	u32 blank_time;    /* blank time in ns */
2005	bool interlaced;    /* mode is interlaced */
2006	fixed20_12 vsc;    /* vertical scale ratio */
2007	u32 num_heads;     /* number of active crtcs */
2008	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2009	u32 lb_size;       /* line buffer allocated to pipe */
2010	u32 vtaps;         /* vertical scaler taps */
2011};
2012
2013static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
2014{
2015	/* Calculate raw DRAM Bandwidth */
2016	fixed20_12 dram_efficiency; /* 0.7 */
2017	fixed20_12 yclk, dram_channels, bandwidth;
2018	fixed20_12 a;
2019
2020	a.full = dfixed_const(1000);
2021	yclk.full = dfixed_const(wm->yclk);
2022	yclk.full = dfixed_div(yclk, a);
2023	dram_channels.full = dfixed_const(wm->dram_channels * 4);
2024	a.full = dfixed_const(10);
2025	dram_efficiency.full = dfixed_const(7);
2026	dram_efficiency.full = dfixed_div(dram_efficiency, a);
2027	bandwidth.full = dfixed_mul(dram_channels, yclk);
2028	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2029
2030	return dfixed_trunc(bandwidth);
2031}
2032
2033static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2034{
2035	/* Calculate DRAM Bandwidth and the part allocated to display. */
2036	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2037	fixed20_12 yclk, dram_channels, bandwidth;
2038	fixed20_12 a;
2039
2040	a.full = dfixed_const(1000);
2041	yclk.full = dfixed_const(wm->yclk);
2042	yclk.full = dfixed_div(yclk, a);
2043	dram_channels.full = dfixed_const(wm->dram_channels * 4);
2044	a.full = dfixed_const(10);
2045	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2046	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2047	bandwidth.full = dfixed_mul(dram_channels, yclk);
2048	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2049
2050	return dfixed_trunc(bandwidth);
2051}
2052
2053static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2054{
2055	/* Calculate the display Data return Bandwidth */
2056	fixed20_12 return_efficiency; /* 0.8 */
2057	fixed20_12 sclk, bandwidth;
2058	fixed20_12 a;
2059
2060	a.full = dfixed_const(1000);
2061	sclk.full = dfixed_const(wm->sclk);
2062	sclk.full = dfixed_div(sclk, a);
2063	a.full = dfixed_const(10);
2064	return_efficiency.full = dfixed_const(8);
2065	return_efficiency.full = dfixed_div(return_efficiency, a);
2066	a.full = dfixed_const(32);
2067	bandwidth.full = dfixed_mul(a, sclk);
2068	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2069
2070	return dfixed_trunc(bandwidth);
2071}
2072
2073static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2074{
2075	return 32;
2076}
2077
2078static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2079{
2080	/* Calculate the DMIF Request Bandwidth */
2081	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2082	fixed20_12 disp_clk, sclk, bandwidth;
2083	fixed20_12 a, b1, b2;
2084	u32 min_bandwidth;
2085
2086	a.full = dfixed_const(1000);
2087	disp_clk.full = dfixed_const(wm->disp_clk);
2088	disp_clk.full = dfixed_div(disp_clk, a);
2089	a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2090	b1.full = dfixed_mul(a, disp_clk);
2091
2092	a.full = dfixed_const(1000);
2093	sclk.full = dfixed_const(wm->sclk);
2094	sclk.full = dfixed_div(sclk, a);
2095	a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2096	b2.full = dfixed_mul(a, sclk);
2097
2098	a.full = dfixed_const(10);
2099	disp_clk_request_efficiency.full = dfixed_const(8);
2100	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2101
2102	min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2103
2104	a.full = dfixed_const(min_bandwidth);
2105	bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2106
2107	return dfixed_trunc(bandwidth);
2108}
2109
2110static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2111{
2112	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2113	u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2114	u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2115	u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2116
2117	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2118}
2119
2120static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2121{
2122	/* Calculate the display mode Average Bandwidth
2123	 * DisplayMode should contain the source and destination dimensions,
2124	 * timing, etc.
2125	 */
2126	fixed20_12 bpp;
2127	fixed20_12 line_time;
2128	fixed20_12 src_width;
2129	fixed20_12 bandwidth;
2130	fixed20_12 a;
2131
2132	a.full = dfixed_const(1000);
2133	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2134	line_time.full = dfixed_div(line_time, a);
2135	bpp.full = dfixed_const(wm->bytes_per_pixel);
2136	src_width.full = dfixed_const(wm->src_width);
2137	bandwidth.full = dfixed_mul(src_width, bpp);
2138	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2139	bandwidth.full = dfixed_div(bandwidth, line_time);
2140
2141	return dfixed_trunc(bandwidth);
2142}
2143
2144static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2145{
2146	/* First calcualte the latency in ns */
2147	u32 mc_latency = 2000; /* 2000 ns. */
2148	u32 available_bandwidth = dce6_available_bandwidth(wm);
2149	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2150	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2151	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2152	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2153		(wm->num_heads * cursor_line_pair_return_time);
2154	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2155	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2156	u32 tmp, dmif_size = 12288;
2157	fixed20_12 a, b, c;
2158
2159	if (wm->num_heads == 0)
2160		return 0;
2161
2162	a.full = dfixed_const(2);
2163	b.full = dfixed_const(1);
2164	if ((wm->vsc.full > a.full) ||
2165	    ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2166	    (wm->vtaps >= 5) ||
2167	    ((wm->vsc.full >= a.full) && wm->interlaced))
2168		max_src_lines_per_dst_line = 4;
2169	else
2170		max_src_lines_per_dst_line = 2;
2171
2172	a.full = dfixed_const(available_bandwidth);
2173	b.full = dfixed_const(wm->num_heads);
2174	a.full = dfixed_div(a, b);
 
 
2175
2176	b.full = dfixed_const(mc_latency + 512);
2177	c.full = dfixed_const(wm->disp_clk);
2178	b.full = dfixed_div(b, c);
2179
2180	c.full = dfixed_const(dmif_size);
2181	b.full = dfixed_div(c, b);
2182
2183	tmp = min(dfixed_trunc(a), dfixed_trunc(b));
2184
2185	b.full = dfixed_const(1000);
2186	c.full = dfixed_const(wm->disp_clk);
2187	b.full = dfixed_div(c, b);
2188	c.full = dfixed_const(wm->bytes_per_pixel);
2189	b.full = dfixed_mul(b, c);
2190
2191	lb_fill_bw = min(tmp, dfixed_trunc(b));
2192
2193	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2194	b.full = dfixed_const(1000);
2195	c.full = dfixed_const(lb_fill_bw);
2196	b.full = dfixed_div(c, b);
2197	a.full = dfixed_div(a, b);
2198	line_fill_time = dfixed_trunc(a);
2199
2200	if (line_fill_time < wm->active_time)
2201		return latency;
2202	else
2203		return latency + (line_fill_time - wm->active_time);
2204
2205}
2206
2207static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2208{
2209	if (dce6_average_bandwidth(wm) <=
2210	    (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2211		return true;
2212	else
2213		return false;
2214};
2215
2216static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2217{
2218	if (dce6_average_bandwidth(wm) <=
2219	    (dce6_available_bandwidth(wm) / wm->num_heads))
2220		return true;
2221	else
2222		return false;
2223};
2224
2225static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2226{
2227	u32 lb_partitions = wm->lb_size / wm->src_width;
2228	u32 line_time = wm->active_time + wm->blank_time;
2229	u32 latency_tolerant_lines;
2230	u32 latency_hiding;
2231	fixed20_12 a;
2232
2233	a.full = dfixed_const(1);
2234	if (wm->vsc.full > a.full)
2235		latency_tolerant_lines = 1;
2236	else {
2237		if (lb_partitions <= (wm->vtaps + 1))
2238			latency_tolerant_lines = 1;
2239		else
2240			latency_tolerant_lines = 2;
2241	}
2242
2243	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2244
2245	if (dce6_latency_watermark(wm) <= latency_hiding)
2246		return true;
2247	else
2248		return false;
2249}
2250
2251static void dce6_program_watermarks(struct radeon_device *rdev,
2252					 struct radeon_crtc *radeon_crtc,
2253					 u32 lb_size, u32 num_heads)
2254{
2255	struct drm_display_mode *mode = &radeon_crtc->base.mode;
2256	struct dce6_wm_params wm_low, wm_high;
2257	u32 dram_channels;
2258	u32 pixel_period;
2259	u32 line_time = 0;
2260	u32 latency_watermark_a = 0, latency_watermark_b = 0;
2261	u32 priority_a_mark = 0, priority_b_mark = 0;
2262	u32 priority_a_cnt = PRIORITY_OFF;
2263	u32 priority_b_cnt = PRIORITY_OFF;
2264	u32 tmp, arb_control3;
2265	fixed20_12 a, b, c;
2266
2267	if (radeon_crtc->base.enabled && num_heads && mode) {
2268		pixel_period = 1000000 / (u32)mode->clock;
2269		line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
 
 
 
2270		priority_a_cnt = 0;
2271		priority_b_cnt = 0;
2272
2273		if (rdev->family == CHIP_ARUBA)
2274			dram_channels = evergreen_get_number_of_dram_channels(rdev);
2275		else
2276			dram_channels = si_get_number_of_dram_channels(rdev);
2277
2278		/* watermark for high clocks */
2279		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2280			wm_high.yclk =
2281				radeon_dpm_get_mclk(rdev, false) * 10;
2282			wm_high.sclk =
2283				radeon_dpm_get_sclk(rdev, false) * 10;
2284		} else {
2285			wm_high.yclk = rdev->pm.current_mclk * 10;
2286			wm_high.sclk = rdev->pm.current_sclk * 10;
2287		}
2288
2289		wm_high.disp_clk = mode->clock;
2290		wm_high.src_width = mode->crtc_hdisplay;
2291		wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2292		wm_high.blank_time = line_time - wm_high.active_time;
2293		wm_high.interlaced = false;
2294		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2295			wm_high.interlaced = true;
2296		wm_high.vsc = radeon_crtc->vsc;
2297		wm_high.vtaps = 1;
2298		if (radeon_crtc->rmx_type != RMX_OFF)
2299			wm_high.vtaps = 2;
2300		wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2301		wm_high.lb_size = lb_size;
2302		wm_high.dram_channels = dram_channels;
2303		wm_high.num_heads = num_heads;
2304
2305		/* watermark for low clocks */
2306		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2307			wm_low.yclk =
2308				radeon_dpm_get_mclk(rdev, true) * 10;
2309			wm_low.sclk =
2310				radeon_dpm_get_sclk(rdev, true) * 10;
2311		} else {
2312			wm_low.yclk = rdev->pm.current_mclk * 10;
2313			wm_low.sclk = rdev->pm.current_sclk * 10;
2314		}
2315
2316		wm_low.disp_clk = mode->clock;
2317		wm_low.src_width = mode->crtc_hdisplay;
2318		wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2319		wm_low.blank_time = line_time - wm_low.active_time;
2320		wm_low.interlaced = false;
2321		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2322			wm_low.interlaced = true;
2323		wm_low.vsc = radeon_crtc->vsc;
2324		wm_low.vtaps = 1;
2325		if (radeon_crtc->rmx_type != RMX_OFF)
2326			wm_low.vtaps = 2;
2327		wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2328		wm_low.lb_size = lb_size;
2329		wm_low.dram_channels = dram_channels;
2330		wm_low.num_heads = num_heads;
2331
2332		/* set for high clocks */
2333		latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2334		/* set for low clocks */
2335		latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2336
2337		/* possibly force display priority to high */
2338		/* should really do this at mode validation time... */
2339		if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2340		    !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2341		    !dce6_check_latency_hiding(&wm_high) ||
2342		    (rdev->disp_priority == 2)) {
2343			DRM_DEBUG_KMS("force priority to high\n");
2344			priority_a_cnt |= PRIORITY_ALWAYS_ON;
2345			priority_b_cnt |= PRIORITY_ALWAYS_ON;
2346		}
2347		if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2348		    !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2349		    !dce6_check_latency_hiding(&wm_low) ||
2350		    (rdev->disp_priority == 2)) {
2351			DRM_DEBUG_KMS("force priority to high\n");
2352			priority_a_cnt |= PRIORITY_ALWAYS_ON;
2353			priority_b_cnt |= PRIORITY_ALWAYS_ON;
2354		}
2355
2356		a.full = dfixed_const(1000);
2357		b.full = dfixed_const(mode->clock);
2358		b.full = dfixed_div(b, a);
2359		c.full = dfixed_const(latency_watermark_a);
2360		c.full = dfixed_mul(c, b);
2361		c.full = dfixed_mul(c, radeon_crtc->hsc);
2362		c.full = dfixed_div(c, a);
2363		a.full = dfixed_const(16);
2364		c.full = dfixed_div(c, a);
2365		priority_a_mark = dfixed_trunc(c);
2366		priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2367
2368		a.full = dfixed_const(1000);
2369		b.full = dfixed_const(mode->clock);
2370		b.full = dfixed_div(b, a);
2371		c.full = dfixed_const(latency_watermark_b);
2372		c.full = dfixed_mul(c, b);
2373		c.full = dfixed_mul(c, radeon_crtc->hsc);
2374		c.full = dfixed_div(c, a);
2375		a.full = dfixed_const(16);
2376		c.full = dfixed_div(c, a);
2377		priority_b_mark = dfixed_trunc(c);
2378		priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2379
2380		/* Save number of lines the linebuffer leads before the scanout */
2381		radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2382	}
2383
2384	/* select wm A */
2385	arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2386	tmp = arb_control3;
2387	tmp &= ~LATENCY_WATERMARK_MASK(3);
2388	tmp |= LATENCY_WATERMARK_MASK(1);
2389	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2390	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2391	       (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2392		LATENCY_HIGH_WATERMARK(line_time)));
2393	/* select wm B */
2394	tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2395	tmp &= ~LATENCY_WATERMARK_MASK(3);
2396	tmp |= LATENCY_WATERMARK_MASK(2);
2397	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2398	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2399	       (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2400		LATENCY_HIGH_WATERMARK(line_time)));
2401	/* restore original selection */
2402	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2403
2404	/* write the priority marks */
2405	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2406	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2407
2408	/* save values for DPM */
2409	radeon_crtc->line_time = line_time;
2410	radeon_crtc->wm_high = latency_watermark_a;
2411	radeon_crtc->wm_low = latency_watermark_b;
2412}
2413
2414void dce6_bandwidth_update(struct radeon_device *rdev)
2415{
2416	struct drm_display_mode *mode0 = NULL;
2417	struct drm_display_mode *mode1 = NULL;
2418	u32 num_heads = 0, lb_size;
2419	int i;
2420
2421	if (!rdev->mode_info.mode_config_initialized)
2422		return;
2423
2424	radeon_update_display_priority(rdev);
2425
2426	for (i = 0; i < rdev->num_crtc; i++) {
2427		if (rdev->mode_info.crtcs[i]->base.enabled)
2428			num_heads++;
2429	}
2430	for (i = 0; i < rdev->num_crtc; i += 2) {
2431		mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2432		mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2433		lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2434		dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2435		lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2436		dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2437	}
2438}
2439
2440/*
2441 * Core functions
2442 */
2443static void si_tiling_mode_table_init(struct radeon_device *rdev)
2444{
2445	u32 *tile = rdev->config.si.tile_mode_array;
2446	const u32 num_tile_mode_states =
2447			ARRAY_SIZE(rdev->config.si.tile_mode_array);
2448	u32 reg_offset, split_equal_to_row_size;
2449
2450	switch (rdev->config.si.mem_row_size_in_kb) {
2451	case 1:
2452		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2453		break;
2454	case 2:
2455	default:
2456		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2457		break;
2458	case 4:
2459		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2460		break;
2461	}
2462
2463	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2464		tile[reg_offset] = 0;
2465
2466	switch(rdev->family) {
2467	case CHIP_TAHITI:
2468	case CHIP_PITCAIRN:
2469		/* non-AA compressed depth or any compressed stencil */
2470		tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2471			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2472			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2473			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2474			   NUM_BANKS(ADDR_SURF_16_BANK) |
2475			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2476			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2477			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2478		/* 2xAA/4xAA compressed depth only */
2479		tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2480			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2481			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2482			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2483			   NUM_BANKS(ADDR_SURF_16_BANK) |
2484			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2486			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2487		/* 8xAA compressed depth only */
2488		tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2489			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2490			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2491			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2492			   NUM_BANKS(ADDR_SURF_16_BANK) |
2493			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2494			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2495			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2496		/* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2497		tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2498			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2499			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2500			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2501			   NUM_BANKS(ADDR_SURF_16_BANK) |
2502			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2503			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2504			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2505		/* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2506		tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2507			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2508			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2509			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2510			   NUM_BANKS(ADDR_SURF_16_BANK) |
2511			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2512			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2513			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2514		/* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2515		tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2516			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2517			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2518			   TILE_SPLIT(split_equal_to_row_size) |
2519			   NUM_BANKS(ADDR_SURF_16_BANK) |
2520			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2521			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2522			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2523		/* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2524		tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2525			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2526			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2527			   TILE_SPLIT(split_equal_to_row_size) |
2528			   NUM_BANKS(ADDR_SURF_16_BANK) |
2529			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2530			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2531			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2532		/* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2533		tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2534			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2535			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2536			   TILE_SPLIT(split_equal_to_row_size) |
2537			   NUM_BANKS(ADDR_SURF_16_BANK) |
2538			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2539			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2540			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2541		/* 1D and 1D Array Surfaces */
2542		tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2543			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2544			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2545			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2546			   NUM_BANKS(ADDR_SURF_16_BANK) |
2547			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2548			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2549			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2550		/* Displayable maps. */
2551		tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2552			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2553			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2554			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2555			   NUM_BANKS(ADDR_SURF_16_BANK) |
2556			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2557			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2558			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2559		/* Display 8bpp. */
2560		tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2561			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2562			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2563			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2564			   NUM_BANKS(ADDR_SURF_16_BANK) |
2565			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2566			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2567			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2568		/* Display 16bpp. */
2569		tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2570			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2571			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2572			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2573			   NUM_BANKS(ADDR_SURF_16_BANK) |
2574			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2575			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2576			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2577		/* Display 32bpp. */
2578		tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2579			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2580			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2581			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2582			   NUM_BANKS(ADDR_SURF_16_BANK) |
2583			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2584			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2585			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2586		/* Thin. */
2587		tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2588			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2589			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2590			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2591			   NUM_BANKS(ADDR_SURF_16_BANK) |
2592			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2593			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2594			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2595		/* Thin 8 bpp. */
2596		tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2597			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2598			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2599			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2600			   NUM_BANKS(ADDR_SURF_16_BANK) |
2601			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2602			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2603			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2604		/* Thin 16 bpp. */
2605		tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2606			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2607			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2608			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2609			   NUM_BANKS(ADDR_SURF_16_BANK) |
2610			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2611			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2612			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2613		/* Thin 32 bpp. */
2614		tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2615			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2616			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2617			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2618			   NUM_BANKS(ADDR_SURF_16_BANK) |
2619			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2620			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2621			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2622		/* Thin 64 bpp. */
2623		tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2624			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2625			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2626			   TILE_SPLIT(split_equal_to_row_size) |
2627			   NUM_BANKS(ADDR_SURF_16_BANK) |
2628			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2629			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2630			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2631		/* 8 bpp PRT. */
2632		tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2633			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2634			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2635			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2636			   NUM_BANKS(ADDR_SURF_16_BANK) |
2637			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2638			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2639			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2640		/* 16 bpp PRT */
2641		tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2643			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2644			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2645			   NUM_BANKS(ADDR_SURF_16_BANK) |
2646			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2647			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2648			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2649		/* 32 bpp PRT */
2650		tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2651			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2652			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2653			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2654			   NUM_BANKS(ADDR_SURF_16_BANK) |
2655			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2656			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2657			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2658		/* 64 bpp PRT */
2659		tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2660			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2661			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2662			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2663			   NUM_BANKS(ADDR_SURF_16_BANK) |
2664			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2665			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2666			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2667		/* 128 bpp PRT */
2668		tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2669			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2670			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2671			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2672			   NUM_BANKS(ADDR_SURF_8_BANK) |
2673			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2674			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2675			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2676
2677		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2678			WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2679		break;
2680
2681	case CHIP_VERDE:
2682	case CHIP_OLAND:
2683	case CHIP_HAINAN:
2684		/* non-AA compressed depth or any compressed stencil */
2685		tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2686			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2687			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2688			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2689			   NUM_BANKS(ADDR_SURF_16_BANK) |
2690			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2692			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2693		/* 2xAA/4xAA compressed depth only */
2694		tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2695			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2696			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2697			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2698			   NUM_BANKS(ADDR_SURF_16_BANK) |
2699			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2700			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2701			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2702		/* 8xAA compressed depth only */
2703		tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2704			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2705			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2706			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2707			   NUM_BANKS(ADDR_SURF_16_BANK) |
2708			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2710			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2711		/* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2712		tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2713			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2714			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2715			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2716			   NUM_BANKS(ADDR_SURF_16_BANK) |
2717			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2718			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2719			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2720		/* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2721		tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2722			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2723			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2724			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2725			   NUM_BANKS(ADDR_SURF_16_BANK) |
2726			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2728			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2729		/* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2730		tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2731			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2732			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2733			   TILE_SPLIT(split_equal_to_row_size) |
2734			   NUM_BANKS(ADDR_SURF_16_BANK) |
2735			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2736			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2737			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2738		/* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2739		tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2740			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2741			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2742			   TILE_SPLIT(split_equal_to_row_size) |
2743			   NUM_BANKS(ADDR_SURF_16_BANK) |
2744			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2745			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2746			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2747		/* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2748		tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2749			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2750			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2751			   TILE_SPLIT(split_equal_to_row_size) |
2752			   NUM_BANKS(ADDR_SURF_16_BANK) |
2753			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2754			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2755			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2756		/* 1D and 1D Array Surfaces */
2757		tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2758			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2759			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2760			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2761			   NUM_BANKS(ADDR_SURF_16_BANK) |
2762			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2763			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2764			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2765		/* Displayable maps. */
2766		tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2767			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2768			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2769			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2770			   NUM_BANKS(ADDR_SURF_16_BANK) |
2771			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2772			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2773			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2774		/* Display 8bpp. */
2775		tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2776			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2777			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2778			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2779			   NUM_BANKS(ADDR_SURF_16_BANK) |
2780			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2781			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2782			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2783		/* Display 16bpp. */
2784		tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2785			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2786			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2787			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2788			   NUM_BANKS(ADDR_SURF_16_BANK) |
2789			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2790			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2791			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2792		/* Display 32bpp. */
2793		tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2794			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2795			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2796			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2797			   NUM_BANKS(ADDR_SURF_16_BANK) |
2798			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2799			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2800			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2801		/* Thin. */
2802		tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2803			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2804			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2805			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2806			   NUM_BANKS(ADDR_SURF_16_BANK) |
2807			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2808			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2809			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2810		/* Thin 8 bpp. */
2811		tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2812			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2813			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2815			   NUM_BANKS(ADDR_SURF_16_BANK) |
2816			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2817			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2818			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2819		/* Thin 16 bpp. */
2820		tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2821			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2822			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2823			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2824			   NUM_BANKS(ADDR_SURF_16_BANK) |
2825			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2826			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2827			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2828		/* Thin 32 bpp. */
2829		tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2830			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2831			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2832			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2833			   NUM_BANKS(ADDR_SURF_16_BANK) |
2834			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2835			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2836			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2837		/* Thin 64 bpp. */
2838		tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2839			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2840			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2841			   TILE_SPLIT(split_equal_to_row_size) |
2842			   NUM_BANKS(ADDR_SURF_16_BANK) |
2843			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2844			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2845			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2846		/* 8 bpp PRT. */
2847		tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2848			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2849			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2850			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2851			   NUM_BANKS(ADDR_SURF_16_BANK) |
2852			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2853			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2854			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2855		/* 16 bpp PRT */
2856		tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2857			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2858			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2859			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2860			   NUM_BANKS(ADDR_SURF_16_BANK) |
2861			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2862			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2863			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2864		/* 32 bpp PRT */
2865		tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2866			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2867			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2868			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2869			   NUM_BANKS(ADDR_SURF_16_BANK) |
2870			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2871			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2872			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2873		/* 64 bpp PRT */
2874		tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2875			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2876			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2877			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2878			   NUM_BANKS(ADDR_SURF_16_BANK) |
2879			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2880			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2881			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2882		/* 128 bpp PRT */
2883		tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2884			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2885			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2886			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2887			   NUM_BANKS(ADDR_SURF_8_BANK) |
2888			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2889			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2890			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2891
2892		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2893			WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2894		break;
2895
2896	default:
2897		DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2898	}
2899}
2900
2901static void si_select_se_sh(struct radeon_device *rdev,
2902			    u32 se_num, u32 sh_num)
2903{
2904	u32 data = INSTANCE_BROADCAST_WRITES;
2905
2906	if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2907		data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2908	else if (se_num == 0xffffffff)
2909		data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2910	else if (sh_num == 0xffffffff)
2911		data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2912	else
2913		data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2914	WREG32(GRBM_GFX_INDEX, data);
2915}
2916
2917static u32 si_create_bitmask(u32 bit_width)
2918{
2919	u32 i, mask = 0;
2920
2921	for (i = 0; i < bit_width; i++) {
2922		mask <<= 1;
2923		mask |= 1;
2924	}
2925	return mask;
2926}
2927
2928static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2929{
2930	u32 data, mask;
2931
2932	data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2933	if (data & 1)
2934		data &= INACTIVE_CUS_MASK;
2935	else
2936		data = 0;
2937	data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2938
2939	data >>= INACTIVE_CUS_SHIFT;
2940
2941	mask = si_create_bitmask(cu_per_sh);
2942
2943	return ~data & mask;
2944}
2945
2946static void si_setup_spi(struct radeon_device *rdev,
2947			 u32 se_num, u32 sh_per_se,
2948			 u32 cu_per_sh)
2949{
2950	int i, j, k;
2951	u32 data, mask, active_cu;
2952
2953	for (i = 0; i < se_num; i++) {
2954		for (j = 0; j < sh_per_se; j++) {
2955			si_select_se_sh(rdev, i, j);
2956			data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2957			active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2958
2959			mask = 1;
2960			for (k = 0; k < 16; k++) {
2961				mask <<= k;
2962				if (active_cu & mask) {
2963					data &= ~mask;
2964					WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2965					break;
2966				}
2967			}
2968		}
2969	}
2970	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2971}
2972
2973static u32 si_get_rb_disabled(struct radeon_device *rdev,
2974			      u32 max_rb_num_per_se,
2975			      u32 sh_per_se)
2976{
2977	u32 data, mask;
2978
2979	data = RREG32(CC_RB_BACKEND_DISABLE);
2980	if (data & 1)
2981		data &= BACKEND_DISABLE_MASK;
2982	else
2983		data = 0;
2984	data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2985
2986	data >>= BACKEND_DISABLE_SHIFT;
2987
2988	mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
2989
2990	return data & mask;
2991}
2992
2993static void si_setup_rb(struct radeon_device *rdev,
2994			u32 se_num, u32 sh_per_se,
2995			u32 max_rb_num_per_se)
2996{
2997	int i, j;
2998	u32 data, mask;
2999	u32 disabled_rbs = 0;
3000	u32 enabled_rbs = 0;
3001
3002	for (i = 0; i < se_num; i++) {
3003		for (j = 0; j < sh_per_se; j++) {
3004			si_select_se_sh(rdev, i, j);
3005			data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3006			disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
3007		}
3008	}
3009	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3010
3011	mask = 1;
3012	for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3013		if (!(disabled_rbs & mask))
3014			enabled_rbs |= mask;
3015		mask <<= 1;
3016	}
3017
3018	rdev->config.si.backend_enable_mask = enabled_rbs;
3019
3020	for (i = 0; i < se_num; i++) {
3021		si_select_se_sh(rdev, i, 0xffffffff);
3022		data = 0;
3023		for (j = 0; j < sh_per_se; j++) {
3024			switch (enabled_rbs & 3) {
3025			case 1:
3026				data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3027				break;
3028			case 2:
3029				data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3030				break;
3031			case 3:
3032			default:
3033				data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3034				break;
3035			}
3036			enabled_rbs >>= 2;
3037		}
3038		WREG32(PA_SC_RASTER_CONFIG, data);
3039	}
3040	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3041}
3042
3043static void si_gpu_init(struct radeon_device *rdev)
3044{
3045	u32 gb_addr_config = 0;
3046	u32 mc_shared_chmap, mc_arb_ramcfg;
3047	u32 sx_debug_1;
3048	u32 hdp_host_path_cntl;
3049	u32 tmp;
3050	int i, j;
3051
3052	switch (rdev->family) {
3053	case CHIP_TAHITI:
3054		rdev->config.si.max_shader_engines = 2;
3055		rdev->config.si.max_tile_pipes = 12;
3056		rdev->config.si.max_cu_per_sh = 8;
3057		rdev->config.si.max_sh_per_se = 2;
3058		rdev->config.si.max_backends_per_se = 4;
3059		rdev->config.si.max_texture_channel_caches = 12;
3060		rdev->config.si.max_gprs = 256;
3061		rdev->config.si.max_gs_threads = 32;
3062		rdev->config.si.max_hw_contexts = 8;
3063
3064		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3065		rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3066		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3067		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3068		gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3069		break;
3070	case CHIP_PITCAIRN:
3071		rdev->config.si.max_shader_engines = 2;
3072		rdev->config.si.max_tile_pipes = 8;
3073		rdev->config.si.max_cu_per_sh = 5;
3074		rdev->config.si.max_sh_per_se = 2;
3075		rdev->config.si.max_backends_per_se = 4;
3076		rdev->config.si.max_texture_channel_caches = 8;
3077		rdev->config.si.max_gprs = 256;
3078		rdev->config.si.max_gs_threads = 32;
3079		rdev->config.si.max_hw_contexts = 8;
3080
3081		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3082		rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3083		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3084		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3085		gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3086		break;
3087	case CHIP_VERDE:
3088	default:
3089		rdev->config.si.max_shader_engines = 1;
3090		rdev->config.si.max_tile_pipes = 4;
3091		rdev->config.si.max_cu_per_sh = 5;
3092		rdev->config.si.max_sh_per_se = 2;
3093		rdev->config.si.max_backends_per_se = 4;
3094		rdev->config.si.max_texture_channel_caches = 4;
3095		rdev->config.si.max_gprs = 256;
3096		rdev->config.si.max_gs_threads = 32;
3097		rdev->config.si.max_hw_contexts = 8;
3098
3099		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3100		rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3101		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3102		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3103		gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3104		break;
3105	case CHIP_OLAND:
3106		rdev->config.si.max_shader_engines = 1;
3107		rdev->config.si.max_tile_pipes = 4;
3108		rdev->config.si.max_cu_per_sh = 6;
3109		rdev->config.si.max_sh_per_se = 1;
3110		rdev->config.si.max_backends_per_se = 2;
3111		rdev->config.si.max_texture_channel_caches = 4;
3112		rdev->config.si.max_gprs = 256;
3113		rdev->config.si.max_gs_threads = 16;
3114		rdev->config.si.max_hw_contexts = 8;
3115
3116		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3117		rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3118		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3119		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3120		gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3121		break;
3122	case CHIP_HAINAN:
3123		rdev->config.si.max_shader_engines = 1;
3124		rdev->config.si.max_tile_pipes = 4;
3125		rdev->config.si.max_cu_per_sh = 5;
3126		rdev->config.si.max_sh_per_se = 1;
3127		rdev->config.si.max_backends_per_se = 1;
3128		rdev->config.si.max_texture_channel_caches = 2;
3129		rdev->config.si.max_gprs = 256;
3130		rdev->config.si.max_gs_threads = 16;
3131		rdev->config.si.max_hw_contexts = 8;
3132
3133		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3134		rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3135		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3136		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3137		gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3138		break;
3139	}
3140
3141	/* Initialize HDP */
3142	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3143		WREG32((0x2c14 + j), 0x00000000);
3144		WREG32((0x2c18 + j), 0x00000000);
3145		WREG32((0x2c1c + j), 0x00000000);
3146		WREG32((0x2c20 + j), 0x00000000);
3147		WREG32((0x2c24 + j), 0x00000000);
3148	}
3149
3150	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3151	WREG32(SRBM_INT_CNTL, 1);
3152	WREG32(SRBM_INT_ACK, 1);
3153
3154	evergreen_fix_pci_max_read_req_size(rdev);
3155
3156	WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3157
3158	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3159	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3160
3161	rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3162	rdev->config.si.mem_max_burst_length_bytes = 256;
3163	tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3164	rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3165	if (rdev->config.si.mem_row_size_in_kb > 4)
3166		rdev->config.si.mem_row_size_in_kb = 4;
3167	/* XXX use MC settings? */
3168	rdev->config.si.shader_engine_tile_size = 32;
3169	rdev->config.si.num_gpus = 1;
3170	rdev->config.si.multi_gpu_tile_size = 64;
3171
3172	/* fix up row size */
3173	gb_addr_config &= ~ROW_SIZE_MASK;
3174	switch (rdev->config.si.mem_row_size_in_kb) {
3175	case 1:
3176	default:
3177		gb_addr_config |= ROW_SIZE(0);
3178		break;
3179	case 2:
3180		gb_addr_config |= ROW_SIZE(1);
3181		break;
3182	case 4:
3183		gb_addr_config |= ROW_SIZE(2);
3184		break;
3185	}
3186
3187	/* setup tiling info dword.  gb_addr_config is not adequate since it does
3188	 * not have bank info, so create a custom tiling dword.
3189	 * bits 3:0   num_pipes
3190	 * bits 7:4   num_banks
3191	 * bits 11:8  group_size
3192	 * bits 15:12 row_size
3193	 */
3194	rdev->config.si.tile_config = 0;
3195	switch (rdev->config.si.num_tile_pipes) {
3196	case 1:
3197		rdev->config.si.tile_config |= (0 << 0);
3198		break;
3199	case 2:
3200		rdev->config.si.tile_config |= (1 << 0);
3201		break;
3202	case 4:
3203		rdev->config.si.tile_config |= (2 << 0);
3204		break;
3205	case 8:
3206	default:
3207		/* XXX what about 12? */
3208		rdev->config.si.tile_config |= (3 << 0);
3209		break;
3210	}	
3211	switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3212	case 0: /* four banks */
3213		rdev->config.si.tile_config |= 0 << 4;
3214		break;
3215	case 1: /* eight banks */
3216		rdev->config.si.tile_config |= 1 << 4;
3217		break;
3218	case 2: /* sixteen banks */
3219	default:
3220		rdev->config.si.tile_config |= 2 << 4;
3221		break;
3222	}
3223	rdev->config.si.tile_config |=
3224		((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3225	rdev->config.si.tile_config |=
3226		((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3227
3228	WREG32(GB_ADDR_CONFIG, gb_addr_config);
3229	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3230	WREG32(DMIF_ADDR_CALC, gb_addr_config);
3231	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3232	WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3233	WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3234	if (rdev->has_uvd) {
3235		WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3236		WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3237		WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3238	}
3239
3240	si_tiling_mode_table_init(rdev);
3241
3242	si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3243		    rdev->config.si.max_sh_per_se,
3244		    rdev->config.si.max_backends_per_se);
3245
3246	si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3247		     rdev->config.si.max_sh_per_se,
3248		     rdev->config.si.max_cu_per_sh);
3249
3250	rdev->config.si.active_cus = 0;
3251	for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3252		for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3253			rdev->config.si.active_cus +=
3254				hweight32(si_get_cu_active_bitmap(rdev, i, j));
3255		}
3256	}
3257
3258	/* set HW defaults for 3D engine */
3259	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3260				     ROQ_IB2_START(0x2b)));
3261	WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3262
3263	sx_debug_1 = RREG32(SX_DEBUG_1);
3264	WREG32(SX_DEBUG_1, sx_debug_1);
3265
3266	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3267
3268	WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3269				 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3270				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3271				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3272
3273	WREG32(VGT_NUM_INSTANCES, 1);
3274
3275	WREG32(CP_PERFMON_CNTL, 0);
3276
3277	WREG32(SQ_CONFIG, 0);
3278
3279	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3280					  FORCE_EOV_MAX_REZ_CNT(255)));
3281
3282	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3283	       AUTO_INVLD_EN(ES_AND_GS_AUTO));
3284
3285	WREG32(VGT_GS_VERTEX_REUSE, 16);
3286	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3287
3288	WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3289	WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3290	WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3291	WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3292	WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3293	WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3294	WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3295	WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3296
3297	tmp = RREG32(HDP_MISC_CNTL);
3298	tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3299	WREG32(HDP_MISC_CNTL, tmp);
3300
3301	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3302	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3303
3304	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3305
3306	udelay(50);
3307}
3308
3309/*
3310 * GPU scratch registers helpers function.
3311 */
3312static void si_scratch_init(struct radeon_device *rdev)
3313{
3314	int i;
3315
3316	rdev->scratch.num_reg = 7;
3317	rdev->scratch.reg_base = SCRATCH_REG0;
3318	for (i = 0; i < rdev->scratch.num_reg; i++) {
3319		rdev->scratch.free[i] = true;
3320		rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3321	}
3322}
3323
3324void si_fence_ring_emit(struct radeon_device *rdev,
3325			struct radeon_fence *fence)
3326{
3327	struct radeon_ring *ring = &rdev->ring[fence->ring];
3328	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3329
3330	/* flush read cache over gart */
3331	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3332	radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3333	radeon_ring_write(ring, 0);
3334	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3335	radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3336			  PACKET3_TC_ACTION_ENA |
3337			  PACKET3_SH_KCACHE_ACTION_ENA |
3338			  PACKET3_SH_ICACHE_ACTION_ENA);
3339	radeon_ring_write(ring, 0xFFFFFFFF);
3340	radeon_ring_write(ring, 0);
3341	radeon_ring_write(ring, 10); /* poll interval */
3342	/* EVENT_WRITE_EOP - flush caches, send int */
3343	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3344	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3345	radeon_ring_write(ring, lower_32_bits(addr));
3346	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3347	radeon_ring_write(ring, fence->seq);
3348	radeon_ring_write(ring, 0);
3349}
3350
3351/*
3352 * IB stuff
3353 */
3354void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3355{
3356	struct radeon_ring *ring = &rdev->ring[ib->ring];
3357	unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3358	u32 header;
3359
3360	if (ib->is_const_ib) {
3361		/* set switch buffer packet before const IB */
3362		radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3363		radeon_ring_write(ring, 0);
3364
3365		header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3366	} else {
3367		u32 next_rptr;
3368		if (ring->rptr_save_reg) {
3369			next_rptr = ring->wptr + 3 + 4 + 8;
3370			radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3371			radeon_ring_write(ring, ((ring->rptr_save_reg -
3372						  PACKET3_SET_CONFIG_REG_START) >> 2));
3373			radeon_ring_write(ring, next_rptr);
3374		} else if (rdev->wb.enabled) {
3375			next_rptr = ring->wptr + 5 + 4 + 8;
3376			radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3377			radeon_ring_write(ring, (1 << 8));
3378			radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3379			radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3380			radeon_ring_write(ring, next_rptr);
3381		}
3382
3383		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3384	}
3385
3386	radeon_ring_write(ring, header);
3387	radeon_ring_write(ring,
3388#ifdef __BIG_ENDIAN
3389			  (2 << 0) |
3390#endif
3391			  (ib->gpu_addr & 0xFFFFFFFC));
3392	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3393	radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
3394
3395	if (!ib->is_const_ib) {
3396		/* flush read cache over gart for this vmid */
3397		radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3398		radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3399		radeon_ring_write(ring, vm_id);
3400		radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3401		radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3402				  PACKET3_TC_ACTION_ENA |
3403				  PACKET3_SH_KCACHE_ACTION_ENA |
3404				  PACKET3_SH_ICACHE_ACTION_ENA);
3405		radeon_ring_write(ring, 0xFFFFFFFF);
3406		radeon_ring_write(ring, 0);
3407		radeon_ring_write(ring, 10); /* poll interval */
3408	}
3409}
3410
3411/*
3412 * CP.
3413 */
3414static void si_cp_enable(struct radeon_device *rdev, bool enable)
3415{
3416	if (enable)
3417		WREG32(CP_ME_CNTL, 0);
3418	else {
3419		if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3420			radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3421		WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3422		WREG32(SCRATCH_UMSK, 0);
3423		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3424		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3425		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3426	}
3427	udelay(50);
3428}
3429
3430static int si_cp_load_microcode(struct radeon_device *rdev)
3431{
3432	int i;
3433
3434	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3435		return -EINVAL;
3436
3437	si_cp_enable(rdev, false);
3438
3439	if (rdev->new_fw) {
3440		const struct gfx_firmware_header_v1_0 *pfp_hdr =
3441			(const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3442		const struct gfx_firmware_header_v1_0 *ce_hdr =
3443			(const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3444		const struct gfx_firmware_header_v1_0 *me_hdr =
3445			(const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3446		const __le32 *fw_data;
3447		u32 fw_size;
3448
3449		radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3450		radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3451		radeon_ucode_print_gfx_hdr(&me_hdr->header);
3452
3453		/* PFP */
3454		fw_data = (const __le32 *)
3455			(rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3456		fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3457		WREG32(CP_PFP_UCODE_ADDR, 0);
3458		for (i = 0; i < fw_size; i++)
3459			WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3460		WREG32(CP_PFP_UCODE_ADDR, 0);
3461
3462		/* CE */
3463		fw_data = (const __le32 *)
3464			(rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3465		fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3466		WREG32(CP_CE_UCODE_ADDR, 0);
3467		for (i = 0; i < fw_size; i++)
3468			WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3469		WREG32(CP_CE_UCODE_ADDR, 0);
3470
3471		/* ME */
3472		fw_data = (const __be32 *)
3473			(rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3474		fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3475		WREG32(CP_ME_RAM_WADDR, 0);
3476		for (i = 0; i < fw_size; i++)
3477			WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3478		WREG32(CP_ME_RAM_WADDR, 0);
3479	} else {
3480		const __be32 *fw_data;
3481
3482		/* PFP */
3483		fw_data = (const __be32 *)rdev->pfp_fw->data;
3484		WREG32(CP_PFP_UCODE_ADDR, 0);
3485		for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3486			WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3487		WREG32(CP_PFP_UCODE_ADDR, 0);
3488
3489		/* CE */
3490		fw_data = (const __be32 *)rdev->ce_fw->data;
3491		WREG32(CP_CE_UCODE_ADDR, 0);
3492		for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3493			WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3494		WREG32(CP_CE_UCODE_ADDR, 0);
3495
3496		/* ME */
3497		fw_data = (const __be32 *)rdev->me_fw->data;
3498		WREG32(CP_ME_RAM_WADDR, 0);
3499		for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3500			WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3501		WREG32(CP_ME_RAM_WADDR, 0);
3502	}
3503
3504	WREG32(CP_PFP_UCODE_ADDR, 0);
3505	WREG32(CP_CE_UCODE_ADDR, 0);
3506	WREG32(CP_ME_RAM_WADDR, 0);
3507	WREG32(CP_ME_RAM_RADDR, 0);
3508	return 0;
3509}
3510
3511static int si_cp_start(struct radeon_device *rdev)
3512{
3513	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3514	int r, i;
3515
3516	r = radeon_ring_lock(rdev, ring, 7 + 4);
3517	if (r) {
3518		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3519		return r;
3520	}
3521	/* init the CP */
3522	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3523	radeon_ring_write(ring, 0x1);
3524	radeon_ring_write(ring, 0x0);
3525	radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3526	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3527	radeon_ring_write(ring, 0);
3528	radeon_ring_write(ring, 0);
3529
3530	/* init the CE partitions */
3531	radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3532	radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3533	radeon_ring_write(ring, 0xc000);
3534	radeon_ring_write(ring, 0xe000);
3535	radeon_ring_unlock_commit(rdev, ring, false);
3536
3537	si_cp_enable(rdev, true);
3538
3539	r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3540	if (r) {
3541		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3542		return r;
3543	}
3544
3545	/* setup clear context state */
3546	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3547	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3548
3549	for (i = 0; i < si_default_size; i++)
3550		radeon_ring_write(ring, si_default_state[i]);
3551
3552	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3553	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3554
3555	/* set clear context state */
3556	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3557	radeon_ring_write(ring, 0);
3558
3559	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3560	radeon_ring_write(ring, 0x00000316);
3561	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3562	radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3563
3564	radeon_ring_unlock_commit(rdev, ring, false);
3565
3566	for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3567		ring = &rdev->ring[i];
3568		r = radeon_ring_lock(rdev, ring, 2);
 
 
 
 
3569
3570		/* clear the compute context state */
3571		radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3572		radeon_ring_write(ring, 0);
3573
3574		radeon_ring_unlock_commit(rdev, ring, false);
3575	}
3576
3577	return 0;
3578}
3579
3580static void si_cp_fini(struct radeon_device *rdev)
3581{
3582	struct radeon_ring *ring;
3583	si_cp_enable(rdev, false);
3584
3585	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3586	radeon_ring_fini(rdev, ring);
3587	radeon_scratch_free(rdev, ring->rptr_save_reg);
3588
3589	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3590	radeon_ring_fini(rdev, ring);
3591	radeon_scratch_free(rdev, ring->rptr_save_reg);
3592
3593	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3594	radeon_ring_fini(rdev, ring);
3595	radeon_scratch_free(rdev, ring->rptr_save_reg);
3596}
3597
3598static int si_cp_resume(struct radeon_device *rdev)
3599{
3600	struct radeon_ring *ring;
3601	u32 tmp;
3602	u32 rb_bufsz;
3603	int r;
3604
3605	si_enable_gui_idle_interrupt(rdev, false);
3606
3607	WREG32(CP_SEM_WAIT_TIMER, 0x0);
3608	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3609
3610	/* Set the write pointer delay */
3611	WREG32(CP_RB_WPTR_DELAY, 0);
3612
3613	WREG32(CP_DEBUG, 0);
3614	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3615
3616	/* ring 0 - compute and gfx */
3617	/* Set ring buffer size */
3618	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3619	rb_bufsz = order_base_2(ring->ring_size / 8);
3620	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3621#ifdef __BIG_ENDIAN
3622	tmp |= BUF_SWAP_32BIT;
3623#endif
3624	WREG32(CP_RB0_CNTL, tmp);
3625
3626	/* Initialize the ring buffer's read and write pointers */
3627	WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3628	ring->wptr = 0;
3629	WREG32(CP_RB0_WPTR, ring->wptr);
3630
3631	/* set the wb address whether it's enabled or not */
3632	WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3633	WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3634
3635	if (rdev->wb.enabled)
3636		WREG32(SCRATCH_UMSK, 0xff);
3637	else {
3638		tmp |= RB_NO_UPDATE;
3639		WREG32(SCRATCH_UMSK, 0);
3640	}
3641
3642	mdelay(1);
3643	WREG32(CP_RB0_CNTL, tmp);
3644
3645	WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3646
3647	/* ring1  - compute only */
3648	/* Set ring buffer size */
3649	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3650	rb_bufsz = order_base_2(ring->ring_size / 8);
3651	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3652#ifdef __BIG_ENDIAN
3653	tmp |= BUF_SWAP_32BIT;
3654#endif
3655	WREG32(CP_RB1_CNTL, tmp);
3656
3657	/* Initialize the ring buffer's read and write pointers */
3658	WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3659	ring->wptr = 0;
3660	WREG32(CP_RB1_WPTR, ring->wptr);
3661
3662	/* set the wb address whether it's enabled or not */
3663	WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3664	WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3665
3666	mdelay(1);
3667	WREG32(CP_RB1_CNTL, tmp);
3668
3669	WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3670
3671	/* ring2 - compute only */
3672	/* Set ring buffer size */
3673	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3674	rb_bufsz = order_base_2(ring->ring_size / 8);
3675	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3676#ifdef __BIG_ENDIAN
3677	tmp |= BUF_SWAP_32BIT;
3678#endif
3679	WREG32(CP_RB2_CNTL, tmp);
3680
3681	/* Initialize the ring buffer's read and write pointers */
3682	WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3683	ring->wptr = 0;
3684	WREG32(CP_RB2_WPTR, ring->wptr);
3685
3686	/* set the wb address whether it's enabled or not */
3687	WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3688	WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3689
3690	mdelay(1);
3691	WREG32(CP_RB2_CNTL, tmp);
3692
3693	WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3694
3695	/* start the rings */
3696	si_cp_start(rdev);
3697	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3698	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3699	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3700	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3701	if (r) {
3702		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3703		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3704		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3705		return r;
3706	}
3707	r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3708	if (r) {
3709		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3710	}
3711	r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3712	if (r) {
3713		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3714	}
3715
3716	si_enable_gui_idle_interrupt(rdev, true);
3717
3718	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3719		radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3720
3721	return 0;
3722}
3723
3724u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3725{
3726	u32 reset_mask = 0;
3727	u32 tmp;
3728
3729	/* GRBM_STATUS */
3730	tmp = RREG32(GRBM_STATUS);
3731	if (tmp & (PA_BUSY | SC_BUSY |
3732		   BCI_BUSY | SX_BUSY |
3733		   TA_BUSY | VGT_BUSY |
3734		   DB_BUSY | CB_BUSY |
3735		   GDS_BUSY | SPI_BUSY |
3736		   IA_BUSY | IA_BUSY_NO_DMA))
3737		reset_mask |= RADEON_RESET_GFX;
3738
3739	if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3740		   CP_BUSY | CP_COHERENCY_BUSY))
3741		reset_mask |= RADEON_RESET_CP;
3742
3743	if (tmp & GRBM_EE_BUSY)
3744		reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3745
3746	/* GRBM_STATUS2 */
3747	tmp = RREG32(GRBM_STATUS2);
3748	if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3749		reset_mask |= RADEON_RESET_RLC;
3750
3751	/* DMA_STATUS_REG 0 */
3752	tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3753	if (!(tmp & DMA_IDLE))
3754		reset_mask |= RADEON_RESET_DMA;
3755
3756	/* DMA_STATUS_REG 1 */
3757	tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3758	if (!(tmp & DMA_IDLE))
3759		reset_mask |= RADEON_RESET_DMA1;
3760
3761	/* SRBM_STATUS2 */
3762	tmp = RREG32(SRBM_STATUS2);
3763	if (tmp & DMA_BUSY)
3764		reset_mask |= RADEON_RESET_DMA;
3765
3766	if (tmp & DMA1_BUSY)
3767		reset_mask |= RADEON_RESET_DMA1;
3768
3769	/* SRBM_STATUS */
3770	tmp = RREG32(SRBM_STATUS);
3771
3772	if (tmp & IH_BUSY)
3773		reset_mask |= RADEON_RESET_IH;
3774
3775	if (tmp & SEM_BUSY)
3776		reset_mask |= RADEON_RESET_SEM;
3777
3778	if (tmp & GRBM_RQ_PENDING)
3779		reset_mask |= RADEON_RESET_GRBM;
3780
3781	if (tmp & VMC_BUSY)
3782		reset_mask |= RADEON_RESET_VMC;
3783
3784	if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3785		   MCC_BUSY | MCD_BUSY))
3786		reset_mask |= RADEON_RESET_MC;
3787
3788	if (evergreen_is_display_hung(rdev))
3789		reset_mask |= RADEON_RESET_DISPLAY;
3790
3791	/* VM_L2_STATUS */
3792	tmp = RREG32(VM_L2_STATUS);
3793	if (tmp & L2_BUSY)
3794		reset_mask |= RADEON_RESET_VMC;
3795
3796	/* Skip MC reset as it's mostly likely not hung, just busy */
3797	if (reset_mask & RADEON_RESET_MC) {
3798		DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3799		reset_mask &= ~RADEON_RESET_MC;
3800	}
3801
3802	return reset_mask;
3803}
3804
3805static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3806{
3807	struct evergreen_mc_save save;
3808	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3809	u32 tmp;
3810
3811	if (reset_mask == 0)
3812		return;
3813
3814	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3815
3816	evergreen_print_gpu_status_regs(rdev);
3817	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
3818		 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3819	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3820		 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3821
3822	/* disable PG/CG */
3823	si_fini_pg(rdev);
3824	si_fini_cg(rdev);
3825
3826	/* stop the rlc */
3827	si_rlc_stop(rdev);
3828
3829	/* Disable CP parsing/prefetching */
3830	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3831
3832	if (reset_mask & RADEON_RESET_DMA) {
3833		/* dma0 */
3834		tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3835		tmp &= ~DMA_RB_ENABLE;
3836		WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3837	}
3838	if (reset_mask & RADEON_RESET_DMA1) {
3839		/* dma1 */
3840		tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3841		tmp &= ~DMA_RB_ENABLE;
3842		WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3843	}
3844
3845	udelay(50);
3846
3847	evergreen_mc_stop(rdev, &save);
3848	if (evergreen_mc_wait_for_idle(rdev)) {
3849		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3850	}
3851
3852	if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3853		grbm_soft_reset = SOFT_RESET_CB |
3854			SOFT_RESET_DB |
3855			SOFT_RESET_GDS |
3856			SOFT_RESET_PA |
3857			SOFT_RESET_SC |
3858			SOFT_RESET_BCI |
3859			SOFT_RESET_SPI |
3860			SOFT_RESET_SX |
3861			SOFT_RESET_TC |
3862			SOFT_RESET_TA |
3863			SOFT_RESET_VGT |
3864			SOFT_RESET_IA;
3865	}
3866
3867	if (reset_mask & RADEON_RESET_CP) {
3868		grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3869
3870		srbm_soft_reset |= SOFT_RESET_GRBM;
3871	}
3872
3873	if (reset_mask & RADEON_RESET_DMA)
3874		srbm_soft_reset |= SOFT_RESET_DMA;
3875
3876	if (reset_mask & RADEON_RESET_DMA1)
3877		srbm_soft_reset |= SOFT_RESET_DMA1;
3878
3879	if (reset_mask & RADEON_RESET_DISPLAY)
3880		srbm_soft_reset |= SOFT_RESET_DC;
3881
3882	if (reset_mask & RADEON_RESET_RLC)
3883		grbm_soft_reset |= SOFT_RESET_RLC;
3884
3885	if (reset_mask & RADEON_RESET_SEM)
3886		srbm_soft_reset |= SOFT_RESET_SEM;
3887
3888	if (reset_mask & RADEON_RESET_IH)
3889		srbm_soft_reset |= SOFT_RESET_IH;
3890
3891	if (reset_mask & RADEON_RESET_GRBM)
3892		srbm_soft_reset |= SOFT_RESET_GRBM;
3893
3894	if (reset_mask & RADEON_RESET_VMC)
3895		srbm_soft_reset |= SOFT_RESET_VMC;
3896
3897	if (reset_mask & RADEON_RESET_MC)
3898		srbm_soft_reset |= SOFT_RESET_MC;
3899
3900	if (grbm_soft_reset) {
3901		tmp = RREG32(GRBM_SOFT_RESET);
3902		tmp |= grbm_soft_reset;
3903		dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3904		WREG32(GRBM_SOFT_RESET, tmp);
3905		tmp = RREG32(GRBM_SOFT_RESET);
3906
3907		udelay(50);
3908
3909		tmp &= ~grbm_soft_reset;
3910		WREG32(GRBM_SOFT_RESET, tmp);
3911		tmp = RREG32(GRBM_SOFT_RESET);
3912	}
3913
3914	if (srbm_soft_reset) {
3915		tmp = RREG32(SRBM_SOFT_RESET);
3916		tmp |= srbm_soft_reset;
3917		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3918		WREG32(SRBM_SOFT_RESET, tmp);
3919		tmp = RREG32(SRBM_SOFT_RESET);
3920
3921		udelay(50);
3922
3923		tmp &= ~srbm_soft_reset;
3924		WREG32(SRBM_SOFT_RESET, tmp);
3925		tmp = RREG32(SRBM_SOFT_RESET);
3926	}
3927
3928	/* Wait a little for things to settle down */
3929	udelay(50);
3930
3931	evergreen_mc_resume(rdev, &save);
3932	udelay(50);
3933
3934	evergreen_print_gpu_status_regs(rdev);
3935}
3936
3937static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3938{
3939	u32 tmp, i;
3940
3941	tmp = RREG32(CG_SPLL_FUNC_CNTL);
3942	tmp |= SPLL_BYPASS_EN;
3943	WREG32(CG_SPLL_FUNC_CNTL, tmp);
3944
3945	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3946	tmp |= SPLL_CTLREQ_CHG;
3947	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3948
3949	for (i = 0; i < rdev->usec_timeout; i++) {
3950		if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3951			break;
3952		udelay(1);
3953	}
3954
3955	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3956	tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3957	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3958
3959	tmp = RREG32(MPLL_CNTL_MODE);
3960	tmp &= ~MPLL_MCLK_SEL;
3961	WREG32(MPLL_CNTL_MODE, tmp);
3962}
3963
3964static void si_spll_powerdown(struct radeon_device *rdev)
3965{
3966	u32 tmp;
3967
3968	tmp = RREG32(SPLL_CNTL_MODE);
3969	tmp |= SPLL_SW_DIR_CONTROL;
3970	WREG32(SPLL_CNTL_MODE, tmp);
3971
3972	tmp = RREG32(CG_SPLL_FUNC_CNTL);
3973	tmp |= SPLL_RESET;
3974	WREG32(CG_SPLL_FUNC_CNTL, tmp);
3975
3976	tmp = RREG32(CG_SPLL_FUNC_CNTL);
3977	tmp |= SPLL_SLEEP;
3978	WREG32(CG_SPLL_FUNC_CNTL, tmp);
3979
3980	tmp = RREG32(SPLL_CNTL_MODE);
3981	tmp &= ~SPLL_SW_DIR_CONTROL;
3982	WREG32(SPLL_CNTL_MODE, tmp);
3983}
3984
3985static void si_gpu_pci_config_reset(struct radeon_device *rdev)
3986{
3987	struct evergreen_mc_save save;
3988	u32 tmp, i;
3989
3990	dev_info(rdev->dev, "GPU pci config reset\n");
3991
3992	/* disable dpm? */
3993
3994	/* disable cg/pg */
3995	si_fini_pg(rdev);
3996	si_fini_cg(rdev);
3997
3998	/* Disable CP parsing/prefetching */
3999	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4000	/* dma0 */
4001	tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
4002	tmp &= ~DMA_RB_ENABLE;
4003	WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
4004	/* dma1 */
4005	tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
4006	tmp &= ~DMA_RB_ENABLE;
4007	WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
4008	/* XXX other engines? */
4009
4010	/* halt the rlc, disable cp internal ints */
4011	si_rlc_stop(rdev);
4012
4013	udelay(50);
4014
4015	/* disable mem access */
4016	evergreen_mc_stop(rdev, &save);
4017	if (evergreen_mc_wait_for_idle(rdev)) {
4018		dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4019	}
4020
4021	/* set mclk/sclk to bypass */
4022	si_set_clk_bypass_mode(rdev);
4023	/* powerdown spll */
4024	si_spll_powerdown(rdev);
4025	/* disable BM */
4026	pci_clear_master(rdev->pdev);
4027	/* reset */
4028	radeon_pci_config_reset(rdev);
4029	/* wait for asic to come out of reset */
4030	for (i = 0; i < rdev->usec_timeout; i++) {
4031		if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4032			break;
4033		udelay(1);
4034	}
4035}
4036
4037int si_asic_reset(struct radeon_device *rdev)
4038{
4039	u32 reset_mask;
4040
 
 
 
 
 
4041	reset_mask = si_gpu_check_soft_reset(rdev);
4042
4043	if (reset_mask)
4044		r600_set_bios_scratch_engine_hung(rdev, true);
4045
4046	/* try soft reset */
4047	si_gpu_soft_reset(rdev, reset_mask);
4048
4049	reset_mask = si_gpu_check_soft_reset(rdev);
4050
4051	/* try pci config reset */
4052	if (reset_mask && radeon_hard_reset)
4053		si_gpu_pci_config_reset(rdev);
4054
4055	reset_mask = si_gpu_check_soft_reset(rdev);
4056
4057	if (!reset_mask)
4058		r600_set_bios_scratch_engine_hung(rdev, false);
4059
4060	return 0;
4061}
4062
4063/**
4064 * si_gfx_is_lockup - Check if the GFX engine is locked up
4065 *
4066 * @rdev: radeon_device pointer
4067 * @ring: radeon_ring structure holding ring information
4068 *
4069 * Check if the GFX engine is locked up.
4070 * Returns true if the engine appears to be locked up, false if not.
4071 */
4072bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4073{
4074	u32 reset_mask = si_gpu_check_soft_reset(rdev);
4075
4076	if (!(reset_mask & (RADEON_RESET_GFX |
4077			    RADEON_RESET_COMPUTE |
4078			    RADEON_RESET_CP))) {
4079		radeon_ring_lockup_update(rdev, ring);
4080		return false;
4081	}
4082	return radeon_ring_test_lockup(rdev, ring);
4083}
4084
4085/* MC */
4086static void si_mc_program(struct radeon_device *rdev)
4087{
4088	struct evergreen_mc_save save;
4089	u32 tmp;
4090	int i, j;
4091
4092	/* Initialize HDP */
4093	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4094		WREG32((0x2c14 + j), 0x00000000);
4095		WREG32((0x2c18 + j), 0x00000000);
4096		WREG32((0x2c1c + j), 0x00000000);
4097		WREG32((0x2c20 + j), 0x00000000);
4098		WREG32((0x2c24 + j), 0x00000000);
4099	}
4100	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4101
4102	evergreen_mc_stop(rdev, &save);
4103	if (radeon_mc_wait_for_idle(rdev)) {
4104		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4105	}
4106	if (!ASIC_IS_NODCE(rdev))
4107		/* Lockout access through VGA aperture*/
4108		WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4109	/* Update configuration */
4110	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4111	       rdev->mc.vram_start >> 12);
4112	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4113	       rdev->mc.vram_end >> 12);
4114	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4115	       rdev->vram_scratch.gpu_addr >> 12);
4116	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4117	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4118	WREG32(MC_VM_FB_LOCATION, tmp);
4119	/* XXX double check these! */
4120	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4121	WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4122	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4123	WREG32(MC_VM_AGP_BASE, 0);
4124	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4125	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4126	if (radeon_mc_wait_for_idle(rdev)) {
4127		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4128	}
4129	evergreen_mc_resume(rdev, &save);
4130	if (!ASIC_IS_NODCE(rdev)) {
4131		/* we need to own VRAM, so turn off the VGA renderer here
4132		 * to stop it overwriting our objects */
4133		rv515_vga_render_disable(rdev);
4134	}
4135}
4136
4137void si_vram_gtt_location(struct radeon_device *rdev,
4138			  struct radeon_mc *mc)
4139{
4140	if (mc->mc_vram_size > 0xFFC0000000ULL) {
4141		/* leave room for at least 1024M GTT */
4142		dev_warn(rdev->dev, "limiting VRAM\n");
4143		mc->real_vram_size = 0xFFC0000000ULL;
4144		mc->mc_vram_size = 0xFFC0000000ULL;
4145	}
4146	radeon_vram_location(rdev, &rdev->mc, 0);
4147	rdev->mc.gtt_base_align = 0;
4148	radeon_gtt_location(rdev, mc);
4149}
4150
4151static int si_mc_init(struct radeon_device *rdev)
4152{
4153	u32 tmp;
4154	int chansize, numchan;
4155
4156	/* Get VRAM informations */
4157	rdev->mc.vram_is_ddr = true;
4158	tmp = RREG32(MC_ARB_RAMCFG);
4159	if (tmp & CHANSIZE_OVERRIDE) {
4160		chansize = 16;
4161	} else if (tmp & CHANSIZE_MASK) {
4162		chansize = 64;
4163	} else {
4164		chansize = 32;
4165	}
4166	tmp = RREG32(MC_SHARED_CHMAP);
4167	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4168	case 0:
4169	default:
4170		numchan = 1;
4171		break;
4172	case 1:
4173		numchan = 2;
4174		break;
4175	case 2:
4176		numchan = 4;
4177		break;
4178	case 3:
4179		numchan = 8;
4180		break;
4181	case 4:
4182		numchan = 3;
4183		break;
4184	case 5:
4185		numchan = 6;
4186		break;
4187	case 6:
4188		numchan = 10;
4189		break;
4190	case 7:
4191		numchan = 12;
4192		break;
4193	case 8:
4194		numchan = 16;
4195		break;
4196	}
4197	rdev->mc.vram_width = numchan * chansize;
4198	/* Could aper size report 0 ? */
4199	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4200	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4201	/* size in MB on si */
4202	tmp = RREG32(CONFIG_MEMSIZE);
4203	/* some boards may have garbage in the upper 16 bits */
4204	if (tmp & 0xffff0000) {
4205		DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4206		if (tmp & 0xffff)
4207			tmp &= 0xffff;
4208	}
4209	rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4210	rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4211	rdev->mc.visible_vram_size = rdev->mc.aper_size;
4212	si_vram_gtt_location(rdev, &rdev->mc);
4213	radeon_update_bandwidth_info(rdev);
4214
4215	return 0;
4216}
4217
4218/*
4219 * GART
4220 */
4221void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4222{
4223	/* flush hdp cache */
4224	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4225
4226	/* bits 0-15 are the VM contexts0-15 */
4227	WREG32(VM_INVALIDATE_REQUEST, 1);
4228}
4229
4230static int si_pcie_gart_enable(struct radeon_device *rdev)
4231{
4232	int r, i;
4233
4234	if (rdev->gart.robj == NULL) {
4235		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4236		return -EINVAL;
4237	}
4238	r = radeon_gart_table_vram_pin(rdev);
4239	if (r)
4240		return r;
4241	/* Setup TLB control */
4242	WREG32(MC_VM_MX_L1_TLB_CNTL,
4243	       (0xA << 7) |
4244	       ENABLE_L1_TLB |
4245	       ENABLE_L1_FRAGMENT_PROCESSING |
4246	       SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4247	       ENABLE_ADVANCED_DRIVER_MODEL |
4248	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4249	/* Setup L2 cache */
4250	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4251	       ENABLE_L2_FRAGMENT_PROCESSING |
4252	       ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4253	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4254	       EFFECTIVE_L2_QUEUE_SIZE(7) |
4255	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
4256	WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4257	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4258	       BANK_SELECT(4) |
4259	       L2_CACHE_BIGK_FRAGMENT_SIZE(4));
4260	/* setup context0 */
4261	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4262	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4263	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4264	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4265			(u32)(rdev->dummy_page.addr >> 12));
4266	WREG32(VM_CONTEXT0_CNTL2, 0);
4267	WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4268				  RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4269
4270	WREG32(0x15D4, 0);
4271	WREG32(0x15D8, 0);
4272	WREG32(0x15DC, 0);
4273
4274	/* empty context1-15 */
4275	/* set vm size, must be a multiple of 4 */
4276	WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4277	WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
4278	/* Assign the pt base to something valid for now; the pts used for
4279	 * the VMs are determined by the application and setup and assigned
4280	 * on the fly in the vm part of radeon_gart.c
4281	 */
4282	for (i = 1; i < 16; i++) {
4283		if (i < 8)
4284			WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4285			       rdev->vm_manager.saved_table_addr[i]);
4286		else
4287			WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4288			       rdev->vm_manager.saved_table_addr[i]);
4289	}
4290
4291	/* enable context1-15 */
4292	WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4293	       (u32)(rdev->dummy_page.addr >> 12));
4294	WREG32(VM_CONTEXT1_CNTL2, 4);
4295	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4296				PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
4297				RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4298				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4299				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4300				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4301				PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4302				PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4303				VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4304				VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4305				READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4306				READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4307				WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4308				WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4309
4310	si_pcie_gart_tlb_flush(rdev);
4311	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4312		 (unsigned)(rdev->mc.gtt_size >> 20),
4313		 (unsigned long long)rdev->gart.table_addr);
4314	rdev->gart.ready = true;
4315	return 0;
4316}
4317
4318static void si_pcie_gart_disable(struct radeon_device *rdev)
4319{
4320	unsigned i;
4321
4322	for (i = 1; i < 16; ++i) {
4323		uint32_t reg;
4324		if (i < 8)
4325			reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4326		else
4327			reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4328		rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4329	}
4330
4331	/* Disable all tables */
4332	WREG32(VM_CONTEXT0_CNTL, 0);
4333	WREG32(VM_CONTEXT1_CNTL, 0);
4334	/* Setup TLB control */
4335	WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4336	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4337	/* Setup L2 cache */
4338	WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4339	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4340	       EFFECTIVE_L2_QUEUE_SIZE(7) |
4341	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
4342	WREG32(VM_L2_CNTL2, 0);
4343	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4344	       L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4345	radeon_gart_table_vram_unpin(rdev);
4346}
4347
4348static void si_pcie_gart_fini(struct radeon_device *rdev)
4349{
4350	si_pcie_gart_disable(rdev);
4351	radeon_gart_table_vram_free(rdev);
4352	radeon_gart_fini(rdev);
4353}
4354
4355/* vm parser */
4356static bool si_vm_reg_valid(u32 reg)
4357{
4358	/* context regs are fine */
4359	if (reg >= 0x28000)
4360		return true;
4361
 
 
 
 
4362	/* check config regs */
4363	switch (reg) {
4364	case GRBM_GFX_INDEX:
4365	case CP_STRMOUT_CNTL:
4366	case VGT_VTX_VECT_EJECT_REG:
4367	case VGT_CACHE_INVALIDATION:
4368	case VGT_ESGS_RING_SIZE:
4369	case VGT_GSVS_RING_SIZE:
4370	case VGT_GS_VERTEX_REUSE:
4371	case VGT_PRIMITIVE_TYPE:
4372	case VGT_INDEX_TYPE:
4373	case VGT_NUM_INDICES:
4374	case VGT_NUM_INSTANCES:
4375	case VGT_TF_RING_SIZE:
4376	case VGT_HS_OFFCHIP_PARAM:
4377	case VGT_TF_MEMORY_BASE:
4378	case PA_CL_ENHANCE:
4379	case PA_SU_LINE_STIPPLE_VALUE:
4380	case PA_SC_LINE_STIPPLE_STATE:
4381	case PA_SC_ENHANCE:
4382	case SQC_CACHES:
4383	case SPI_STATIC_THREAD_MGMT_1:
4384	case SPI_STATIC_THREAD_MGMT_2:
4385	case SPI_STATIC_THREAD_MGMT_3:
4386	case SPI_PS_MAX_WAVE_ID:
4387	case SPI_CONFIG_CNTL:
4388	case SPI_CONFIG_CNTL_1:
4389	case TA_CNTL_AUX:
 
4390		return true;
4391	default:
4392		DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4393		return false;
4394	}
4395}
4396
4397static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4398				  u32 *ib, struct radeon_cs_packet *pkt)
4399{
4400	switch (pkt->opcode) {
4401	case PACKET3_NOP:
4402	case PACKET3_SET_BASE:
4403	case PACKET3_SET_CE_DE_COUNTERS:
4404	case PACKET3_LOAD_CONST_RAM:
4405	case PACKET3_WRITE_CONST_RAM:
4406	case PACKET3_WRITE_CONST_RAM_OFFSET:
4407	case PACKET3_DUMP_CONST_RAM:
4408	case PACKET3_INCREMENT_CE_COUNTER:
4409	case PACKET3_WAIT_ON_DE_COUNTER:
4410	case PACKET3_CE_WRITE:
4411		break;
4412	default:
4413		DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4414		return -EINVAL;
4415	}
4416	return 0;
4417}
4418
4419static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4420{
4421	u32 start_reg, reg, i;
4422	u32 command = ib[idx + 4];
4423	u32 info = ib[idx + 1];
4424	u32 idx_value = ib[idx];
4425	if (command & PACKET3_CP_DMA_CMD_SAS) {
4426		/* src address space is register */
4427		if (((info & 0x60000000) >> 29) == 0) {
4428			start_reg = idx_value << 2;
4429			if (command & PACKET3_CP_DMA_CMD_SAIC) {
4430				reg = start_reg;
4431				if (!si_vm_reg_valid(reg)) {
4432					DRM_ERROR("CP DMA Bad SRC register\n");
4433					return -EINVAL;
4434				}
4435			} else {
4436				for (i = 0; i < (command & 0x1fffff); i++) {
4437					reg = start_reg + (4 * i);
4438					if (!si_vm_reg_valid(reg)) {
4439						DRM_ERROR("CP DMA Bad SRC register\n");
4440						return -EINVAL;
4441					}
4442				}
4443			}
4444		}
4445	}
4446	if (command & PACKET3_CP_DMA_CMD_DAS) {
4447		/* dst address space is register */
4448		if (((info & 0x00300000) >> 20) == 0) {
4449			start_reg = ib[idx + 2];
4450			if (command & PACKET3_CP_DMA_CMD_DAIC) {
4451				reg = start_reg;
4452				if (!si_vm_reg_valid(reg)) {
4453					DRM_ERROR("CP DMA Bad DST register\n");
4454					return -EINVAL;
4455				}
4456			} else {
4457				for (i = 0; i < (command & 0x1fffff); i++) {
4458					reg = start_reg + (4 * i);
4459				if (!si_vm_reg_valid(reg)) {
4460						DRM_ERROR("CP DMA Bad DST register\n");
4461						return -EINVAL;
4462					}
4463				}
4464			}
4465		}
4466	}
4467	return 0;
4468}
4469
4470static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4471				   u32 *ib, struct radeon_cs_packet *pkt)
4472{
4473	int r;
4474	u32 idx = pkt->idx + 1;
4475	u32 idx_value = ib[idx];
4476	u32 start_reg, end_reg, reg, i;
4477
4478	switch (pkt->opcode) {
4479	case PACKET3_NOP:
4480	case PACKET3_SET_BASE:
4481	case PACKET3_CLEAR_STATE:
4482	case PACKET3_INDEX_BUFFER_SIZE:
4483	case PACKET3_DISPATCH_DIRECT:
4484	case PACKET3_DISPATCH_INDIRECT:
4485	case PACKET3_ALLOC_GDS:
4486	case PACKET3_WRITE_GDS_RAM:
4487	case PACKET3_ATOMIC_GDS:
4488	case PACKET3_ATOMIC:
4489	case PACKET3_OCCLUSION_QUERY:
4490	case PACKET3_SET_PREDICATION:
4491	case PACKET3_COND_EXEC:
4492	case PACKET3_PRED_EXEC:
4493	case PACKET3_DRAW_INDIRECT:
4494	case PACKET3_DRAW_INDEX_INDIRECT:
4495	case PACKET3_INDEX_BASE:
4496	case PACKET3_DRAW_INDEX_2:
4497	case PACKET3_CONTEXT_CONTROL:
4498	case PACKET3_INDEX_TYPE:
4499	case PACKET3_DRAW_INDIRECT_MULTI:
4500	case PACKET3_DRAW_INDEX_AUTO:
4501	case PACKET3_DRAW_INDEX_IMMD:
4502	case PACKET3_NUM_INSTANCES:
4503	case PACKET3_DRAW_INDEX_MULTI_AUTO:
4504	case PACKET3_STRMOUT_BUFFER_UPDATE:
4505	case PACKET3_DRAW_INDEX_OFFSET_2:
4506	case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4507	case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4508	case PACKET3_MPEG_INDEX:
4509	case PACKET3_WAIT_REG_MEM:
4510	case PACKET3_MEM_WRITE:
4511	case PACKET3_PFP_SYNC_ME:
4512	case PACKET3_SURFACE_SYNC:
4513	case PACKET3_EVENT_WRITE:
4514	case PACKET3_EVENT_WRITE_EOP:
4515	case PACKET3_EVENT_WRITE_EOS:
4516	case PACKET3_SET_CONTEXT_REG:
4517	case PACKET3_SET_CONTEXT_REG_INDIRECT:
4518	case PACKET3_SET_SH_REG:
4519	case PACKET3_SET_SH_REG_OFFSET:
4520	case PACKET3_INCREMENT_DE_COUNTER:
4521	case PACKET3_WAIT_ON_CE_COUNTER:
4522	case PACKET3_WAIT_ON_AVAIL_BUFFER:
4523	case PACKET3_ME_WRITE:
4524		break;
4525	case PACKET3_COPY_DATA:
4526		if ((idx_value & 0xf00) == 0) {
4527			reg = ib[idx + 3] * 4;
4528			if (!si_vm_reg_valid(reg))
4529				return -EINVAL;
4530		}
4531		break;
4532	case PACKET3_WRITE_DATA:
4533		if ((idx_value & 0xf00) == 0) {
4534			start_reg = ib[idx + 1] * 4;
4535			if (idx_value & 0x10000) {
4536				if (!si_vm_reg_valid(start_reg))
4537					return -EINVAL;
4538			} else {
4539				for (i = 0; i < (pkt->count - 2); i++) {
4540					reg = start_reg + (4 * i);
4541					if (!si_vm_reg_valid(reg))
4542						return -EINVAL;
4543				}
4544			}
4545		}
4546		break;
4547	case PACKET3_COND_WRITE:
4548		if (idx_value & 0x100) {
4549			reg = ib[idx + 5] * 4;
4550			if (!si_vm_reg_valid(reg))
4551				return -EINVAL;
4552		}
4553		break;
4554	case PACKET3_COPY_DW:
4555		if (idx_value & 0x2) {
4556			reg = ib[idx + 3] * 4;
4557			if (!si_vm_reg_valid(reg))
4558				return -EINVAL;
4559		}
4560		break;
4561	case PACKET3_SET_CONFIG_REG:
4562		start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4563		end_reg = 4 * pkt->count + start_reg - 4;
4564		if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4565		    (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4566		    (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4567			DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4568			return -EINVAL;
4569		}
4570		for (i = 0; i < pkt->count; i++) {
4571			reg = start_reg + (4 * i);
4572			if (!si_vm_reg_valid(reg))
4573				return -EINVAL;
4574		}
4575		break;
4576	case PACKET3_CP_DMA:
4577		r = si_vm_packet3_cp_dma_check(ib, idx);
4578		if (r)
4579			return r;
4580		break;
4581	default:
4582		DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4583		return -EINVAL;
4584	}
4585	return 0;
4586}
4587
4588static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4589				       u32 *ib, struct radeon_cs_packet *pkt)
4590{
4591	int r;
4592	u32 idx = pkt->idx + 1;
4593	u32 idx_value = ib[idx];
4594	u32 start_reg, reg, i;
4595
4596	switch (pkt->opcode) {
4597	case PACKET3_NOP:
4598	case PACKET3_SET_BASE:
4599	case PACKET3_CLEAR_STATE:
4600	case PACKET3_DISPATCH_DIRECT:
4601	case PACKET3_DISPATCH_INDIRECT:
4602	case PACKET3_ALLOC_GDS:
4603	case PACKET3_WRITE_GDS_RAM:
4604	case PACKET3_ATOMIC_GDS:
4605	case PACKET3_ATOMIC:
4606	case PACKET3_OCCLUSION_QUERY:
4607	case PACKET3_SET_PREDICATION:
4608	case PACKET3_COND_EXEC:
4609	case PACKET3_PRED_EXEC:
4610	case PACKET3_CONTEXT_CONTROL:
4611	case PACKET3_STRMOUT_BUFFER_UPDATE:
4612	case PACKET3_WAIT_REG_MEM:
4613	case PACKET3_MEM_WRITE:
4614	case PACKET3_PFP_SYNC_ME:
4615	case PACKET3_SURFACE_SYNC:
4616	case PACKET3_EVENT_WRITE:
4617	case PACKET3_EVENT_WRITE_EOP:
4618	case PACKET3_EVENT_WRITE_EOS:
4619	case PACKET3_SET_CONTEXT_REG:
4620	case PACKET3_SET_CONTEXT_REG_INDIRECT:
4621	case PACKET3_SET_SH_REG:
4622	case PACKET3_SET_SH_REG_OFFSET:
4623	case PACKET3_INCREMENT_DE_COUNTER:
4624	case PACKET3_WAIT_ON_CE_COUNTER:
4625	case PACKET3_WAIT_ON_AVAIL_BUFFER:
4626	case PACKET3_ME_WRITE:
4627		break;
4628	case PACKET3_COPY_DATA:
4629		if ((idx_value & 0xf00) == 0) {
4630			reg = ib[idx + 3] * 4;
4631			if (!si_vm_reg_valid(reg))
4632				return -EINVAL;
4633		}
4634		break;
4635	case PACKET3_WRITE_DATA:
4636		if ((idx_value & 0xf00) == 0) {
4637			start_reg = ib[idx + 1] * 4;
4638			if (idx_value & 0x10000) {
4639				if (!si_vm_reg_valid(start_reg))
4640					return -EINVAL;
4641			} else {
4642				for (i = 0; i < (pkt->count - 2); i++) {
4643					reg = start_reg + (4 * i);
4644					if (!si_vm_reg_valid(reg))
4645						return -EINVAL;
4646				}
4647			}
4648		}
4649		break;
4650	case PACKET3_COND_WRITE:
4651		if (idx_value & 0x100) {
4652			reg = ib[idx + 5] * 4;
4653			if (!si_vm_reg_valid(reg))
4654				return -EINVAL;
4655		}
4656		break;
4657	case PACKET3_COPY_DW:
4658		if (idx_value & 0x2) {
4659			reg = ib[idx + 3] * 4;
4660			if (!si_vm_reg_valid(reg))
4661				return -EINVAL;
4662		}
4663		break;
4664	case PACKET3_CP_DMA:
4665		r = si_vm_packet3_cp_dma_check(ib, idx);
4666		if (r)
4667			return r;
4668		break;
4669	default:
4670		DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4671		return -EINVAL;
4672	}
4673	return 0;
4674}
4675
4676int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4677{
4678	int ret = 0;
4679	u32 idx = 0, i;
4680	struct radeon_cs_packet pkt;
4681
4682	do {
4683		pkt.idx = idx;
4684		pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4685		pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4686		pkt.one_reg_wr = 0;
4687		switch (pkt.type) {
4688		case RADEON_PACKET_TYPE0:
4689			dev_err(rdev->dev, "Packet0 not allowed!\n");
4690			ret = -EINVAL;
4691			break;
4692		case RADEON_PACKET_TYPE2:
4693			idx += 1;
4694			break;
4695		case RADEON_PACKET_TYPE3:
4696			pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4697			if (ib->is_const_ib)
4698				ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4699			else {
4700				switch (ib->ring) {
4701				case RADEON_RING_TYPE_GFX_INDEX:
4702					ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4703					break;
4704				case CAYMAN_RING_TYPE_CP1_INDEX:
4705				case CAYMAN_RING_TYPE_CP2_INDEX:
4706					ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4707					break;
4708				default:
4709					dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4710					ret = -EINVAL;
4711					break;
4712				}
4713			}
4714			idx += pkt.count + 2;
4715			break;
4716		default:
4717			dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4718			ret = -EINVAL;
4719			break;
4720		}
4721		if (ret) {
4722			for (i = 0; i < ib->length_dw; i++) {
4723				if (i == idx)
4724					printk("\t0x%08x <---\n", ib->ptr[i]);
4725				else
4726					printk("\t0x%08x\n", ib->ptr[i]);
4727			}
4728			break;
4729		}
4730	} while (idx < ib->length_dw);
4731
4732	return ret;
4733}
4734
4735/*
4736 * vm
4737 */
4738int si_vm_init(struct radeon_device *rdev)
4739{
4740	/* number of VMs */
4741	rdev->vm_manager.nvm = 16;
4742	/* base offset of vram pages */
4743	rdev->vm_manager.vram_base_offset = 0;
4744
4745	return 0;
4746}
4747
4748void si_vm_fini(struct radeon_device *rdev)
4749{
4750}
4751
4752/**
4753 * si_vm_decode_fault - print human readable fault info
4754 *
4755 * @rdev: radeon_device pointer
4756 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4757 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4758 *
4759 * Print human readable fault information (SI).
4760 */
4761static void si_vm_decode_fault(struct radeon_device *rdev,
4762			       u32 status, u32 addr)
4763{
4764	u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4765	u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4766	u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4767	char *block;
4768
4769	if (rdev->family == CHIP_TAHITI) {
4770		switch (mc_id) {
4771		case 160:
4772		case 144:
4773		case 96:
4774		case 80:
4775		case 224:
4776		case 208:
4777		case 32:
4778		case 16:
4779			block = "CB";
4780			break;
4781		case 161:
4782		case 145:
4783		case 97:
4784		case 81:
4785		case 225:
4786		case 209:
4787		case 33:
4788		case 17:
4789			block = "CB_FMASK";
4790			break;
4791		case 162:
4792		case 146:
4793		case 98:
4794		case 82:
4795		case 226:
4796		case 210:
4797		case 34:
4798		case 18:
4799			block = "CB_CMASK";
4800			break;
4801		case 163:
4802		case 147:
4803		case 99:
4804		case 83:
4805		case 227:
4806		case 211:
4807		case 35:
4808		case 19:
4809			block = "CB_IMMED";
4810			break;
4811		case 164:
4812		case 148:
4813		case 100:
4814		case 84:
4815		case 228:
4816		case 212:
4817		case 36:
4818		case 20:
4819			block = "DB";
4820			break;
4821		case 165:
4822		case 149:
4823		case 101:
4824		case 85:
4825		case 229:
4826		case 213:
4827		case 37:
4828		case 21:
4829			block = "DB_HTILE";
4830			break;
4831		case 167:
4832		case 151:
4833		case 103:
4834		case 87:
4835		case 231:
4836		case 215:
4837		case 39:
4838		case 23:
4839			block = "DB_STEN";
4840			break;
4841		case 72:
4842		case 68:
4843		case 64:
4844		case 8:
4845		case 4:
4846		case 0:
4847		case 136:
4848		case 132:
4849		case 128:
4850		case 200:
4851		case 196:
4852		case 192:
4853			block = "TC";
4854			break;
4855		case 112:
4856		case 48:
4857			block = "CP";
4858			break;
4859		case 49:
4860		case 177:
4861		case 50:
4862		case 178:
4863			block = "SH";
4864			break;
4865		case 53:
4866		case 190:
4867			block = "VGT";
4868			break;
4869		case 117:
4870			block = "IH";
4871			break;
4872		case 51:
4873		case 115:
4874			block = "RLC";
4875			break;
4876		case 119:
4877		case 183:
4878			block = "DMA0";
4879			break;
4880		case 61:
4881			block = "DMA1";
4882			break;
4883		case 248:
4884		case 120:
4885			block = "HDP";
4886			break;
4887		default:
4888			block = "unknown";
4889			break;
4890		}
4891	} else {
4892		switch (mc_id) {
4893		case 32:
4894		case 16:
4895		case 96:
4896		case 80:
4897		case 160:
4898		case 144:
4899		case 224:
4900		case 208:
4901			block = "CB";
4902			break;
4903		case 33:
4904		case 17:
4905		case 97:
4906		case 81:
4907		case 161:
4908		case 145:
4909		case 225:
4910		case 209:
4911			block = "CB_FMASK";
4912			break;
4913		case 34:
4914		case 18:
4915		case 98:
4916		case 82:
4917		case 162:
4918		case 146:
4919		case 226:
4920		case 210:
4921			block = "CB_CMASK";
4922			break;
4923		case 35:
4924		case 19:
4925		case 99:
4926		case 83:
4927		case 163:
4928		case 147:
4929		case 227:
4930		case 211:
4931			block = "CB_IMMED";
4932			break;
4933		case 36:
4934		case 20:
4935		case 100:
4936		case 84:
4937		case 164:
4938		case 148:
4939		case 228:
4940		case 212:
4941			block = "DB";
4942			break;
4943		case 37:
4944		case 21:
4945		case 101:
4946		case 85:
4947		case 165:
4948		case 149:
4949		case 229:
4950		case 213:
4951			block = "DB_HTILE";
4952			break;
4953		case 39:
4954		case 23:
4955		case 103:
4956		case 87:
4957		case 167:
4958		case 151:
4959		case 231:
4960		case 215:
4961			block = "DB_STEN";
4962			break;
4963		case 72:
4964		case 68:
4965		case 8:
4966		case 4:
4967		case 136:
4968		case 132:
4969		case 200:
4970		case 196:
4971			block = "TC";
4972			break;
4973		case 112:
4974		case 48:
4975			block = "CP";
4976			break;
4977		case 49:
4978		case 177:
4979		case 50:
4980		case 178:
4981			block = "SH";
4982			break;
4983		case 53:
4984			block = "VGT";
4985			break;
4986		case 117:
4987			block = "IH";
4988			break;
4989		case 51:
4990		case 115:
4991			block = "RLC";
4992			break;
4993		case 119:
4994		case 183:
4995			block = "DMA0";
4996			break;
4997		case 61:
4998			block = "DMA1";
4999			break;
5000		case 248:
5001		case 120:
5002			block = "HDP";
5003			break;
5004		default:
5005			block = "unknown";
5006			break;
5007		}
5008	}
5009
5010	printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
5011	       protections, vmid, addr,
5012	       (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5013	       block, mc_id);
5014}
5015
5016void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5017		 unsigned vm_id, uint64_t pd_addr)
5018{
5019	/* write new base address */
5020	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5021	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5022				 WRITE_DATA_DST_SEL(0)));
5023
5024	if (vm_id < 8) {
5025		radeon_ring_write(ring,
5026				  (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
5027	} else {
5028		radeon_ring_write(ring,
5029				  (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
5030	}
5031	radeon_ring_write(ring, 0);
5032	radeon_ring_write(ring, pd_addr >> 12);
5033
5034	/* flush hdp cache */
5035	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5036	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5037				 WRITE_DATA_DST_SEL(0)));
5038	radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5039	radeon_ring_write(ring, 0);
5040	radeon_ring_write(ring, 0x1);
5041
5042	/* bits 0-15 are the VM contexts0-15 */
5043	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5044	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5045				 WRITE_DATA_DST_SEL(0)));
5046	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5047	radeon_ring_write(ring, 0);
5048	radeon_ring_write(ring, 1 << vm_id);
5049
5050	/* wait for the invalidate to complete */
5051	radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5052	radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
5053				 WAIT_REG_MEM_ENGINE(0))); /* me */
5054	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5055	radeon_ring_write(ring, 0);
5056	radeon_ring_write(ring, 0); /* ref */
5057	radeon_ring_write(ring, 0); /* mask */
5058	radeon_ring_write(ring, 0x20); /* poll interval */
5059
5060	/* sync PFP to ME, otherwise we might get invalid PFP reads */
5061	radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5062	radeon_ring_write(ring, 0x0);
5063}
5064
5065/*
5066 *  Power and clock gating
5067 */
5068static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5069{
5070	int i;
5071
5072	for (i = 0; i < rdev->usec_timeout; i++) {
5073		if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5074			break;
5075		udelay(1);
5076	}
5077
5078	for (i = 0; i < rdev->usec_timeout; i++) {
5079		if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5080			break;
5081		udelay(1);
5082	}
5083}
5084
5085static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5086					 bool enable)
5087{
5088	u32 tmp = RREG32(CP_INT_CNTL_RING0);
5089	u32 mask;
5090	int i;
5091
5092	if (enable)
5093		tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5094	else
5095		tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5096	WREG32(CP_INT_CNTL_RING0, tmp);
5097
5098	if (!enable) {
5099		/* read a gfx register */
5100		tmp = RREG32(DB_DEPTH_INFO);
5101
5102		mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5103		for (i = 0; i < rdev->usec_timeout; i++) {
5104			if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5105				break;
5106			udelay(1);
5107		}
5108	}
5109}
5110
5111static void si_set_uvd_dcm(struct radeon_device *rdev,
5112			   bool sw_mode)
5113{
5114	u32 tmp, tmp2;
5115
5116	tmp = RREG32(UVD_CGC_CTRL);
5117	tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5118	tmp |= DCM | CG_DT(1) | CLK_OD(4);
5119
5120	if (sw_mode) {
5121		tmp &= ~0x7ffff800;
5122		tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5123	} else {
5124		tmp |= 0x7ffff800;
5125		tmp2 = 0;
5126	}
5127
5128	WREG32(UVD_CGC_CTRL, tmp);
5129	WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5130}
5131
5132void si_init_uvd_internal_cg(struct radeon_device *rdev)
5133{
5134	bool hw_mode = true;
5135
5136	if (hw_mode) {
5137		si_set_uvd_dcm(rdev, false);
5138	} else {
5139		u32 tmp = RREG32(UVD_CGC_CTRL);
5140		tmp &= ~DCM;
5141		WREG32(UVD_CGC_CTRL, tmp);
5142	}
5143}
5144
5145static u32 si_halt_rlc(struct radeon_device *rdev)
5146{
5147	u32 data, orig;
5148
5149	orig = data = RREG32(RLC_CNTL);
5150
5151	if (data & RLC_ENABLE) {
5152		data &= ~RLC_ENABLE;
5153		WREG32(RLC_CNTL, data);
5154
5155		si_wait_for_rlc_serdes(rdev);
5156	}
5157
5158	return orig;
5159}
5160
5161static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5162{
5163	u32 tmp;
5164
5165	tmp = RREG32(RLC_CNTL);
5166	if (tmp != rlc)
5167		WREG32(RLC_CNTL, rlc);
5168}
5169
5170static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5171{
5172	u32 data, orig;
5173
5174	orig = data = RREG32(DMA_PG);
5175	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
5176		data |= PG_CNTL_ENABLE;
5177	else
5178		data &= ~PG_CNTL_ENABLE;
5179	if (orig != data)
5180		WREG32(DMA_PG, data);
5181}
5182
5183static void si_init_dma_pg(struct radeon_device *rdev)
5184{
5185	u32 tmp;
5186
5187	WREG32(DMA_PGFSM_WRITE,  0x00002000);
5188	WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5189
5190	for (tmp = 0; tmp < 5; tmp++)
5191		WREG32(DMA_PGFSM_WRITE, 0);
5192}
5193
5194static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5195			       bool enable)
5196{
5197	u32 tmp;
5198
5199	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5200		tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5201		WREG32(RLC_TTOP_D, tmp);
5202
5203		tmp = RREG32(RLC_PG_CNTL);
5204		tmp |= GFX_PG_ENABLE;
5205		WREG32(RLC_PG_CNTL, tmp);
5206
5207		tmp = RREG32(RLC_AUTO_PG_CTRL);
5208		tmp |= AUTO_PG_EN;
5209		WREG32(RLC_AUTO_PG_CTRL, tmp);
5210	} else {
5211		tmp = RREG32(RLC_AUTO_PG_CTRL);
5212		tmp &= ~AUTO_PG_EN;
5213		WREG32(RLC_AUTO_PG_CTRL, tmp);
5214
5215		tmp = RREG32(DB_RENDER_CONTROL);
5216	}
5217}
5218
5219static void si_init_gfx_cgpg(struct radeon_device *rdev)
5220{
5221	u32 tmp;
5222
5223	WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5224
5225	tmp = RREG32(RLC_PG_CNTL);
5226	tmp |= GFX_PG_SRC;
5227	WREG32(RLC_PG_CNTL, tmp);
5228
5229	WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5230
5231	tmp = RREG32(RLC_AUTO_PG_CTRL);
5232
5233	tmp &= ~GRBM_REG_SGIT_MASK;
5234	tmp |= GRBM_REG_SGIT(0x700);
5235	tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5236	WREG32(RLC_AUTO_PG_CTRL, tmp);
5237}
5238
5239static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5240{
5241	u32 mask = 0, tmp, tmp1;
5242	int i;
5243
5244	si_select_se_sh(rdev, se, sh);
5245	tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5246	tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5247	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5248
5249	tmp &= 0xffff0000;
5250
5251	tmp |= tmp1;
5252	tmp >>= 16;
5253
5254	for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5255		mask <<= 1;
5256		mask |= 1;
5257	}
5258
5259	return (~tmp) & mask;
5260}
5261
5262static void si_init_ao_cu_mask(struct radeon_device *rdev)
5263{
5264	u32 i, j, k, active_cu_number = 0;
5265	u32 mask, counter, cu_bitmap;
5266	u32 tmp = 0;
5267
5268	for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5269		for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5270			mask = 1;
5271			cu_bitmap = 0;
5272			counter  = 0;
5273			for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5274				if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5275					if (counter < 2)
5276						cu_bitmap |= mask;
5277					counter++;
5278				}
5279				mask <<= 1;
5280			}
5281
5282			active_cu_number += counter;
5283			tmp |= (cu_bitmap << (i * 16 + j * 8));
5284		}
5285	}
5286
5287	WREG32(RLC_PG_AO_CU_MASK, tmp);
5288
5289	tmp = RREG32(RLC_MAX_PG_CU);
5290	tmp &= ~MAX_PU_CU_MASK;
5291	tmp |= MAX_PU_CU(active_cu_number);
5292	WREG32(RLC_MAX_PG_CU, tmp);
5293}
5294
5295static void si_enable_cgcg(struct radeon_device *rdev,
5296			   bool enable)
5297{
5298	u32 data, orig, tmp;
5299
5300	orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5301
5302	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5303		si_enable_gui_idle_interrupt(rdev, true);
5304
5305		WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5306
5307		tmp = si_halt_rlc(rdev);
5308
5309		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5310		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5311		WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5312
5313		si_wait_for_rlc_serdes(rdev);
5314
5315		si_update_rlc(rdev, tmp);
5316
5317		WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5318
5319		data |= CGCG_EN | CGLS_EN;
5320	} else {
5321		si_enable_gui_idle_interrupt(rdev, false);
5322
5323		RREG32(CB_CGTT_SCLK_CTRL);
5324		RREG32(CB_CGTT_SCLK_CTRL);
5325		RREG32(CB_CGTT_SCLK_CTRL);
5326		RREG32(CB_CGTT_SCLK_CTRL);
5327
5328		data &= ~(CGCG_EN | CGLS_EN);
5329	}
5330
5331	if (orig != data)
5332		WREG32(RLC_CGCG_CGLS_CTRL, data);
5333}
5334
5335static void si_enable_mgcg(struct radeon_device *rdev,
5336			   bool enable)
5337{
5338	u32 data, orig, tmp = 0;
5339
5340	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5341		orig = data = RREG32(CGTS_SM_CTRL_REG);
5342		data = 0x96940200;
5343		if (orig != data)
5344			WREG32(CGTS_SM_CTRL_REG, data);
5345
5346		if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5347			orig = data = RREG32(CP_MEM_SLP_CNTL);
5348			data |= CP_MEM_LS_EN;
5349			if (orig != data)
5350				WREG32(CP_MEM_SLP_CNTL, data);
5351		}
5352
5353		orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5354		data &= 0xffffffc0;
5355		if (orig != data)
5356			WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5357
5358		tmp = si_halt_rlc(rdev);
5359
5360		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5361		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5362		WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5363
5364		si_update_rlc(rdev, tmp);
5365	} else {
5366		orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5367		data |= 0x00000003;
5368		if (orig != data)
5369			WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5370
5371		data = RREG32(CP_MEM_SLP_CNTL);
5372		if (data & CP_MEM_LS_EN) {
5373			data &= ~CP_MEM_LS_EN;
5374			WREG32(CP_MEM_SLP_CNTL, data);
5375		}
5376		orig = data = RREG32(CGTS_SM_CTRL_REG);
5377		data |= LS_OVERRIDE | OVERRIDE;
5378		if (orig != data)
5379			WREG32(CGTS_SM_CTRL_REG, data);
5380
5381		tmp = si_halt_rlc(rdev);
5382
5383		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5384		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5385		WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5386
5387		si_update_rlc(rdev, tmp);
5388	}
5389}
5390
5391static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5392			       bool enable)
5393{
5394	u32 orig, data, tmp;
5395
5396	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5397		tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5398		tmp |= 0x3fff;
5399		WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5400
5401		orig = data = RREG32(UVD_CGC_CTRL);
5402		data |= DCM;
5403		if (orig != data)
5404			WREG32(UVD_CGC_CTRL, data);
5405
5406		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5407		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5408	} else {
5409		tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5410		tmp &= ~0x3fff;
5411		WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5412
5413		orig = data = RREG32(UVD_CGC_CTRL);
5414		data &= ~DCM;
5415		if (orig != data)
5416			WREG32(UVD_CGC_CTRL, data);
5417
5418		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5419		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5420	}
5421}
5422
5423static const u32 mc_cg_registers[] =
5424{
5425	MC_HUB_MISC_HUB_CG,
5426	MC_HUB_MISC_SIP_CG,
5427	MC_HUB_MISC_VM_CG,
5428	MC_XPB_CLK_GAT,
5429	ATC_MISC_CG,
5430	MC_CITF_MISC_WR_CG,
5431	MC_CITF_MISC_RD_CG,
5432	MC_CITF_MISC_VM_CG,
5433	VM_L2_CG,
5434};
5435
5436static void si_enable_mc_ls(struct radeon_device *rdev,
5437			    bool enable)
5438{
5439	int i;
5440	u32 orig, data;
5441
5442	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5443		orig = data = RREG32(mc_cg_registers[i]);
5444		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5445			data |= MC_LS_ENABLE;
5446		else
5447			data &= ~MC_LS_ENABLE;
5448		if (data != orig)
5449			WREG32(mc_cg_registers[i], data);
5450	}
5451}
5452
5453static void si_enable_mc_mgcg(struct radeon_device *rdev,
5454			       bool enable)
5455{
5456	int i;
5457	u32 orig, data;
5458
5459	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5460		orig = data = RREG32(mc_cg_registers[i]);
5461		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5462			data |= MC_CG_ENABLE;
5463		else
5464			data &= ~MC_CG_ENABLE;
5465		if (data != orig)
5466			WREG32(mc_cg_registers[i], data);
5467	}
5468}
5469
5470static void si_enable_dma_mgcg(struct radeon_device *rdev,
5471			       bool enable)
5472{
5473	u32 orig, data, offset;
5474	int i;
5475
5476	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5477		for (i = 0; i < 2; i++) {
5478			if (i == 0)
5479				offset = DMA0_REGISTER_OFFSET;
5480			else
5481				offset = DMA1_REGISTER_OFFSET;
5482			orig = data = RREG32(DMA_POWER_CNTL + offset);
5483			data &= ~MEM_POWER_OVERRIDE;
5484			if (data != orig)
5485				WREG32(DMA_POWER_CNTL + offset, data);
5486			WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5487		}
5488	} else {
5489		for (i = 0; i < 2; i++) {
5490			if (i == 0)
5491				offset = DMA0_REGISTER_OFFSET;
5492			else
5493				offset = DMA1_REGISTER_OFFSET;
5494			orig = data = RREG32(DMA_POWER_CNTL + offset);
5495			data |= MEM_POWER_OVERRIDE;
5496			if (data != orig)
5497				WREG32(DMA_POWER_CNTL + offset, data);
5498
5499			orig = data = RREG32(DMA_CLK_CTRL + offset);
5500			data = 0xff000000;
5501			if (data != orig)
5502				WREG32(DMA_CLK_CTRL + offset, data);
5503		}
5504	}
5505}
5506
5507static void si_enable_bif_mgls(struct radeon_device *rdev,
5508			       bool enable)
5509{
5510	u32 orig, data;
5511
5512	orig = data = RREG32_PCIE(PCIE_CNTL2);
5513
5514	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5515		data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5516			REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5517	else
5518		data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5519			  REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5520
5521	if (orig != data)
5522		WREG32_PCIE(PCIE_CNTL2, data);
5523}
5524
5525static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5526			       bool enable)
5527{
5528	u32 orig, data;
5529
5530	orig = data = RREG32(HDP_HOST_PATH_CNTL);
5531
5532	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5533		data &= ~CLOCK_GATING_DIS;
5534	else
5535		data |= CLOCK_GATING_DIS;
5536
5537	if (orig != data)
5538		WREG32(HDP_HOST_PATH_CNTL, data);
5539}
5540
5541static void si_enable_hdp_ls(struct radeon_device *rdev,
5542			     bool enable)
5543{
5544	u32 orig, data;
5545
5546	orig = data = RREG32(HDP_MEM_POWER_LS);
5547
5548	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5549		data |= HDP_LS_ENABLE;
5550	else
5551		data &= ~HDP_LS_ENABLE;
5552
5553	if (orig != data)
5554		WREG32(HDP_MEM_POWER_LS, data);
5555}
5556
5557static void si_update_cg(struct radeon_device *rdev,
5558			 u32 block, bool enable)
5559{
5560	if (block & RADEON_CG_BLOCK_GFX) {
5561		si_enable_gui_idle_interrupt(rdev, false);
5562		/* order matters! */
5563		if (enable) {
5564			si_enable_mgcg(rdev, true);
5565			si_enable_cgcg(rdev, true);
5566		} else {
5567			si_enable_cgcg(rdev, false);
5568			si_enable_mgcg(rdev, false);
5569		}
5570		si_enable_gui_idle_interrupt(rdev, true);
5571	}
5572
5573	if (block & RADEON_CG_BLOCK_MC) {
5574		si_enable_mc_mgcg(rdev, enable);
5575		si_enable_mc_ls(rdev, enable);
5576	}
5577
5578	if (block & RADEON_CG_BLOCK_SDMA) {
5579		si_enable_dma_mgcg(rdev, enable);
5580	}
5581
5582	if (block & RADEON_CG_BLOCK_BIF) {
5583		si_enable_bif_mgls(rdev, enable);
5584	}
5585
5586	if (block & RADEON_CG_BLOCK_UVD) {
5587		if (rdev->has_uvd) {
5588			si_enable_uvd_mgcg(rdev, enable);
5589		}
5590	}
5591
5592	if (block & RADEON_CG_BLOCK_HDP) {
5593		si_enable_hdp_mgcg(rdev, enable);
5594		si_enable_hdp_ls(rdev, enable);
5595	}
5596}
5597
5598static void si_init_cg(struct radeon_device *rdev)
5599{
5600	si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5601			    RADEON_CG_BLOCK_MC |
5602			    RADEON_CG_BLOCK_SDMA |
5603			    RADEON_CG_BLOCK_BIF |
5604			    RADEON_CG_BLOCK_HDP), true);
5605	if (rdev->has_uvd) {
5606		si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5607		si_init_uvd_internal_cg(rdev);
5608	}
5609}
5610
5611static void si_fini_cg(struct radeon_device *rdev)
5612{
5613	if (rdev->has_uvd) {
5614		si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5615	}
5616	si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5617			    RADEON_CG_BLOCK_MC |
5618			    RADEON_CG_BLOCK_SDMA |
5619			    RADEON_CG_BLOCK_BIF |
5620			    RADEON_CG_BLOCK_HDP), false);
5621}
5622
5623u32 si_get_csb_size(struct radeon_device *rdev)
5624{
5625	u32 count = 0;
5626	const struct cs_section_def *sect = NULL;
5627	const struct cs_extent_def *ext = NULL;
5628
5629	if (rdev->rlc.cs_data == NULL)
5630		return 0;
5631
5632	/* begin clear state */
5633	count += 2;
5634	/* context control state */
5635	count += 3;
5636
5637	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5638		for (ext = sect->section; ext->extent != NULL; ++ext) {
5639			if (sect->id == SECT_CONTEXT)
5640				count += 2 + ext->reg_count;
5641			else
5642				return 0;
5643		}
5644	}
5645	/* pa_sc_raster_config */
5646	count += 3;
5647	/* end clear state */
5648	count += 2;
5649	/* clear state */
5650	count += 2;
5651
5652	return count;
5653}
5654
5655void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5656{
5657	u32 count = 0, i;
5658	const struct cs_section_def *sect = NULL;
5659	const struct cs_extent_def *ext = NULL;
5660
5661	if (rdev->rlc.cs_data == NULL)
5662		return;
5663	if (buffer == NULL)
5664		return;
5665
5666	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5667	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5668
5669	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5670	buffer[count++] = cpu_to_le32(0x80000000);
5671	buffer[count++] = cpu_to_le32(0x80000000);
5672
5673	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5674		for (ext = sect->section; ext->extent != NULL; ++ext) {
5675			if (sect->id == SECT_CONTEXT) {
5676				buffer[count++] =
5677					cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5678				buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5679				for (i = 0; i < ext->reg_count; i++)
5680					buffer[count++] = cpu_to_le32(ext->extent[i]);
5681			} else {
5682				return;
5683			}
5684		}
5685	}
5686
5687	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5688	buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5689	switch (rdev->family) {
5690	case CHIP_TAHITI:
5691	case CHIP_PITCAIRN:
5692		buffer[count++] = cpu_to_le32(0x2a00126a);
5693		break;
5694	case CHIP_VERDE:
5695		buffer[count++] = cpu_to_le32(0x0000124a);
5696		break;
5697	case CHIP_OLAND:
5698		buffer[count++] = cpu_to_le32(0x00000082);
5699		break;
5700	case CHIP_HAINAN:
5701		buffer[count++] = cpu_to_le32(0x00000000);
5702		break;
5703	default:
5704		buffer[count++] = cpu_to_le32(0x00000000);
5705		break;
5706	}
5707
5708	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5709	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5710
5711	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5712	buffer[count++] = cpu_to_le32(0);
5713}
5714
5715static void si_init_pg(struct radeon_device *rdev)
5716{
5717	if (rdev->pg_flags) {
5718		if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5719			si_init_dma_pg(rdev);
5720		}
5721		si_init_ao_cu_mask(rdev);
5722		if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5723			si_init_gfx_cgpg(rdev);
5724		} else {
5725			WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5726			WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5727		}
5728		si_enable_dma_pg(rdev, true);
5729		si_enable_gfx_cgpg(rdev, true);
5730	} else {
5731		WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5732		WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5733	}
5734}
5735
5736static void si_fini_pg(struct radeon_device *rdev)
5737{
5738	if (rdev->pg_flags) {
5739		si_enable_dma_pg(rdev, false);
5740		si_enable_gfx_cgpg(rdev, false);
5741	}
5742}
5743
5744/*
5745 * RLC
5746 */
5747void si_rlc_reset(struct radeon_device *rdev)
5748{
5749	u32 tmp = RREG32(GRBM_SOFT_RESET);
5750
5751	tmp |= SOFT_RESET_RLC;
5752	WREG32(GRBM_SOFT_RESET, tmp);
5753	udelay(50);
5754	tmp &= ~SOFT_RESET_RLC;
5755	WREG32(GRBM_SOFT_RESET, tmp);
5756	udelay(50);
5757}
5758
5759static void si_rlc_stop(struct radeon_device *rdev)
5760{
5761	WREG32(RLC_CNTL, 0);
5762
5763	si_enable_gui_idle_interrupt(rdev, false);
5764
5765	si_wait_for_rlc_serdes(rdev);
5766}
5767
5768static void si_rlc_start(struct radeon_device *rdev)
5769{
5770	WREG32(RLC_CNTL, RLC_ENABLE);
5771
5772	si_enable_gui_idle_interrupt(rdev, true);
5773
5774	udelay(50);
5775}
5776
5777static bool si_lbpw_supported(struct radeon_device *rdev)
5778{
5779	u32 tmp;
5780
5781	/* Enable LBPW only for DDR3 */
5782	tmp = RREG32(MC_SEQ_MISC0);
5783	if ((tmp & 0xF0000000) == 0xB0000000)
5784		return true;
5785	return false;
5786}
5787
5788static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5789{
5790	u32 tmp;
5791
5792	tmp = RREG32(RLC_LB_CNTL);
5793	if (enable)
5794		tmp |= LOAD_BALANCE_ENABLE;
5795	else
5796		tmp &= ~LOAD_BALANCE_ENABLE;
5797	WREG32(RLC_LB_CNTL, tmp);
5798
5799	if (!enable) {
5800		si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5801		WREG32(SPI_LB_CU_MASK, 0x00ff);
5802	}
5803}
5804
5805static int si_rlc_resume(struct radeon_device *rdev)
5806{
5807	u32 i;
5808
5809	if (!rdev->rlc_fw)
5810		return -EINVAL;
5811
5812	si_rlc_stop(rdev);
5813
5814	si_rlc_reset(rdev);
5815
5816	si_init_pg(rdev);
5817
5818	si_init_cg(rdev);
5819
5820	WREG32(RLC_RL_BASE, 0);
5821	WREG32(RLC_RL_SIZE, 0);
5822	WREG32(RLC_LB_CNTL, 0);
5823	WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5824	WREG32(RLC_LB_CNTR_INIT, 0);
5825	WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5826
5827	WREG32(RLC_MC_CNTL, 0);
5828	WREG32(RLC_UCODE_CNTL, 0);
5829
5830	if (rdev->new_fw) {
5831		const struct rlc_firmware_header_v1_0 *hdr =
5832			(const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5833		u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5834		const __le32 *fw_data = (const __le32 *)
5835			(rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5836
5837		radeon_ucode_print_rlc_hdr(&hdr->header);
5838
5839		for (i = 0; i < fw_size; i++) {
5840			WREG32(RLC_UCODE_ADDR, i);
5841			WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5842		}
5843	} else {
5844		const __be32 *fw_data =
5845			(const __be32 *)rdev->rlc_fw->data;
5846		for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5847			WREG32(RLC_UCODE_ADDR, i);
5848			WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5849		}
5850	}
5851	WREG32(RLC_UCODE_ADDR, 0);
5852
5853	si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5854
5855	si_rlc_start(rdev);
5856
5857	return 0;
5858}
5859
5860static void si_enable_interrupts(struct radeon_device *rdev)
5861{
5862	u32 ih_cntl = RREG32(IH_CNTL);
5863	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5864
5865	ih_cntl |= ENABLE_INTR;
5866	ih_rb_cntl |= IH_RB_ENABLE;
5867	WREG32(IH_CNTL, ih_cntl);
5868	WREG32(IH_RB_CNTL, ih_rb_cntl);
5869	rdev->ih.enabled = true;
5870}
5871
5872static void si_disable_interrupts(struct radeon_device *rdev)
5873{
5874	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5875	u32 ih_cntl = RREG32(IH_CNTL);
5876
5877	ih_rb_cntl &= ~IH_RB_ENABLE;
5878	ih_cntl &= ~ENABLE_INTR;
5879	WREG32(IH_RB_CNTL, ih_rb_cntl);
5880	WREG32(IH_CNTL, ih_cntl);
5881	/* set rptr, wptr to 0 */
5882	WREG32(IH_RB_RPTR, 0);
5883	WREG32(IH_RB_WPTR, 0);
5884	rdev->ih.enabled = false;
5885	rdev->ih.rptr = 0;
5886}
5887
5888static void si_disable_interrupt_state(struct radeon_device *rdev)
5889{
 
5890	u32 tmp;
5891
5892	tmp = RREG32(CP_INT_CNTL_RING0) &
5893		(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5894	WREG32(CP_INT_CNTL_RING0, tmp);
5895	WREG32(CP_INT_CNTL_RING1, 0);
5896	WREG32(CP_INT_CNTL_RING2, 0);
5897	tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5898	WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5899	tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5900	WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5901	WREG32(GRBM_INT_CNTL, 0);
5902	WREG32(SRBM_INT_CNTL, 0);
5903	if (rdev->num_crtc >= 2) {
5904		WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5905		WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5906	}
5907	if (rdev->num_crtc >= 4) {
5908		WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5909		WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5910	}
5911	if (rdev->num_crtc >= 6) {
5912		WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5913		WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5914	}
5915
5916	if (rdev->num_crtc >= 2) {
5917		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5918		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5919	}
5920	if (rdev->num_crtc >= 4) {
5921		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5922		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5923	}
5924	if (rdev->num_crtc >= 6) {
5925		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5926		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5927	}
5928
5929	if (!ASIC_IS_NODCE(rdev)) {
5930		WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5931
5932		tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5933		WREG32(DC_HPD1_INT_CONTROL, tmp);
5934		tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5935		WREG32(DC_HPD2_INT_CONTROL, tmp);
5936		tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5937		WREG32(DC_HPD3_INT_CONTROL, tmp);
5938		tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5939		WREG32(DC_HPD4_INT_CONTROL, tmp);
5940		tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5941		WREG32(DC_HPD5_INT_CONTROL, tmp);
5942		tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5943		WREG32(DC_HPD6_INT_CONTROL, tmp);
5944	}
5945}
5946
5947static int si_irq_init(struct radeon_device *rdev)
5948{
5949	int ret = 0;
5950	int rb_bufsz;
5951	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5952
5953	/* allocate ring */
5954	ret = r600_ih_ring_alloc(rdev);
5955	if (ret)
5956		return ret;
5957
5958	/* disable irqs */
5959	si_disable_interrupts(rdev);
5960
5961	/* init rlc */
5962	ret = si_rlc_resume(rdev);
5963	if (ret) {
5964		r600_ih_ring_fini(rdev);
5965		return ret;
5966	}
5967
5968	/* setup interrupt control */
5969	/* set dummy read address to ring address */
5970	WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5971	interrupt_cntl = RREG32(INTERRUPT_CNTL);
5972	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5973	 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5974	 */
5975	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5976	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5977	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5978	WREG32(INTERRUPT_CNTL, interrupt_cntl);
5979
5980	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5981	rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5982
5983	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5984		      IH_WPTR_OVERFLOW_CLEAR |
5985		      (rb_bufsz << 1));
5986
5987	if (rdev->wb.enabled)
5988		ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5989
5990	/* set the writeback address whether it's enabled or not */
5991	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5992	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5993
5994	WREG32(IH_RB_CNTL, ih_rb_cntl);
5995
5996	/* set rptr, wptr to 0 */
5997	WREG32(IH_RB_RPTR, 0);
5998	WREG32(IH_RB_WPTR, 0);
5999
6000	/* Default settings for IH_CNTL (disabled at first) */
6001	ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6002	/* RPTR_REARM only works if msi's are enabled */
6003	if (rdev->msi_enabled)
6004		ih_cntl |= RPTR_REARM;
6005	WREG32(IH_CNTL, ih_cntl);
6006
6007	/* force the active interrupt state to all disabled */
6008	si_disable_interrupt_state(rdev);
6009
6010	pci_set_master(rdev->pdev);
6011
6012	/* enable irqs */
6013	si_enable_interrupts(rdev);
6014
6015	return ret;
6016}
6017
 
6018int si_irq_set(struct radeon_device *rdev)
6019{
 
6020	u32 cp_int_cntl;
6021	u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
6022	u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6023	u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
6024	u32 grbm_int_cntl = 0;
6025	u32 dma_cntl, dma_cntl1;
6026	u32 thermal_int = 0;
6027
6028	if (!rdev->irq.installed) {
6029		WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6030		return -EINVAL;
6031	}
6032	/* don't enable anything if the ih is disabled */
6033	if (!rdev->ih.enabled) {
6034		si_disable_interrupts(rdev);
6035		/* force the active interrupt state to all disabled */
6036		si_disable_interrupt_state(rdev);
6037		return 0;
6038	}
6039
6040	cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6041		(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6042
6043	if (!ASIC_IS_NODCE(rdev)) {
6044		hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6045		hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6046		hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6047		hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6048		hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6049		hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6050	}
6051
6052	dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6053	dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6054
6055	thermal_int = RREG32(CG_THERMAL_INT) &
6056		~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6057
6058	/* enable CP interrupts on all rings */
6059	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6060		DRM_DEBUG("si_irq_set: sw int gfx\n");
6061		cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6062	}
6063	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6064		DRM_DEBUG("si_irq_set: sw int cp1\n");
6065		cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6066	}
6067	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6068		DRM_DEBUG("si_irq_set: sw int cp2\n");
6069		cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6070	}
6071	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6072		DRM_DEBUG("si_irq_set: sw int dma\n");
6073		dma_cntl |= TRAP_ENABLE;
6074	}
6075
6076	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6077		DRM_DEBUG("si_irq_set: sw int dma1\n");
6078		dma_cntl1 |= TRAP_ENABLE;
6079	}
6080	if (rdev->irq.crtc_vblank_int[0] ||
6081	    atomic_read(&rdev->irq.pflip[0])) {
6082		DRM_DEBUG("si_irq_set: vblank 0\n");
6083		crtc1 |= VBLANK_INT_MASK;
6084	}
6085	if (rdev->irq.crtc_vblank_int[1] ||
6086	    atomic_read(&rdev->irq.pflip[1])) {
6087		DRM_DEBUG("si_irq_set: vblank 1\n");
6088		crtc2 |= VBLANK_INT_MASK;
6089	}
6090	if (rdev->irq.crtc_vblank_int[2] ||
6091	    atomic_read(&rdev->irq.pflip[2])) {
6092		DRM_DEBUG("si_irq_set: vblank 2\n");
6093		crtc3 |= VBLANK_INT_MASK;
6094	}
6095	if (rdev->irq.crtc_vblank_int[3] ||
6096	    atomic_read(&rdev->irq.pflip[3])) {
6097		DRM_DEBUG("si_irq_set: vblank 3\n");
6098		crtc4 |= VBLANK_INT_MASK;
6099	}
6100	if (rdev->irq.crtc_vblank_int[4] ||
6101	    atomic_read(&rdev->irq.pflip[4])) {
6102		DRM_DEBUG("si_irq_set: vblank 4\n");
6103		crtc5 |= VBLANK_INT_MASK;
6104	}
6105	if (rdev->irq.crtc_vblank_int[5] ||
6106	    atomic_read(&rdev->irq.pflip[5])) {
6107		DRM_DEBUG("si_irq_set: vblank 5\n");
6108		crtc6 |= VBLANK_INT_MASK;
6109	}
6110	if (rdev->irq.hpd[0]) {
6111		DRM_DEBUG("si_irq_set: hpd 1\n");
6112		hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6113	}
6114	if (rdev->irq.hpd[1]) {
6115		DRM_DEBUG("si_irq_set: hpd 2\n");
6116		hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6117	}
6118	if (rdev->irq.hpd[2]) {
6119		DRM_DEBUG("si_irq_set: hpd 3\n");
6120		hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6121	}
6122	if (rdev->irq.hpd[3]) {
6123		DRM_DEBUG("si_irq_set: hpd 4\n");
6124		hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6125	}
6126	if (rdev->irq.hpd[4]) {
6127		DRM_DEBUG("si_irq_set: hpd 5\n");
6128		hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6129	}
6130	if (rdev->irq.hpd[5]) {
6131		DRM_DEBUG("si_irq_set: hpd 6\n");
6132		hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6133	}
6134
6135	WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6136	WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6137	WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6138
6139	WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6140	WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6141
6142	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6143
6144	if (rdev->irq.dpm_thermal) {
6145		DRM_DEBUG("dpm thermal\n");
6146		thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6147	}
6148
6149	if (rdev->num_crtc >= 2) {
6150		WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6151		WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6152	}
6153	if (rdev->num_crtc >= 4) {
6154		WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6155		WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6156	}
6157	if (rdev->num_crtc >= 6) {
6158		WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6159		WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6160	}
6161
6162	if (rdev->num_crtc >= 2) {
6163		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
6164		       GRPH_PFLIP_INT_MASK);
6165		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
6166		       GRPH_PFLIP_INT_MASK);
6167	}
6168	if (rdev->num_crtc >= 4) {
6169		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
6170		       GRPH_PFLIP_INT_MASK);
6171		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
6172		       GRPH_PFLIP_INT_MASK);
6173	}
6174	if (rdev->num_crtc >= 6) {
6175		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
6176		       GRPH_PFLIP_INT_MASK);
6177		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
6178		       GRPH_PFLIP_INT_MASK);
6179	}
6180
 
 
 
6181	if (!ASIC_IS_NODCE(rdev)) {
6182		WREG32(DC_HPD1_INT_CONTROL, hpd1);
6183		WREG32(DC_HPD2_INT_CONTROL, hpd2);
6184		WREG32(DC_HPD3_INT_CONTROL, hpd3);
6185		WREG32(DC_HPD4_INT_CONTROL, hpd4);
6186		WREG32(DC_HPD5_INT_CONTROL, hpd5);
6187		WREG32(DC_HPD6_INT_CONTROL, hpd6);
6188	}
6189
6190	WREG32(CG_THERMAL_INT, thermal_int);
6191
6192	/* posting read */
6193	RREG32(SRBM_STATUS);
6194
6195	return 0;
6196}
6197
 
6198static inline void si_irq_ack(struct radeon_device *rdev)
6199{
6200	u32 tmp;
 
 
6201
6202	if (ASIC_IS_NODCE(rdev))
6203		return;
6204
6205	rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6206	rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6207	rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6208	rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6209	rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6210	rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6211	rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
6212	rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
6213	if (rdev->num_crtc >= 4) {
6214		rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
6215		rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
6216	}
6217	if (rdev->num_crtc >= 6) {
6218		rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
6219		rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
6220	}
6221
6222	if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
6223		WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6224	if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
6225		WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6226	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
6227		WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6228	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
6229		WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6230	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6231		WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6232	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6233		WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6234
6235	if (rdev->num_crtc >= 4) {
6236		if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
6237			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6238		if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
6239			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6240		if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6241			WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6242		if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6243			WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6244		if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6245			WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6246		if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6247			WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6248	}
6249
6250	if (rdev->num_crtc >= 6) {
6251		if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
6252			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6253		if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
6254			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6255		if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6256			WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6257		if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6258			WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6259		if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6260			WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6261		if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6262			WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6263	}
6264
6265	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6266		tmp = RREG32(DC_HPD1_INT_CONTROL);
6267		tmp |= DC_HPDx_INT_ACK;
6268		WREG32(DC_HPD1_INT_CONTROL, tmp);
6269	}
6270	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6271		tmp = RREG32(DC_HPD2_INT_CONTROL);
6272		tmp |= DC_HPDx_INT_ACK;
6273		WREG32(DC_HPD2_INT_CONTROL, tmp);
6274	}
6275	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6276		tmp = RREG32(DC_HPD3_INT_CONTROL);
6277		tmp |= DC_HPDx_INT_ACK;
6278		WREG32(DC_HPD3_INT_CONTROL, tmp);
6279	}
6280	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6281		tmp = RREG32(DC_HPD4_INT_CONTROL);
6282		tmp |= DC_HPDx_INT_ACK;
6283		WREG32(DC_HPD4_INT_CONTROL, tmp);
6284	}
6285	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6286		tmp = RREG32(DC_HPD5_INT_CONTROL);
6287		tmp |= DC_HPDx_INT_ACK;
6288		WREG32(DC_HPD5_INT_CONTROL, tmp);
6289	}
6290	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6291		tmp = RREG32(DC_HPD5_INT_CONTROL);
6292		tmp |= DC_HPDx_INT_ACK;
6293		WREG32(DC_HPD6_INT_CONTROL, tmp);
6294	}
6295
6296	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
6297		tmp = RREG32(DC_HPD1_INT_CONTROL);
6298		tmp |= DC_HPDx_RX_INT_ACK;
6299		WREG32(DC_HPD1_INT_CONTROL, tmp);
6300	}
6301	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
6302		tmp = RREG32(DC_HPD2_INT_CONTROL);
6303		tmp |= DC_HPDx_RX_INT_ACK;
6304		WREG32(DC_HPD2_INT_CONTROL, tmp);
6305	}
6306	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
6307		tmp = RREG32(DC_HPD3_INT_CONTROL);
6308		tmp |= DC_HPDx_RX_INT_ACK;
6309		WREG32(DC_HPD3_INT_CONTROL, tmp);
6310	}
6311	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
6312		tmp = RREG32(DC_HPD4_INT_CONTROL);
6313		tmp |= DC_HPDx_RX_INT_ACK;
6314		WREG32(DC_HPD4_INT_CONTROL, tmp);
6315	}
6316	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
6317		tmp = RREG32(DC_HPD5_INT_CONTROL);
6318		tmp |= DC_HPDx_RX_INT_ACK;
6319		WREG32(DC_HPD5_INT_CONTROL, tmp);
6320	}
6321	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
6322		tmp = RREG32(DC_HPD5_INT_CONTROL);
6323		tmp |= DC_HPDx_RX_INT_ACK;
6324		WREG32(DC_HPD6_INT_CONTROL, tmp);
6325	}
6326}
6327
6328static void si_irq_disable(struct radeon_device *rdev)
6329{
6330	si_disable_interrupts(rdev);
6331	/* Wait and acknowledge irq */
6332	mdelay(1);
6333	si_irq_ack(rdev);
6334	si_disable_interrupt_state(rdev);
6335}
6336
6337static void si_irq_suspend(struct radeon_device *rdev)
6338{
6339	si_irq_disable(rdev);
6340	si_rlc_stop(rdev);
6341}
6342
6343static void si_irq_fini(struct radeon_device *rdev)
6344{
6345	si_irq_suspend(rdev);
6346	r600_ih_ring_fini(rdev);
6347}
6348
6349static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6350{
6351	u32 wptr, tmp;
6352
6353	if (rdev->wb.enabled)
6354		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6355	else
6356		wptr = RREG32(IH_RB_WPTR);
6357
6358	if (wptr & RB_OVERFLOW) {
6359		wptr &= ~RB_OVERFLOW;
6360		/* When a ring buffer overflow happen start parsing interrupt
6361		 * from the last not overwritten vector (wptr + 16). Hopefully
6362		 * this should allow us to catchup.
6363		 */
6364		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6365			 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
6366		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6367		tmp = RREG32(IH_RB_CNTL);
6368		tmp |= IH_WPTR_OVERFLOW_CLEAR;
6369		WREG32(IH_RB_CNTL, tmp);
6370	}
6371	return (wptr & rdev->ih.ptr_mask);
6372}
6373
6374/*        SI IV Ring
6375 * Each IV ring entry is 128 bits:
6376 * [7:0]    - interrupt source id
6377 * [31:8]   - reserved
6378 * [59:32]  - interrupt source data
6379 * [63:60]  - reserved
6380 * [71:64]  - RINGID
6381 * [79:72]  - VMID
6382 * [127:80] - reserved
6383 */
6384int si_irq_process(struct radeon_device *rdev)
6385{
 
 
 
6386	u32 wptr;
6387	u32 rptr;
6388	u32 src_id, src_data, ring_id;
6389	u32 ring_index;
6390	bool queue_hotplug = false;
6391	bool queue_dp = false;
6392	bool queue_thermal = false;
6393	u32 status, addr;
 
6394
6395	if (!rdev->ih.enabled || rdev->shutdown)
6396		return IRQ_NONE;
6397
6398	wptr = si_get_ih_wptr(rdev);
6399
6400restart_ih:
6401	/* is somebody else already processing irqs? */
6402	if (atomic_xchg(&rdev->ih.lock, 1))
6403		return IRQ_NONE;
6404
6405	rptr = rdev->ih.rptr;
6406	DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6407
6408	/* Order reading of wptr vs. reading of IH ring data */
6409	rmb();
6410
6411	/* display interrupts */
6412	si_irq_ack(rdev);
6413
6414	while (rptr != wptr) {
6415		/* wptr/rptr are in bytes! */
6416		ring_index = rptr / 4;
6417		src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6418		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6419		ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6420
6421		switch (src_id) {
6422		case 1: /* D1 vblank/vline */
6423			switch (src_data) {
6424			case 0: /* D1 vblank */
6425				if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
6426					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6427
6428				if (rdev->irq.crtc_vblank_int[0]) {
6429					drm_handle_vblank(rdev->ddev, 0);
6430					rdev->pm.vblank_sync = true;
6431					wake_up(&rdev->irq.vblank_queue);
6432				}
6433				if (atomic_read(&rdev->irq.pflip[0]))
6434					radeon_crtc_handle_vblank(rdev, 0);
6435				rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6436				DRM_DEBUG("IH: D1 vblank\n");
6437
6438				break;
6439			case 1: /* D1 vline */
6440				if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
6441					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6442
6443				rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6444				DRM_DEBUG("IH: D1 vline\n");
6445
6446				break;
6447			default:
6448				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6449				break;
6450			}
6451			break;
6452		case 2: /* D2 vblank/vline */
6453			switch (src_data) {
6454			case 0: /* D2 vblank */
6455				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
6456					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6457
6458				if (rdev->irq.crtc_vblank_int[1]) {
6459					drm_handle_vblank(rdev->ddev, 1);
6460					rdev->pm.vblank_sync = true;
6461					wake_up(&rdev->irq.vblank_queue);
6462				}
6463				if (atomic_read(&rdev->irq.pflip[1]))
6464					radeon_crtc_handle_vblank(rdev, 1);
6465				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6466				DRM_DEBUG("IH: D2 vblank\n");
6467
6468				break;
6469			case 1: /* D2 vline */
6470				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
6471					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6472
6473				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6474				DRM_DEBUG("IH: D2 vline\n");
6475
6476				break;
6477			default:
6478				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6479				break;
6480			}
6481			break;
6482		case 3: /* D3 vblank/vline */
6483			switch (src_data) {
6484			case 0: /* D3 vblank */
6485				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
6486					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6487
6488				if (rdev->irq.crtc_vblank_int[2]) {
6489					drm_handle_vblank(rdev->ddev, 2);
6490					rdev->pm.vblank_sync = true;
6491					wake_up(&rdev->irq.vblank_queue);
6492				}
6493				if (atomic_read(&rdev->irq.pflip[2]))
6494					radeon_crtc_handle_vblank(rdev, 2);
6495				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6496				DRM_DEBUG("IH: D3 vblank\n");
6497
6498				break;
6499			case 1: /* D3 vline */
6500				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
6501					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6502
6503				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6504				DRM_DEBUG("IH: D3 vline\n");
6505
6506				break;
6507			default:
6508				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6509				break;
6510			}
6511			break;
6512		case 4: /* D4 vblank/vline */
6513			switch (src_data) {
6514			case 0: /* D4 vblank */
6515				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
6516					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6517
6518				if (rdev->irq.crtc_vblank_int[3]) {
6519					drm_handle_vblank(rdev->ddev, 3);
6520					rdev->pm.vblank_sync = true;
6521					wake_up(&rdev->irq.vblank_queue);
6522				}
6523				if (atomic_read(&rdev->irq.pflip[3]))
6524					radeon_crtc_handle_vblank(rdev, 3);
6525				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6526				DRM_DEBUG("IH: D4 vblank\n");
6527
6528				break;
6529			case 1: /* D4 vline */
6530				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
6531					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6532
6533				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6534				DRM_DEBUG("IH: D4 vline\n");
6535
6536				break;
6537			default:
6538				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6539				break;
6540			}
6541			break;
6542		case 5: /* D5 vblank/vline */
6543			switch (src_data) {
6544			case 0: /* D5 vblank */
6545				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
6546					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6547
6548				if (rdev->irq.crtc_vblank_int[4]) {
6549					drm_handle_vblank(rdev->ddev, 4);
6550					rdev->pm.vblank_sync = true;
6551					wake_up(&rdev->irq.vblank_queue);
6552				}
6553				if (atomic_read(&rdev->irq.pflip[4]))
6554					radeon_crtc_handle_vblank(rdev, 4);
6555				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6556				DRM_DEBUG("IH: D5 vblank\n");
6557
6558				break;
6559			case 1: /* D5 vline */
6560				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
6561					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6562
6563				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6564				DRM_DEBUG("IH: D5 vline\n");
6565
6566				break;
6567			default:
6568				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6569				break;
6570			}
6571			break;
6572		case 6: /* D6 vblank/vline */
6573			switch (src_data) {
6574			case 0: /* D6 vblank */
6575				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
6576					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6577
6578				if (rdev->irq.crtc_vblank_int[5]) {
6579					drm_handle_vblank(rdev->ddev, 5);
6580					rdev->pm.vblank_sync = true;
6581					wake_up(&rdev->irq.vblank_queue);
6582				}
6583				if (atomic_read(&rdev->irq.pflip[5]))
6584					radeon_crtc_handle_vblank(rdev, 5);
6585				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6586				DRM_DEBUG("IH: D6 vblank\n");
6587
6588				break;
6589			case 1: /* D6 vline */
6590				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
6591					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6592
6593				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6594				DRM_DEBUG("IH: D6 vline\n");
6595
6596				break;
6597			default:
6598				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6599				break;
6600			}
6601			break;
6602		case 8: /* D1 page flip */
6603		case 10: /* D2 page flip */
6604		case 12: /* D3 page flip */
6605		case 14: /* D4 page flip */
6606		case 16: /* D5 page flip */
6607		case 18: /* D6 page flip */
6608			DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
6609			if (radeon_use_pflipirq > 0)
6610				radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6611			break;
6612		case 42: /* HPD hotplug */
6613			switch (src_data) {
6614			case 0:
6615				if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
6616					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6617
6618				rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6619				queue_hotplug = true;
6620				DRM_DEBUG("IH: HPD1\n");
6621
6622				break;
6623			case 1:
6624				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
6625					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6626
6627				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6628				queue_hotplug = true;
6629				DRM_DEBUG("IH: HPD2\n");
6630
6631				break;
6632			case 2:
6633				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
6634					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6635
6636				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6637				queue_hotplug = true;
6638				DRM_DEBUG("IH: HPD3\n");
6639
6640				break;
6641			case 3:
6642				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
6643					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6644
6645				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6646				queue_hotplug = true;
6647				DRM_DEBUG("IH: HPD4\n");
6648
6649				break;
6650			case 4:
6651				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
6652					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6653
6654				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6655				queue_hotplug = true;
6656				DRM_DEBUG("IH: HPD5\n");
6657
6658				break;
6659			case 5:
6660				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
6661					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6662
6663				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6664				queue_hotplug = true;
6665				DRM_DEBUG("IH: HPD6\n");
6666
6667				break;
6668			case 6:
6669				if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
6670					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6671
6672				rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
6673				queue_dp = true;
6674				DRM_DEBUG("IH: HPD_RX 1\n");
6675
 
 
 
6676				break;
6677			case 7:
6678				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
6679					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6680
6681				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
6682				queue_dp = true;
6683				DRM_DEBUG("IH: HPD_RX 2\n");
6684
6685				break;
6686			case 8:
6687				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
6688					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6689
6690				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
6691				queue_dp = true;
6692				DRM_DEBUG("IH: HPD_RX 3\n");
6693
6694				break;
6695			case 9:
6696				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
6697					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6698
6699				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
6700				queue_dp = true;
6701				DRM_DEBUG("IH: HPD_RX 4\n");
6702
6703				break;
6704			case 10:
6705				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
6706					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6707
6708				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
6709				queue_dp = true;
6710				DRM_DEBUG("IH: HPD_RX 5\n");
6711
6712				break;
6713			case 11:
6714				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
6715					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6716
6717				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
6718				queue_dp = true;
6719				DRM_DEBUG("IH: HPD_RX 6\n");
6720
6721				break;
6722			default:
6723				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6724				break;
6725			}
6726			break;
6727		case 96:
6728			DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
6729			WREG32(SRBM_INT_ACK, 0x1);
6730			break;
6731		case 124: /* UVD */
6732			DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6733			radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6734			break;
6735		case 146:
6736		case 147:
6737			addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6738			status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6739			/* reset addr and status */
6740			WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6741			if (addr == 0x0 && status == 0x0)
6742				break;
6743			dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6744			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
6745				addr);
6746			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6747				status);
6748			si_vm_decode_fault(rdev, status, addr);
6749			break;
6750		case 176: /* RINGID0 CP_INT */
6751			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6752			break;
6753		case 177: /* RINGID1 CP_INT */
6754			radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6755			break;
6756		case 178: /* RINGID2 CP_INT */
6757			radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6758			break;
6759		case 181: /* CP EOP event */
6760			DRM_DEBUG("IH: CP EOP\n");
6761			switch (ring_id) {
6762			case 0:
6763				radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6764				break;
6765			case 1:
6766				radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6767				break;
6768			case 2:
6769				radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6770				break;
6771			}
6772			break;
6773		case 224: /* DMA trap event */
6774			DRM_DEBUG("IH: DMA trap\n");
6775			radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6776			break;
6777		case 230: /* thermal low to high */
6778			DRM_DEBUG("IH: thermal low to high\n");
6779			rdev->pm.dpm.thermal.high_to_low = false;
6780			queue_thermal = true;
6781			break;
6782		case 231: /* thermal high to low */
6783			DRM_DEBUG("IH: thermal high to low\n");
6784			rdev->pm.dpm.thermal.high_to_low = true;
6785			queue_thermal = true;
6786			break;
6787		case 233: /* GUI IDLE */
6788			DRM_DEBUG("IH: GUI idle\n");
6789			break;
6790		case 244: /* DMA trap event */
6791			DRM_DEBUG("IH: DMA1 trap\n");
6792			radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6793			break;
6794		default:
6795			DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6796			break;
6797		}
6798
6799		/* wptr/rptr are in bytes! */
6800		rptr += 16;
6801		rptr &= rdev->ih.ptr_mask;
6802		WREG32(IH_RB_RPTR, rptr);
6803	}
6804	if (queue_dp)
6805		schedule_work(&rdev->dp_work);
6806	if (queue_hotplug)
6807		schedule_delayed_work(&rdev->hotplug_work, 0);
6808	if (queue_thermal && rdev->pm.dpm_enabled)
6809		schedule_work(&rdev->pm.dpm.thermal.work);
6810	rdev->ih.rptr = rptr;
6811	atomic_set(&rdev->ih.lock, 0);
6812
6813	/* make sure wptr hasn't changed while processing */
6814	wptr = si_get_ih_wptr(rdev);
6815	if (wptr != rptr)
6816		goto restart_ih;
6817
6818	return IRQ_HANDLED;
6819}
6820
6821/*
6822 * startup/shutdown callbacks
6823 */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6824static int si_startup(struct radeon_device *rdev)
6825{
6826	struct radeon_ring *ring;
6827	int r;
6828
6829	/* enable pcie gen2/3 link */
6830	si_pcie_gen3_enable(rdev);
6831	/* enable aspm */
6832	si_program_aspm(rdev);
6833
6834	/* scratch needs to be initialized before MC */
6835	r = r600_vram_scratch_init(rdev);
6836	if (r)
6837		return r;
6838
6839	si_mc_program(rdev);
6840
6841	if (!rdev->pm.dpm_enabled) {
6842		r = si_mc_load_microcode(rdev);
6843		if (r) {
6844			DRM_ERROR("Failed to load MC firmware!\n");
6845			return r;
6846		}
6847	}
6848
6849	r = si_pcie_gart_enable(rdev);
6850	if (r)
6851		return r;
6852	si_gpu_init(rdev);
6853
6854	/* allocate rlc buffers */
6855	if (rdev->family == CHIP_VERDE) {
6856		rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6857		rdev->rlc.reg_list_size =
6858			(u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6859	}
6860	rdev->rlc.cs_data = si_cs_data;
6861	r = sumo_rlc_init(rdev);
6862	if (r) {
6863		DRM_ERROR("Failed to init rlc BOs!\n");
6864		return r;
6865	}
6866
6867	/* allocate wb buffer */
6868	r = radeon_wb_init(rdev);
6869	if (r)
6870		return r;
6871
6872	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6873	if (r) {
6874		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6875		return r;
6876	}
6877
6878	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6879	if (r) {
6880		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6881		return r;
6882	}
6883
6884	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6885	if (r) {
6886		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6887		return r;
6888	}
6889
6890	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6891	if (r) {
6892		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6893		return r;
6894	}
6895
6896	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6897	if (r) {
6898		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6899		return r;
6900	}
6901
6902	if (rdev->has_uvd) {
6903		r = uvd_v2_2_resume(rdev);
6904		if (!r) {
6905			r = radeon_fence_driver_start_ring(rdev,
6906							   R600_RING_TYPE_UVD_INDEX);
6907			if (r)
6908				dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6909		}
6910		if (r)
6911			rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6912	}
6913
6914	r = radeon_vce_resume(rdev);
6915	if (!r) {
6916		r = vce_v1_0_resume(rdev);
6917		if (!r)
6918			r = radeon_fence_driver_start_ring(rdev,
6919							   TN_RING_TYPE_VCE1_INDEX);
6920		if (!r)
6921			r = radeon_fence_driver_start_ring(rdev,
6922							   TN_RING_TYPE_VCE2_INDEX);
6923	}
6924	if (r) {
6925		dev_err(rdev->dev, "VCE init error (%d).\n", r);
6926		rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
6927		rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
6928	}
6929
6930	/* Enable IRQ */
6931	if (!rdev->irq.installed) {
6932		r = radeon_irq_kms_init(rdev);
6933		if (r)
6934			return r;
6935	}
6936
6937	r = si_irq_init(rdev);
6938	if (r) {
6939		DRM_ERROR("radeon: IH init failed (%d).\n", r);
6940		radeon_irq_kms_fini(rdev);
6941		return r;
6942	}
6943	si_irq_set(rdev);
6944
6945	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6946	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6947			     RADEON_CP_PACKET2);
6948	if (r)
6949		return r;
6950
6951	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6952	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6953			     RADEON_CP_PACKET2);
6954	if (r)
6955		return r;
6956
6957	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6958	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6959			     RADEON_CP_PACKET2);
6960	if (r)
6961		return r;
6962
6963	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6964	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6965			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6966	if (r)
6967		return r;
6968
6969	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6970	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6971			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6972	if (r)
6973		return r;
6974
6975	r = si_cp_load_microcode(rdev);
6976	if (r)
6977		return r;
6978	r = si_cp_resume(rdev);
6979	if (r)
6980		return r;
6981
6982	r = cayman_dma_resume(rdev);
6983	if (r)
6984		return r;
6985
6986	if (rdev->has_uvd) {
6987		ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6988		if (ring->ring_size) {
6989			r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
6990					     RADEON_CP_PACKET2);
6991			if (!r)
6992				r = uvd_v1_0_init(rdev);
6993			if (r)
6994				DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6995		}
6996	}
6997
6998	r = -ENOENT;
6999
7000	ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
7001	if (ring->ring_size)
7002		r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7003				     VCE_CMD_NO_OP);
7004
7005	ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
7006	if (ring->ring_size)
7007		r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7008				     VCE_CMD_NO_OP);
7009
7010	if (!r)
7011		r = vce_v1_0_init(rdev);
7012	else if (r != -ENOENT)
7013		DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
7014
7015	r = radeon_ib_pool_init(rdev);
7016	if (r) {
7017		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7018		return r;
7019	}
7020
7021	r = radeon_vm_manager_init(rdev);
7022	if (r) {
7023		dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7024		return r;
7025	}
7026
7027	r = radeon_audio_init(rdev);
7028	if (r)
7029		return r;
7030
7031	return 0;
7032}
7033
7034int si_resume(struct radeon_device *rdev)
7035{
7036	int r;
7037
7038	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
7039	 * posting will perform necessary task to bring back GPU into good
7040	 * shape.
7041	 */
7042	/* post card */
7043	atom_asic_init(rdev->mode_info.atom_context);
7044
7045	/* init golden registers */
7046	si_init_golden_registers(rdev);
7047
7048	if (rdev->pm.pm_method == PM_METHOD_DPM)
7049		radeon_pm_resume(rdev);
7050
7051	rdev->accel_working = true;
7052	r = si_startup(rdev);
7053	if (r) {
7054		DRM_ERROR("si startup failed on resume\n");
7055		rdev->accel_working = false;
7056		return r;
7057	}
7058
7059	return r;
7060
7061}
7062
7063int si_suspend(struct radeon_device *rdev)
7064{
7065	radeon_pm_suspend(rdev);
7066	radeon_audio_fini(rdev);
7067	radeon_vm_manager_fini(rdev);
7068	si_cp_enable(rdev, false);
7069	cayman_dma_stop(rdev);
7070	if (rdev->has_uvd) {
 
7071		uvd_v1_0_fini(rdev);
7072		radeon_uvd_suspend(rdev);
 
7073		radeon_vce_suspend(rdev);
7074	}
7075	si_fini_pg(rdev);
7076	si_fini_cg(rdev);
7077	si_irq_suspend(rdev);
7078	radeon_wb_disable(rdev);
7079	si_pcie_gart_disable(rdev);
7080	return 0;
7081}
7082
7083/* Plan is to move initialization in that function and use
7084 * helper function so that radeon_device_init pretty much
7085 * do nothing more than calling asic specific function. This
7086 * should also allow to remove a bunch of callback function
7087 * like vram_info.
7088 */
7089int si_init(struct radeon_device *rdev)
7090{
7091	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7092	int r;
7093
7094	/* Read BIOS */
7095	if (!radeon_get_bios(rdev)) {
7096		if (ASIC_IS_AVIVO(rdev))
7097			return -EINVAL;
7098	}
7099	/* Must be an ATOMBIOS */
7100	if (!rdev->is_atom_bios) {
7101		dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7102		return -EINVAL;
7103	}
7104	r = radeon_atombios_init(rdev);
7105	if (r)
7106		return r;
7107
7108	/* Post card if necessary */
7109	if (!radeon_card_posted(rdev)) {
7110		if (!rdev->bios) {
7111			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7112			return -EINVAL;
7113		}
7114		DRM_INFO("GPU not posted. posting now...\n");
7115		atom_asic_init(rdev->mode_info.atom_context);
7116	}
7117	/* init golden registers */
7118	si_init_golden_registers(rdev);
7119	/* Initialize scratch registers */
7120	si_scratch_init(rdev);
7121	/* Initialize surface registers */
7122	radeon_surface_init(rdev);
7123	/* Initialize clocks */
7124	radeon_get_clock_info(rdev->ddev);
7125
7126	/* Fence driver */
7127	r = radeon_fence_driver_init(rdev);
7128	if (r)
7129		return r;
7130
7131	/* initialize memory controller */
7132	r = si_mc_init(rdev);
7133	if (r)
7134		return r;
7135	/* Memory manager */
7136	r = radeon_bo_init(rdev);
7137	if (r)
7138		return r;
7139
7140	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7141	    !rdev->rlc_fw || !rdev->mc_fw) {
7142		r = si_init_microcode(rdev);
7143		if (r) {
7144			DRM_ERROR("Failed to load firmware!\n");
7145			return r;
7146		}
7147	}
7148
7149	/* Initialize power management */
7150	radeon_pm_init(rdev);
7151
7152	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7153	ring->ring_obj = NULL;
7154	r600_ring_init(rdev, ring, 1024 * 1024);
7155
7156	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7157	ring->ring_obj = NULL;
7158	r600_ring_init(rdev, ring, 1024 * 1024);
7159
7160	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7161	ring->ring_obj = NULL;
7162	r600_ring_init(rdev, ring, 1024 * 1024);
7163
7164	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7165	ring->ring_obj = NULL;
7166	r600_ring_init(rdev, ring, 64 * 1024);
7167
7168	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7169	ring->ring_obj = NULL;
7170	r600_ring_init(rdev, ring, 64 * 1024);
7171
7172	if (rdev->has_uvd) {
7173		r = radeon_uvd_init(rdev);
7174		if (!r) {
7175			ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7176			ring->ring_obj = NULL;
7177			r600_ring_init(rdev, ring, 4096);
7178		}
7179	}
7180
7181	r = radeon_vce_init(rdev);
7182	if (!r) {
7183		ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
7184		ring->ring_obj = NULL;
7185		r600_ring_init(rdev, ring, 4096);
7186
7187		ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
7188		ring->ring_obj = NULL;
7189		r600_ring_init(rdev, ring, 4096);
7190	}
7191
7192	rdev->ih.ring_obj = NULL;
7193	r600_ih_ring_init(rdev, 64 * 1024);
7194
7195	r = r600_pcie_gart_init(rdev);
7196	if (r)
7197		return r;
7198
7199	rdev->accel_working = true;
7200	r = si_startup(rdev);
7201	if (r) {
7202		dev_err(rdev->dev, "disabling GPU acceleration\n");
7203		si_cp_fini(rdev);
7204		cayman_dma_fini(rdev);
7205		si_irq_fini(rdev);
7206		sumo_rlc_fini(rdev);
7207		radeon_wb_fini(rdev);
7208		radeon_ib_pool_fini(rdev);
7209		radeon_vm_manager_fini(rdev);
7210		radeon_irq_kms_fini(rdev);
7211		si_pcie_gart_fini(rdev);
7212		rdev->accel_working = false;
7213	}
7214
7215	/* Don't start up if the MC ucode is missing.
7216	 * The default clocks and voltages before the MC ucode
7217	 * is loaded are not suffient for advanced operations.
7218	 */
7219	if (!rdev->mc_fw) {
7220		DRM_ERROR("radeon: MC ucode required for NI+.\n");
7221		return -EINVAL;
7222	}
7223
7224	return 0;
7225}
7226
7227void si_fini(struct radeon_device *rdev)
7228{
7229	radeon_pm_fini(rdev);
7230	si_cp_fini(rdev);
7231	cayman_dma_fini(rdev);
7232	si_fini_pg(rdev);
7233	si_fini_cg(rdev);
7234	si_irq_fini(rdev);
7235	sumo_rlc_fini(rdev);
7236	radeon_wb_fini(rdev);
7237	radeon_vm_manager_fini(rdev);
7238	radeon_ib_pool_fini(rdev);
7239	radeon_irq_kms_fini(rdev);
7240	if (rdev->has_uvd) {
7241		uvd_v1_0_fini(rdev);
7242		radeon_uvd_fini(rdev);
 
 
7243		radeon_vce_fini(rdev);
7244	}
7245	si_pcie_gart_fini(rdev);
7246	r600_vram_scratch_fini(rdev);
7247	radeon_gem_fini(rdev);
7248	radeon_fence_driver_fini(rdev);
7249	radeon_bo_fini(rdev);
7250	radeon_atombios_fini(rdev);
7251	kfree(rdev->bios);
7252	rdev->bios = NULL;
7253}
7254
7255/**
7256 * si_get_gpu_clock_counter - return GPU clock counter snapshot
7257 *
7258 * @rdev: radeon_device pointer
7259 *
7260 * Fetches a GPU clock counter snapshot (SI).
7261 * Returns the 64 bit clock counter snapshot.
7262 */
7263uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
7264{
7265	uint64_t clock;
7266
7267	mutex_lock(&rdev->gpu_clock_mutex);
7268	WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
7269	clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
7270		((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
7271	mutex_unlock(&rdev->gpu_clock_mutex);
7272	return clock;
7273}
7274
7275int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
7276{
7277	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
7278	int r;
7279
7280	/* bypass vclk and dclk with bclk */
7281	WREG32_P(CG_UPLL_FUNC_CNTL_2,
7282		VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
7283		~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7284
7285	/* put PLL in bypass mode */
7286	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
7287
7288	if (!vclk || !dclk) {
7289		/* keep the Bypass mode */
7290		return 0;
7291	}
7292
7293	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
7294					  16384, 0x03FFFFFF, 0, 128, 5,
7295					  &fb_div, &vclk_div, &dclk_div);
7296	if (r)
7297		return r;
7298
7299	/* set RESET_ANTI_MUX to 0 */
7300	WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7301
7302	/* set VCO_MODE to 1 */
7303	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7304
7305	/* disable sleep mode */
7306	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7307
7308	/* deassert UPLL_RESET */
7309	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7310
7311	mdelay(1);
7312
7313	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7314	if (r)
7315		return r;
7316
7317	/* assert UPLL_RESET again */
7318	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7319
7320	/* disable spread spectrum. */
7321	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7322
7323	/* set feedback divider */
7324	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
7325
7326	/* set ref divider to 0 */
7327	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7328
7329	if (fb_div < 307200)
7330		WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7331	else
7332		WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7333
7334	/* set PDIV_A and PDIV_B */
7335	WREG32_P(CG_UPLL_FUNC_CNTL_2,
7336		UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
7337		~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7338
7339	/* give the PLL some time to settle */
7340	mdelay(15);
7341
7342	/* deassert PLL_RESET */
7343	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7344
7345	mdelay(15);
7346
7347	/* switch from bypass mode to normal mode */
7348	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7349
7350	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7351	if (r)
7352		return r;
7353
7354	/* switch VCLK and DCLK selection */
7355	WREG32_P(CG_UPLL_FUNC_CNTL_2,
7356		VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7357		~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7358
7359	mdelay(100);
7360
7361	return 0;
7362}
7363
7364static void si_pcie_gen3_enable(struct radeon_device *rdev)
7365{
7366	struct pci_dev *root = rdev->pdev->bus->self;
7367	int bridge_pos, gpu_pos;
7368	u32 speed_cntl, mask, current_data_rate;
7369	int ret, i;
7370	u16 tmp16;
7371
7372	if (pci_is_root_bus(rdev->pdev->bus))
7373		return;
7374
7375	if (radeon_pcie_gen2 == 0)
7376		return;
7377
7378	if (rdev->flags & RADEON_IS_IGP)
7379		return;
7380
7381	if (!(rdev->flags & RADEON_IS_PCIE))
7382		return;
7383
7384	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
7385	if (ret != 0)
7386		return;
7387
7388	if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
 
7389		return;
7390
7391	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7392	current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7393		LC_CURRENT_DATA_RATE_SHIFT;
7394	if (mask & DRM_PCIE_SPEED_80) {
7395		if (current_data_rate == 2) {
7396			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7397			return;
7398		}
7399		DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7400	} else if (mask & DRM_PCIE_SPEED_50) {
7401		if (current_data_rate == 1) {
7402			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7403			return;
7404		}
7405		DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7406	}
7407
7408	bridge_pos = pci_pcie_cap(root);
7409	if (!bridge_pos)
7410		return;
7411
7412	gpu_pos = pci_pcie_cap(rdev->pdev);
7413	if (!gpu_pos)
7414		return;
7415
7416	if (mask & DRM_PCIE_SPEED_80) {
7417		/* re-try equalization if gen3 is not already enabled */
7418		if (current_data_rate != 2) {
7419			u16 bridge_cfg, gpu_cfg;
7420			u16 bridge_cfg2, gpu_cfg2;
7421			u32 max_lw, current_lw, tmp;
7422
7423			pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7424			pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7425
7426			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
7427			pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7428
7429			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
7430			pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7431
7432			tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7433			max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7434			current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7435
7436			if (current_lw < max_lw) {
7437				tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7438				if (tmp & LC_RENEGOTIATION_SUPPORT) {
7439					tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7440					tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7441					tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7442					WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7443				}
7444			}
7445
7446			for (i = 0; i < 10; i++) {
7447				/* check status */
7448				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
 
 
7449				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7450					break;
7451
7452				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7453				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7454
7455				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7456				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
 
 
 
 
 
 
7457
7458				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7459				tmp |= LC_SET_QUIESCE;
7460				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7461
7462				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7463				tmp |= LC_REDO_EQ;
7464				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7465
7466				mdelay(100);
7467
7468				/* linkctl */
7469				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7470				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7471				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7472				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7473
7474				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7475				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7476				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7477				pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7478
7479				/* linkctl2 */
7480				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7481				tmp16 &= ~((1 << 4) | (7 << 9));
7482				tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7483				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7484
7485				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7486				tmp16 &= ~((1 << 4) | (7 << 9));
7487				tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7488				pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
 
 
 
 
 
 
 
 
 
 
 
 
 
7489
7490				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7491				tmp &= ~LC_SET_QUIESCE;
7492				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7493			}
7494		}
7495	}
7496
7497	/* set the link speed */
7498	speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7499	speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7500	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7501
7502	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7503	tmp16 &= ~0xf;
7504	if (mask & DRM_PCIE_SPEED_80)
7505		tmp16 |= 3; /* gen3 */
7506	else if (mask & DRM_PCIE_SPEED_50)
7507		tmp16 |= 2; /* gen2 */
7508	else
7509		tmp16 |= 1; /* gen1 */
7510	pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7511
7512	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7513	speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7514	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7515
7516	for (i = 0; i < rdev->usec_timeout; i++) {
7517		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7518		if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7519			break;
7520		udelay(1);
7521	}
7522}
7523
7524static void si_program_aspm(struct radeon_device *rdev)
7525{
7526	u32 data, orig;
7527	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7528	bool disable_clkreq = false;
7529
7530	if (radeon_aspm == 0)
7531		return;
7532
7533	if (!(rdev->flags & RADEON_IS_PCIE))
7534		return;
7535
7536	orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7537	data &= ~LC_XMIT_N_FTS_MASK;
7538	data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7539	if (orig != data)
7540		WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7541
7542	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7543	data |= LC_GO_TO_RECOVERY;
7544	if (orig != data)
7545		WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7546
7547	orig = data = RREG32_PCIE(PCIE_P_CNTL);
7548	data |= P_IGNORE_EDB_ERR;
7549	if (orig != data)
7550		WREG32_PCIE(PCIE_P_CNTL, data);
7551
7552	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7553	data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7554	data |= LC_PMI_TO_L1_DIS;
7555	if (!disable_l0s)
7556		data |= LC_L0S_INACTIVITY(7);
7557
7558	if (!disable_l1) {
7559		data |= LC_L1_INACTIVITY(7);
7560		data &= ~LC_PMI_TO_L1_DIS;
7561		if (orig != data)
7562			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7563
7564		if (!disable_plloff_in_l1) {
7565			bool clk_req_support;
7566
7567			orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7568			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7569			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7570			if (orig != data)
7571				WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7572
7573			orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7574			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7575			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7576			if (orig != data)
7577				WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7578
7579			orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7580			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7581			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7582			if (orig != data)
7583				WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7584
7585			orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7586			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7587			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7588			if (orig != data)
7589				WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7590
7591			if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7592				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7593				data &= ~PLL_RAMP_UP_TIME_0_MASK;
7594				if (orig != data)
7595					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7596
7597				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7598				data &= ~PLL_RAMP_UP_TIME_1_MASK;
7599				if (orig != data)
7600					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7601
7602				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7603				data &= ~PLL_RAMP_UP_TIME_2_MASK;
7604				if (orig != data)
7605					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7606
7607				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7608				data &= ~PLL_RAMP_UP_TIME_3_MASK;
7609				if (orig != data)
7610					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7611
7612				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7613				data &= ~PLL_RAMP_UP_TIME_0_MASK;
7614				if (orig != data)
7615					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7616
7617				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7618				data &= ~PLL_RAMP_UP_TIME_1_MASK;
7619				if (orig != data)
7620					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7621
7622				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7623				data &= ~PLL_RAMP_UP_TIME_2_MASK;
7624				if (orig != data)
7625					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7626
7627				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7628				data &= ~PLL_RAMP_UP_TIME_3_MASK;
7629				if (orig != data)
7630					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7631			}
7632			orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7633			data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7634			data |= LC_DYN_LANES_PWR_STATE(3);
7635			if (orig != data)
7636				WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7637
7638			orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7639			data &= ~LS2_EXIT_TIME_MASK;
7640			if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7641				data |= LS2_EXIT_TIME(5);
7642			if (orig != data)
7643				WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7644
7645			orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7646			data &= ~LS2_EXIT_TIME_MASK;
7647			if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7648				data |= LS2_EXIT_TIME(5);
7649			if (orig != data)
7650				WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7651
7652			if (!disable_clkreq &&
7653			    !pci_is_root_bus(rdev->pdev->bus)) {
7654				struct pci_dev *root = rdev->pdev->bus->self;
7655				u32 lnkcap;
7656
7657				clk_req_support = false;
7658				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7659				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7660					clk_req_support = true;
7661			} else {
7662				clk_req_support = false;
7663			}
7664
7665			if (clk_req_support) {
7666				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7667				data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7668				if (orig != data)
7669					WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7670
7671				orig = data = RREG32(THM_CLK_CNTL);
7672				data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7673				data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7674				if (orig != data)
7675					WREG32(THM_CLK_CNTL, data);
7676
7677				orig = data = RREG32(MISC_CLK_CNTL);
7678				data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7679				data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7680				if (orig != data)
7681					WREG32(MISC_CLK_CNTL, data);
7682
7683				orig = data = RREG32(CG_CLKPIN_CNTL);
7684				data &= ~BCLK_AS_XCLK;
7685				if (orig != data)
7686					WREG32(CG_CLKPIN_CNTL, data);
7687
7688				orig = data = RREG32(CG_CLKPIN_CNTL_2);
7689				data &= ~FORCE_BIF_REFCLK_EN;
7690				if (orig != data)
7691					WREG32(CG_CLKPIN_CNTL_2, data);
7692
7693				orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7694				data &= ~MPLL_CLKOUT_SEL_MASK;
7695				data |= MPLL_CLKOUT_SEL(4);
7696				if (orig != data)
7697					WREG32(MPLL_BYPASSCLK_SEL, data);
7698
7699				orig = data = RREG32(SPLL_CNTL_MODE);
7700				data &= ~SPLL_REFCLK_SEL_MASK;
7701				if (orig != data)
7702					WREG32(SPLL_CNTL_MODE, data);
7703			}
7704		}
7705	} else {
7706		if (orig != data)
7707			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7708	}
7709
7710	orig = data = RREG32_PCIE(PCIE_CNTL2);
7711	data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7712	if (orig != data)
7713		WREG32_PCIE(PCIE_CNTL2, data);
7714
7715	if (!disable_l0s) {
7716		data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7717		if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7718			data = RREG32_PCIE(PCIE_LC_STATUS1);
7719			if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7720				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7721				data &= ~LC_L0S_INACTIVITY_MASK;
7722				if (orig != data)
7723					WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7724			}
7725		}
7726	}
7727}
7728
7729int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
7730{
7731	unsigned i;
7732
7733	/* make sure VCEPLL_CTLREQ is deasserted */
7734	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7735
7736	mdelay(10);
7737
7738	/* assert UPLL_CTLREQ */
7739	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
7740
7741	/* wait for CTLACK and CTLACK2 to get asserted */
7742	for (i = 0; i < 100; ++i) {
7743		uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
7744		if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
7745			break;
7746		mdelay(10);
7747	}
7748
7749	/* deassert UPLL_CTLREQ */
7750	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7751
7752	if (i == 100) {
7753		DRM_ERROR("Timeout setting UVD clocks!\n");
7754		return -ETIMEDOUT;
7755	}
7756
7757	return 0;
7758}
7759
7760int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
7761{
7762	unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
7763	int r;
7764
7765	/* bypass evclk and ecclk with bclk */
7766	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7767		     EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
7768		     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7769
7770	/* put PLL in bypass mode */
7771	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
7772		     ~VCEPLL_BYPASS_EN_MASK);
7773
7774	if (!evclk || !ecclk) {
7775		/* keep the Bypass mode, put PLL to sleep */
7776		WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7777			     ~VCEPLL_SLEEP_MASK);
7778		return 0;
7779	}
7780
7781	r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
7782					  16384, 0x03FFFFFF, 0, 128, 5,
7783					  &fb_div, &evclk_div, &ecclk_div);
7784	if (r)
7785		return r;
7786
7787	/* set RESET_ANTI_MUX to 0 */
7788	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7789
7790	/* set VCO_MODE to 1 */
7791	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
7792		     ~VCEPLL_VCO_MODE_MASK);
7793
7794	/* toggle VCEPLL_SLEEP to 1 then back to 0 */
7795	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7796		     ~VCEPLL_SLEEP_MASK);
7797	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
7798
7799	/* deassert VCEPLL_RESET */
7800	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7801
7802	mdelay(1);
7803
7804	r = si_vce_send_vcepll_ctlreq(rdev);
7805	if (r)
7806		return r;
7807
7808	/* assert VCEPLL_RESET again */
7809	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
7810
7811	/* disable spread spectrum. */
7812	WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7813
7814	/* set feedback divider */
7815	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK);
7816
7817	/* set ref divider to 0 */
7818	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
7819
7820	/* set PDIV_A and PDIV_B */
7821	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7822		     VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
7823		     ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
7824
7825	/* give the PLL some time to settle */
7826	mdelay(15);
7827
7828	/* deassert PLL_RESET */
7829	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7830
7831	mdelay(15);
7832
7833	/* switch from bypass mode to normal mode */
7834	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
7835
7836	r = si_vce_send_vcepll_ctlreq(rdev);
7837	if (r)
7838		return r;
7839
7840	/* switch VCLK and DCLK selection */
7841	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7842		     EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
7843		     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7844
7845	mdelay(100);
7846
7847	return 0;
7848}