1/*
   2 * Copyright 2010 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24#include <linux/firmware.h>
 
  25#include <linux/slab.h>
  26#include <drm/drmP.h>
  27#include "radeon.h"
  28#include "radeon_asic.h"
  29#include "radeon_audio.h"
  30#include <drm/radeon_drm.h>
  31#include "evergreend.h"
  32#include "atom.h"
  33#include "avivod.h"
  34#include "evergreen_reg.h"
  35#include "evergreen_blit_shaders.h"
  36#include "radeon_ucode.h"
  37
  38/*
  39 * Indirect registers accessor
  40 */
  41u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
  42{
  43	unsigned long flags;
  44	u32 r;
  45
  46	spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  47	WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  48	r = RREG32(EVERGREEN_CG_IND_DATA);
  49	spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  50	return r;
  51}
  52
  53void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  54{
  55	unsigned long flags;
  56
  57	spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  58	WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  59	WREG32(EVERGREEN_CG_IND_DATA, (v));
  60	spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  61}
  62
  63u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
  64{
  65	unsigned long flags;
  66	u32 r;
  67
  68	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  69	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  70	r = RREG32(EVERGREEN_PIF_PHY0_DATA);
  71	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  72	return r;
  73}
  74
  75void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  76{
  77	unsigned long flags;
  78
  79	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  80	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  81	WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
  82	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  83}
  84
  85u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
  86{
  87	unsigned long flags;
  88	u32 r;
  89
  90	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  91	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
  92	r = RREG32(EVERGREEN_PIF_PHY1_DATA);
  93	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  94	return r;
  95}
  96
  97void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  98{
  99	unsigned long flags;
 100
 101	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 102	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 103	WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
 104	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 105}
 106
 107static const u32 crtc_offsets[6] =
 108{
 109	EVERGREEN_CRTC0_REGISTER_OFFSET,
 110	EVERGREEN_CRTC1_REGISTER_OFFSET,
 111	EVERGREEN_CRTC2_REGISTER_OFFSET,
 112	EVERGREEN_CRTC3_REGISTER_OFFSET,
 113	EVERGREEN_CRTC4_REGISTER_OFFSET,
 114	EVERGREEN_CRTC5_REGISTER_OFFSET
 115};
 116
 117#include "clearstate_evergreen.h"
 118
 119static const u32 sumo_rlc_save_restore_register_list[] =
 120{
 121	0x98fc,
 122	0x9830,
 123	0x9834,
 124	0x9838,
 125	0x9870,
 126	0x9874,
 127	0x8a14,
 128	0x8b24,
 129	0x8bcc,
 130	0x8b10,
 131	0x8d00,
 132	0x8d04,
 133	0x8c00,
 134	0x8c04,
 135	0x8c08,
 136	0x8c0c,
 137	0x8d8c,
 138	0x8c20,
 139	0x8c24,
 140	0x8c28,
 141	0x8c18,
 142	0x8c1c,
 143	0x8cf0,
 144	0x8e2c,
 145	0x8e38,
 146	0x8c30,
 147	0x9508,
 148	0x9688,
 149	0x9608,
 150	0x960c,
 151	0x9610,
 152	0x9614,
 153	0x88c4,
 154	0x88d4,
 155	0xa008,
 156	0x900c,
 157	0x9100,
 158	0x913c,
 159	0x98f8,
 160	0x98f4,
 161	0x9b7c,
 162	0x3f8c,
 163	0x8950,
 164	0x8954,
 165	0x8a18,
 166	0x8b28,
 167	0x9144,
 168	0x9148,
 169	0x914c,
 170	0x3f90,
 171	0x3f94,
 172	0x915c,
 173	0x9160,
 174	0x9178,
 175	0x917c,
 176	0x9180,
 177	0x918c,
 178	0x9190,
 179	0x9194,
 180	0x9198,
 181	0x919c,
 182	0x91a8,
 183	0x91ac,
 184	0x91b0,
 185	0x91b4,
 186	0x91b8,
 187	0x91c4,
 188	0x91c8,
 189	0x91cc,
 190	0x91d0,
 191	0x91d4,
 192	0x91e0,
 193	0x91e4,
 194	0x91ec,
 195	0x91f0,
 196	0x91f4,
 197	0x9200,
 198	0x9204,
 199	0x929c,
 200	0x9150,
 201	0x802c,
 202};
 203
 204static void evergreen_gpu_init(struct radeon_device *rdev);
 205void evergreen_fini(struct radeon_device *rdev);
 206void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 207void evergreen_program_aspm(struct radeon_device *rdev);
 208extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
 209				     int ring, u32 cp_int_cntl);
 210extern void cayman_vm_decode_fault(struct radeon_device *rdev,
 211				   u32 status, u32 addr);
 212void cik_init_cp_pg_table(struct radeon_device *rdev);
 213
 214extern u32 si_get_csb_size(struct radeon_device *rdev);
 215extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 216extern u32 cik_get_csb_size(struct radeon_device *rdev);
 217extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 218extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
 219
 220static const u32 evergreen_golden_registers[] =
 221{
 222	0x3f90, 0xffff0000, 0xff000000,
 223	0x9148, 0xffff0000, 0xff000000,
 224	0x3f94, 0xffff0000, 0xff000000,
 225	0x914c, 0xffff0000, 0xff000000,
 226	0x9b7c, 0xffffffff, 0x00000000,
 227	0x8a14, 0xffffffff, 0x00000007,
 228	0x8b10, 0xffffffff, 0x00000000,
 229	0x960c, 0xffffffff, 0x54763210,
 230	0x88c4, 0xffffffff, 0x000000c2,
 231	0x88d4, 0xffffffff, 0x00000010,
 232	0x8974, 0xffffffff, 0x00000000,
 233	0xc78, 0x00000080, 0x00000080,
 234	0x5eb4, 0xffffffff, 0x00000002,
 235	0x5e78, 0xffffffff, 0x001000f0,
 236	0x6104, 0x01000300, 0x00000000,
 237	0x5bc0, 0x00300000, 0x00000000,
 238	0x7030, 0xffffffff, 0x00000011,
 239	0x7c30, 0xffffffff, 0x00000011,
 240	0x10830, 0xffffffff, 0x00000011,
 241	0x11430, 0xffffffff, 0x00000011,
 242	0x12030, 0xffffffff, 0x00000011,
 243	0x12c30, 0xffffffff, 0x00000011,
 244	0xd02c, 0xffffffff, 0x08421000,
 245	0x240c, 0xffffffff, 0x00000380,
 246	0x8b24, 0xffffffff, 0x00ff0fff,
 247	0x28a4c, 0x06000000, 0x06000000,
 248	0x10c, 0x00000001, 0x00000001,
 249	0x8d00, 0xffffffff, 0x100e4848,
 250	0x8d04, 0xffffffff, 0x00164745,
 251	0x8c00, 0xffffffff, 0xe4000003,
 252	0x8c04, 0xffffffff, 0x40600060,
 253	0x8c08, 0xffffffff, 0x001c001c,
 254	0x8cf0, 0xffffffff, 0x08e00620,
 255	0x8c20, 0xffffffff, 0x00800080,
 256	0x8c24, 0xffffffff, 0x00800080,
 257	0x8c18, 0xffffffff, 0x20202078,
 258	0x8c1c, 0xffffffff, 0x00001010,
 259	0x28350, 0xffffffff, 0x00000000,
 260	0xa008, 0xffffffff, 0x00010000,
 261	0x5c4, 0xffffffff, 0x00000001,
 262	0x9508, 0xffffffff, 0x00000002,
 263	0x913c, 0x0000000f, 0x0000000a
 264};
 265
 266static const u32 evergreen_golden_registers2[] =
 267{
 268	0x2f4c, 0xffffffff, 0x00000000,
 269	0x54f4, 0xffffffff, 0x00000000,
 270	0x54f0, 0xffffffff, 0x00000000,
 271	0x5498, 0xffffffff, 0x00000000,
 272	0x549c, 0xffffffff, 0x00000000,
 273	0x5494, 0xffffffff, 0x00000000,
 274	0x53cc, 0xffffffff, 0x00000000,
 275	0x53c8, 0xffffffff, 0x00000000,
 276	0x53c4, 0xffffffff, 0x00000000,
 277	0x53c0, 0xffffffff, 0x00000000,
 278	0x53bc, 0xffffffff, 0x00000000,
 279	0x53b8, 0xffffffff, 0x00000000,
 280	0x53b4, 0xffffffff, 0x00000000,
 281	0x53b0, 0xffffffff, 0x00000000
 282};
 283
 284static const u32 cypress_mgcg_init[] =
 285{
 286	0x802c, 0xffffffff, 0xc0000000,
 287	0x5448, 0xffffffff, 0x00000100,
 288	0x55e4, 0xffffffff, 0x00000100,
 289	0x160c, 0xffffffff, 0x00000100,
 290	0x5644, 0xffffffff, 0x00000100,
 291	0xc164, 0xffffffff, 0x00000100,
 292	0x8a18, 0xffffffff, 0x00000100,
 293	0x897c, 0xffffffff, 0x06000100,
 294	0x8b28, 0xffffffff, 0x00000100,
 295	0x9144, 0xffffffff, 0x00000100,
 296	0x9a60, 0xffffffff, 0x00000100,
 297	0x9868, 0xffffffff, 0x00000100,
 298	0x8d58, 0xffffffff, 0x00000100,
 299	0x9510, 0xffffffff, 0x00000100,
 300	0x949c, 0xffffffff, 0x00000100,
 301	0x9654, 0xffffffff, 0x00000100,
 302	0x9030, 0xffffffff, 0x00000100,
 303	0x9034, 0xffffffff, 0x00000100,
 304	0x9038, 0xffffffff, 0x00000100,
 305	0x903c, 0xffffffff, 0x00000100,
 306	0x9040, 0xffffffff, 0x00000100,
 307	0xa200, 0xffffffff, 0x00000100,
 308	0xa204, 0xffffffff, 0x00000100,
 309	0xa208, 0xffffffff, 0x00000100,
 310	0xa20c, 0xffffffff, 0x00000100,
 311	0x971c, 0xffffffff, 0x00000100,
 312	0x977c, 0xffffffff, 0x00000100,
 313	0x3f80, 0xffffffff, 0x00000100,
 314	0xa210, 0xffffffff, 0x00000100,
 315	0xa214, 0xffffffff, 0x00000100,
 316	0x4d8, 0xffffffff, 0x00000100,
 317	0x9784, 0xffffffff, 0x00000100,
 318	0x9698, 0xffffffff, 0x00000100,
 319	0x4d4, 0xffffffff, 0x00000200,
 320	0x30cc, 0xffffffff, 0x00000100,
 321	0xd0c0, 0xffffffff, 0xff000100,
 322	0x802c, 0xffffffff, 0x40000000,
 323	0x915c, 0xffffffff, 0x00010000,
 324	0x9160, 0xffffffff, 0x00030002,
 325	0x9178, 0xffffffff, 0x00070000,
 326	0x917c, 0xffffffff, 0x00030002,
 327	0x9180, 0xffffffff, 0x00050004,
 328	0x918c, 0xffffffff, 0x00010006,
 329	0x9190, 0xffffffff, 0x00090008,
 330	0x9194, 0xffffffff, 0x00070000,
 331	0x9198, 0xffffffff, 0x00030002,
 332	0x919c, 0xffffffff, 0x00050004,
 333	0x91a8, 0xffffffff, 0x00010006,
 334	0x91ac, 0xffffffff, 0x00090008,
 335	0x91b0, 0xffffffff, 0x00070000,
 336	0x91b4, 0xffffffff, 0x00030002,
 337	0x91b8, 0xffffffff, 0x00050004,
 338	0x91c4, 0xffffffff, 0x00010006,
 339	0x91c8, 0xffffffff, 0x00090008,
 340	0x91cc, 0xffffffff, 0x00070000,
 341	0x91d0, 0xffffffff, 0x00030002,
 342	0x91d4, 0xffffffff, 0x00050004,
 343	0x91e0, 0xffffffff, 0x00010006,
 344	0x91e4, 0xffffffff, 0x00090008,
 345	0x91e8, 0xffffffff, 0x00000000,
 346	0x91ec, 0xffffffff, 0x00070000,
 347	0x91f0, 0xffffffff, 0x00030002,
 348	0x91f4, 0xffffffff, 0x00050004,
 349	0x9200, 0xffffffff, 0x00010006,
 350	0x9204, 0xffffffff, 0x00090008,
 351	0x9208, 0xffffffff, 0x00070000,
 352	0x920c, 0xffffffff, 0x00030002,
 353	0x9210, 0xffffffff, 0x00050004,
 354	0x921c, 0xffffffff, 0x00010006,
 355	0x9220, 0xffffffff, 0x00090008,
 356	0x9224, 0xffffffff, 0x00070000,
 357	0x9228, 0xffffffff, 0x00030002,
 358	0x922c, 0xffffffff, 0x00050004,
 359	0x9238, 0xffffffff, 0x00010006,
 360	0x923c, 0xffffffff, 0x00090008,
 361	0x9240, 0xffffffff, 0x00070000,
 362	0x9244, 0xffffffff, 0x00030002,
 363	0x9248, 0xffffffff, 0x00050004,
 364	0x9254, 0xffffffff, 0x00010006,
 365	0x9258, 0xffffffff, 0x00090008,
 366	0x925c, 0xffffffff, 0x00070000,
 367	0x9260, 0xffffffff, 0x00030002,
 368	0x9264, 0xffffffff, 0x00050004,
 369	0x9270, 0xffffffff, 0x00010006,
 370	0x9274, 0xffffffff, 0x00090008,
 371	0x9278, 0xffffffff, 0x00070000,
 372	0x927c, 0xffffffff, 0x00030002,
 373	0x9280, 0xffffffff, 0x00050004,
 374	0x928c, 0xffffffff, 0x00010006,
 375	0x9290, 0xffffffff, 0x00090008,
 376	0x9294, 0xffffffff, 0x00000000,
 377	0x929c, 0xffffffff, 0x00000001,
 378	0x802c, 0xffffffff, 0x40010000,
 379	0x915c, 0xffffffff, 0x00010000,
 380	0x9160, 0xffffffff, 0x00030002,
 381	0x9178, 0xffffffff, 0x00070000,
 382	0x917c, 0xffffffff, 0x00030002,
 383	0x9180, 0xffffffff, 0x00050004,
 384	0x918c, 0xffffffff, 0x00010006,
 385	0x9190, 0xffffffff, 0x00090008,
 386	0x9194, 0xffffffff, 0x00070000,
 387	0x9198, 0xffffffff, 0x00030002,
 388	0x919c, 0xffffffff, 0x00050004,
 389	0x91a8, 0xffffffff, 0x00010006,
 390	0x91ac, 0xffffffff, 0x00090008,
 391	0x91b0, 0xffffffff, 0x00070000,
 392	0x91b4, 0xffffffff, 0x00030002,
 393	0x91b8, 0xffffffff, 0x00050004,
 394	0x91c4, 0xffffffff, 0x00010006,
 395	0x91c8, 0xffffffff, 0x00090008,
 396	0x91cc, 0xffffffff, 0x00070000,
 397	0x91d0, 0xffffffff, 0x00030002,
 398	0x91d4, 0xffffffff, 0x00050004,
 399	0x91e0, 0xffffffff, 0x00010006,
 400	0x91e4, 0xffffffff, 0x00090008,
 401	0x91e8, 0xffffffff, 0x00000000,
 402	0x91ec, 0xffffffff, 0x00070000,
 403	0x91f0, 0xffffffff, 0x00030002,
 404	0x91f4, 0xffffffff, 0x00050004,
 405	0x9200, 0xffffffff, 0x00010006,
 406	0x9204, 0xffffffff, 0x00090008,
 407	0x9208, 0xffffffff, 0x00070000,
 408	0x920c, 0xffffffff, 0x00030002,
 409	0x9210, 0xffffffff, 0x00050004,
 410	0x921c, 0xffffffff, 0x00010006,
 411	0x9220, 0xffffffff, 0x00090008,
 412	0x9224, 0xffffffff, 0x00070000,
 413	0x9228, 0xffffffff, 0x00030002,
 414	0x922c, 0xffffffff, 0x00050004,
 415	0x9238, 0xffffffff, 0x00010006,
 416	0x923c, 0xffffffff, 0x00090008,
 417	0x9240, 0xffffffff, 0x00070000,
 418	0x9244, 0xffffffff, 0x00030002,
 419	0x9248, 0xffffffff, 0x00050004,
 420	0x9254, 0xffffffff, 0x00010006,
 421	0x9258, 0xffffffff, 0x00090008,
 422	0x925c, 0xffffffff, 0x00070000,
 423	0x9260, 0xffffffff, 0x00030002,
 424	0x9264, 0xffffffff, 0x00050004,
 425	0x9270, 0xffffffff, 0x00010006,
 426	0x9274, 0xffffffff, 0x00090008,
 427	0x9278, 0xffffffff, 0x00070000,
 428	0x927c, 0xffffffff, 0x00030002,
 429	0x9280, 0xffffffff, 0x00050004,
 430	0x928c, 0xffffffff, 0x00010006,
 431	0x9290, 0xffffffff, 0x00090008,
 432	0x9294, 0xffffffff, 0x00000000,
 433	0x929c, 0xffffffff, 0x00000001,
 434	0x802c, 0xffffffff, 0xc0000000
 435};
 436
 437static const u32 redwood_mgcg_init[] =
 438{
 439	0x802c, 0xffffffff, 0xc0000000,
 440	0x5448, 0xffffffff, 0x00000100,
 441	0x55e4, 0xffffffff, 0x00000100,
 442	0x160c, 0xffffffff, 0x00000100,
 443	0x5644, 0xffffffff, 0x00000100,
 444	0xc164, 0xffffffff, 0x00000100,
 445	0x8a18, 0xffffffff, 0x00000100,
 446	0x897c, 0xffffffff, 0x06000100,
 447	0x8b28, 0xffffffff, 0x00000100,
 448	0x9144, 0xffffffff, 0x00000100,
 449	0x9a60, 0xffffffff, 0x00000100,
 450	0x9868, 0xffffffff, 0x00000100,
 451	0x8d58, 0xffffffff, 0x00000100,
 452	0x9510, 0xffffffff, 0x00000100,
 453	0x949c, 0xffffffff, 0x00000100,
 454	0x9654, 0xffffffff, 0x00000100,
 455	0x9030, 0xffffffff, 0x00000100,
 456	0x9034, 0xffffffff, 0x00000100,
 457	0x9038, 0xffffffff, 0x00000100,
 458	0x903c, 0xffffffff, 0x00000100,
 459	0x9040, 0xffffffff, 0x00000100,
 460	0xa200, 0xffffffff, 0x00000100,
 461	0xa204, 0xffffffff, 0x00000100,
 462	0xa208, 0xffffffff, 0x00000100,
 463	0xa20c, 0xffffffff, 0x00000100,
 464	0x971c, 0xffffffff, 0x00000100,
 465	0x977c, 0xffffffff, 0x00000100,
 466	0x3f80, 0xffffffff, 0x00000100,
 467	0xa210, 0xffffffff, 0x00000100,
 468	0xa214, 0xffffffff, 0x00000100,
 469	0x4d8, 0xffffffff, 0x00000100,
 470	0x9784, 0xffffffff, 0x00000100,
 471	0x9698, 0xffffffff, 0x00000100,
 472	0x4d4, 0xffffffff, 0x00000200,
 473	0x30cc, 0xffffffff, 0x00000100,
 474	0xd0c0, 0xffffffff, 0xff000100,
 475	0x802c, 0xffffffff, 0x40000000,
 476	0x915c, 0xffffffff, 0x00010000,
 477	0x9160, 0xffffffff, 0x00030002,
 478	0x9178, 0xffffffff, 0x00070000,
 479	0x917c, 0xffffffff, 0x00030002,
 480	0x9180, 0xffffffff, 0x00050004,
 481	0x918c, 0xffffffff, 0x00010006,
 482	0x9190, 0xffffffff, 0x00090008,
 483	0x9194, 0xffffffff, 0x00070000,
 484	0x9198, 0xffffffff, 0x00030002,
 485	0x919c, 0xffffffff, 0x00050004,
 486	0x91a8, 0xffffffff, 0x00010006,
 487	0x91ac, 0xffffffff, 0x00090008,
 488	0x91b0, 0xffffffff, 0x00070000,
 489	0x91b4, 0xffffffff, 0x00030002,
 490	0x91b8, 0xffffffff, 0x00050004,
 491	0x91c4, 0xffffffff, 0x00010006,
 492	0x91c8, 0xffffffff, 0x00090008,
 493	0x91cc, 0xffffffff, 0x00070000,
 494	0x91d0, 0xffffffff, 0x00030002,
 495	0x91d4, 0xffffffff, 0x00050004,
 496	0x91e0, 0xffffffff, 0x00010006,
 497	0x91e4, 0xffffffff, 0x00090008,
 498	0x91e8, 0xffffffff, 0x00000000,
 499	0x91ec, 0xffffffff, 0x00070000,
 500	0x91f0, 0xffffffff, 0x00030002,
 501	0x91f4, 0xffffffff, 0x00050004,
 502	0x9200, 0xffffffff, 0x00010006,
 503	0x9204, 0xffffffff, 0x00090008,
 504	0x9294, 0xffffffff, 0x00000000,
 505	0x929c, 0xffffffff, 0x00000001,
 506	0x802c, 0xffffffff, 0xc0000000
 507};
 508
 509static const u32 cedar_golden_registers[] =
 510{
 511	0x3f90, 0xffff0000, 0xff000000,
 512	0x9148, 0xffff0000, 0xff000000,
 513	0x3f94, 0xffff0000, 0xff000000,
 514	0x914c, 0xffff0000, 0xff000000,
 515	0x9b7c, 0xffffffff, 0x00000000,
 516	0x8a14, 0xffffffff, 0x00000007,
 517	0x8b10, 0xffffffff, 0x00000000,
 518	0x960c, 0xffffffff, 0x54763210,
 519	0x88c4, 0xffffffff, 0x000000c2,
 520	0x88d4, 0xffffffff, 0x00000000,
 521	0x8974, 0xffffffff, 0x00000000,
 522	0xc78, 0x00000080, 0x00000080,
 523	0x5eb4, 0xffffffff, 0x00000002,
 524	0x5e78, 0xffffffff, 0x001000f0,
 525	0x6104, 0x01000300, 0x00000000,
 526	0x5bc0, 0x00300000, 0x00000000,
 527	0x7030, 0xffffffff, 0x00000011,
 528	0x7c30, 0xffffffff, 0x00000011,
 529	0x10830, 0xffffffff, 0x00000011,
 530	0x11430, 0xffffffff, 0x00000011,
 531	0xd02c, 0xffffffff, 0x08421000,
 532	0x240c, 0xffffffff, 0x00000380,
 533	0x8b24, 0xffffffff, 0x00ff0fff,
 534	0x28a4c, 0x06000000, 0x06000000,
 535	0x10c, 0x00000001, 0x00000001,
 536	0x8d00, 0xffffffff, 0x100e4848,
 537	0x8d04, 0xffffffff, 0x00164745,
 538	0x8c00, 0xffffffff, 0xe4000003,
 539	0x8c04, 0xffffffff, 0x40600060,
 540	0x8c08, 0xffffffff, 0x001c001c,
 541	0x8cf0, 0xffffffff, 0x08e00410,
 542	0x8c20, 0xffffffff, 0x00800080,
 543	0x8c24, 0xffffffff, 0x00800080,
 544	0x8c18, 0xffffffff, 0x20202078,
 545	0x8c1c, 0xffffffff, 0x00001010,
 546	0x28350, 0xffffffff, 0x00000000,
 547	0xa008, 0xffffffff, 0x00010000,
 548	0x5c4, 0xffffffff, 0x00000001,
 549	0x9508, 0xffffffff, 0x00000002
 550};
 551
 552static const u32 cedar_mgcg_init[] =
 553{
 554	0x802c, 0xffffffff, 0xc0000000,
 555	0x5448, 0xffffffff, 0x00000100,
 556	0x55e4, 0xffffffff, 0x00000100,
 557	0x160c, 0xffffffff, 0x00000100,
 558	0x5644, 0xffffffff, 0x00000100,
 559	0xc164, 0xffffffff, 0x00000100,
 560	0x8a18, 0xffffffff, 0x00000100,
 561	0x897c, 0xffffffff, 0x06000100,
 562	0x8b28, 0xffffffff, 0x00000100,
 563	0x9144, 0xffffffff, 0x00000100,
 564	0x9a60, 0xffffffff, 0x00000100,
 565	0x9868, 0xffffffff, 0x00000100,
 566	0x8d58, 0xffffffff, 0x00000100,
 567	0x9510, 0xffffffff, 0x00000100,
 568	0x949c, 0xffffffff, 0x00000100,
 569	0x9654, 0xffffffff, 0x00000100,
 570	0x9030, 0xffffffff, 0x00000100,
 571	0x9034, 0xffffffff, 0x00000100,
 572	0x9038, 0xffffffff, 0x00000100,
 573	0x903c, 0xffffffff, 0x00000100,
 574	0x9040, 0xffffffff, 0x00000100,
 575	0xa200, 0xffffffff, 0x00000100,
 576	0xa204, 0xffffffff, 0x00000100,
 577	0xa208, 0xffffffff, 0x00000100,
 578	0xa20c, 0xffffffff, 0x00000100,
 579	0x971c, 0xffffffff, 0x00000100,
 580	0x977c, 0xffffffff, 0x00000100,
 581	0x3f80, 0xffffffff, 0x00000100,
 582	0xa210, 0xffffffff, 0x00000100,
 583	0xa214, 0xffffffff, 0x00000100,
 584	0x4d8, 0xffffffff, 0x00000100,
 585	0x9784, 0xffffffff, 0x00000100,
 586	0x9698, 0xffffffff, 0x00000100,
 587	0x4d4, 0xffffffff, 0x00000200,
 588	0x30cc, 0xffffffff, 0x00000100,
 589	0xd0c0, 0xffffffff, 0xff000100,
 590	0x802c, 0xffffffff, 0x40000000,
 591	0x915c, 0xffffffff, 0x00010000,
 592	0x9178, 0xffffffff, 0x00050000,
 593	0x917c, 0xffffffff, 0x00030002,
 594	0x918c, 0xffffffff, 0x00010004,
 595	0x9190, 0xffffffff, 0x00070006,
 596	0x9194, 0xffffffff, 0x00050000,
 597	0x9198, 0xffffffff, 0x00030002,
 598	0x91a8, 0xffffffff, 0x00010004,
 599	0x91ac, 0xffffffff, 0x00070006,
 600	0x91e8, 0xffffffff, 0x00000000,
 601	0x9294, 0xffffffff, 0x00000000,
 602	0x929c, 0xffffffff, 0x00000001,
 603	0x802c, 0xffffffff, 0xc0000000
 604};
 605
 606static const u32 juniper_mgcg_init[] =
 607{
 608	0x802c, 0xffffffff, 0xc0000000,
 609	0x5448, 0xffffffff, 0x00000100,
 610	0x55e4, 0xffffffff, 0x00000100,
 611	0x160c, 0xffffffff, 0x00000100,
 612	0x5644, 0xffffffff, 0x00000100,
 613	0xc164, 0xffffffff, 0x00000100,
 614	0x8a18, 0xffffffff, 0x00000100,
 615	0x897c, 0xffffffff, 0x06000100,
 616	0x8b28, 0xffffffff, 0x00000100,
 617	0x9144, 0xffffffff, 0x00000100,
 618	0x9a60, 0xffffffff, 0x00000100,
 619	0x9868, 0xffffffff, 0x00000100,
 620	0x8d58, 0xffffffff, 0x00000100,
 621	0x9510, 0xffffffff, 0x00000100,
 622	0x949c, 0xffffffff, 0x00000100,
 623	0x9654, 0xffffffff, 0x00000100,
 624	0x9030, 0xffffffff, 0x00000100,
 625	0x9034, 0xffffffff, 0x00000100,
 626	0x9038, 0xffffffff, 0x00000100,
 627	0x903c, 0xffffffff, 0x00000100,
 628	0x9040, 0xffffffff, 0x00000100,
 629	0xa200, 0xffffffff, 0x00000100,
 630	0xa204, 0xffffffff, 0x00000100,
 631	0xa208, 0xffffffff, 0x00000100,
 632	0xa20c, 0xffffffff, 0x00000100,
 633	0x971c, 0xffffffff, 0x00000100,
 634	0xd0c0, 0xffffffff, 0xff000100,
 635	0x802c, 0xffffffff, 0x40000000,
 636	0x915c, 0xffffffff, 0x00010000,
 637	0x9160, 0xffffffff, 0x00030002,
 638	0x9178, 0xffffffff, 0x00070000,
 639	0x917c, 0xffffffff, 0x00030002,
 640	0x9180, 0xffffffff, 0x00050004,
 641	0x918c, 0xffffffff, 0x00010006,
 642	0x9190, 0xffffffff, 0x00090008,
 643	0x9194, 0xffffffff, 0x00070000,
 644	0x9198, 0xffffffff, 0x00030002,
 645	0x919c, 0xffffffff, 0x00050004,
 646	0x91a8, 0xffffffff, 0x00010006,
 647	0x91ac, 0xffffffff, 0x00090008,
 648	0x91b0, 0xffffffff, 0x00070000,
 649	0x91b4, 0xffffffff, 0x00030002,
 650	0x91b8, 0xffffffff, 0x00050004,
 651	0x91c4, 0xffffffff, 0x00010006,
 652	0x91c8, 0xffffffff, 0x00090008,
 653	0x91cc, 0xffffffff, 0x00070000,
 654	0x91d0, 0xffffffff, 0x00030002,
 655	0x91d4, 0xffffffff, 0x00050004,
 656	0x91e0, 0xffffffff, 0x00010006,
 657	0x91e4, 0xffffffff, 0x00090008,
 658	0x91e8, 0xffffffff, 0x00000000,
 659	0x91ec, 0xffffffff, 0x00070000,
 660	0x91f0, 0xffffffff, 0x00030002,
 661	0x91f4, 0xffffffff, 0x00050004,
 662	0x9200, 0xffffffff, 0x00010006,
 663	0x9204, 0xffffffff, 0x00090008,
 664	0x9208, 0xffffffff, 0x00070000,
 665	0x920c, 0xffffffff, 0x00030002,
 666	0x9210, 0xffffffff, 0x00050004,
 667	0x921c, 0xffffffff, 0x00010006,
 668	0x9220, 0xffffffff, 0x00090008,
 669	0x9224, 0xffffffff, 0x00070000,
 670	0x9228, 0xffffffff, 0x00030002,
 671	0x922c, 0xffffffff, 0x00050004,
 672	0x9238, 0xffffffff, 0x00010006,
 673	0x923c, 0xffffffff, 0x00090008,
 674	0x9240, 0xffffffff, 0x00070000,
 675	0x9244, 0xffffffff, 0x00030002,
 676	0x9248, 0xffffffff, 0x00050004,
 677	0x9254, 0xffffffff, 0x00010006,
 678	0x9258, 0xffffffff, 0x00090008,
 679	0x925c, 0xffffffff, 0x00070000,
 680	0x9260, 0xffffffff, 0x00030002,
 681	0x9264, 0xffffffff, 0x00050004,
 682	0x9270, 0xffffffff, 0x00010006,
 683	0x9274, 0xffffffff, 0x00090008,
 684	0x9278, 0xffffffff, 0x00070000,
 685	0x927c, 0xffffffff, 0x00030002,
 686	0x9280, 0xffffffff, 0x00050004,
 687	0x928c, 0xffffffff, 0x00010006,
 688	0x9290, 0xffffffff, 0x00090008,
 689	0x9294, 0xffffffff, 0x00000000,
 690	0x929c, 0xffffffff, 0x00000001,
 691	0x802c, 0xffffffff, 0xc0000000,
 692	0x977c, 0xffffffff, 0x00000100,
 693	0x3f80, 0xffffffff, 0x00000100,
 694	0xa210, 0xffffffff, 0x00000100,
 695	0xa214, 0xffffffff, 0x00000100,
 696	0x4d8, 0xffffffff, 0x00000100,
 697	0x9784, 0xffffffff, 0x00000100,
 698	0x9698, 0xffffffff, 0x00000100,
 699	0x4d4, 0xffffffff, 0x00000200,
 700	0x30cc, 0xffffffff, 0x00000100,
 701	0x802c, 0xffffffff, 0xc0000000
 702};
 703
 704static const u32 supersumo_golden_registers[] =
 705{
 706	0x5eb4, 0xffffffff, 0x00000002,
 707	0x5c4, 0xffffffff, 0x00000001,
 708	0x7030, 0xffffffff, 0x00000011,
 709	0x7c30, 0xffffffff, 0x00000011,
 710	0x6104, 0x01000300, 0x00000000,
 711	0x5bc0, 0x00300000, 0x00000000,
 712	0x8c04, 0xffffffff, 0x40600060,
 713	0x8c08, 0xffffffff, 0x001c001c,
 714	0x8c20, 0xffffffff, 0x00800080,
 715	0x8c24, 0xffffffff, 0x00800080,
 716	0x8c18, 0xffffffff, 0x20202078,
 717	0x8c1c, 0xffffffff, 0x00001010,
 718	0x918c, 0xffffffff, 0x00010006,
 719	0x91a8, 0xffffffff, 0x00010006,
 720	0x91c4, 0xffffffff, 0x00010006,
 721	0x91e0, 0xffffffff, 0x00010006,
 722	0x9200, 0xffffffff, 0x00010006,
 723	0x9150, 0xffffffff, 0x6e944040,
 724	0x917c, 0xffffffff, 0x00030002,
 725	0x9180, 0xffffffff, 0x00050004,
 726	0x9198, 0xffffffff, 0x00030002,
 727	0x919c, 0xffffffff, 0x00050004,
 728	0x91b4, 0xffffffff, 0x00030002,
 729	0x91b8, 0xffffffff, 0x00050004,
 730	0x91d0, 0xffffffff, 0x00030002,
 731	0x91d4, 0xffffffff, 0x00050004,
 732	0x91f0, 0xffffffff, 0x00030002,
 733	0x91f4, 0xffffffff, 0x00050004,
 734	0x915c, 0xffffffff, 0x00010000,
 735	0x9160, 0xffffffff, 0x00030002,
 736	0x3f90, 0xffff0000, 0xff000000,
 737	0x9178, 0xffffffff, 0x00070000,
 738	0x9194, 0xffffffff, 0x00070000,
 739	0x91b0, 0xffffffff, 0x00070000,
 740	0x91cc, 0xffffffff, 0x00070000,
 741	0x91ec, 0xffffffff, 0x00070000,
 742	0x9148, 0xffff0000, 0xff000000,
 743	0x9190, 0xffffffff, 0x00090008,
 744	0x91ac, 0xffffffff, 0x00090008,
 745	0x91c8, 0xffffffff, 0x00090008,
 746	0x91e4, 0xffffffff, 0x00090008,
 747	0x9204, 0xffffffff, 0x00090008,
 748	0x3f94, 0xffff0000, 0xff000000,
 749	0x914c, 0xffff0000, 0xff000000,
 750	0x929c, 0xffffffff, 0x00000001,
 751	0x8a18, 0xffffffff, 0x00000100,
 752	0x8b28, 0xffffffff, 0x00000100,
 753	0x9144, 0xffffffff, 0x00000100,
 754	0x5644, 0xffffffff, 0x00000100,
 755	0x9b7c, 0xffffffff, 0x00000000,
 756	0x8030, 0xffffffff, 0x0000100a,
 757	0x8a14, 0xffffffff, 0x00000007,
 758	0x8b24, 0xffffffff, 0x00ff0fff,
 759	0x8b10, 0xffffffff, 0x00000000,
 760	0x28a4c, 0x06000000, 0x06000000,
 761	0x4d8, 0xffffffff, 0x00000100,
 762	0x913c, 0xffff000f, 0x0100000a,
 763	0x960c, 0xffffffff, 0x54763210,
 764	0x88c4, 0xffffffff, 0x000000c2,
 765	0x88d4, 0xffffffff, 0x00000010,
 766	0x8974, 0xffffffff, 0x00000000,
 767	0xc78, 0x00000080, 0x00000080,
 768	0x5e78, 0xffffffff, 0x001000f0,
 769	0xd02c, 0xffffffff, 0x08421000,
 770	0xa008, 0xffffffff, 0x00010000,
 771	0x8d00, 0xffffffff, 0x100e4848,
 772	0x8d04, 0xffffffff, 0x00164745,
 773	0x8c00, 0xffffffff, 0xe4000003,
 774	0x8cf0, 0x1fffffff, 0x08e00620,
 775	0x28350, 0xffffffff, 0x00000000,
 776	0x9508, 0xffffffff, 0x00000002
 777};
 778
 779static const u32 sumo_golden_registers[] =
 780{
 781	0x900c, 0x00ffffff, 0x0017071f,
 782	0x8c18, 0xffffffff, 0x10101060,
 783	0x8c1c, 0xffffffff, 0x00001010,
 784	0x8c30, 0x0000000f, 0x00000005,
 785	0x9688, 0x0000000f, 0x00000007
 786};
 787
 788static const u32 wrestler_golden_registers[] =
 789{
 790	0x5eb4, 0xffffffff, 0x00000002,
 791	0x5c4, 0xffffffff, 0x00000001,
 792	0x7030, 0xffffffff, 0x00000011,
 793	0x7c30, 0xffffffff, 0x00000011,
 794	0x6104, 0x01000300, 0x00000000,
 795	0x5bc0, 0x00300000, 0x00000000,
 796	0x918c, 0xffffffff, 0x00010006,
 797	0x91a8, 0xffffffff, 0x00010006,
 798	0x9150, 0xffffffff, 0x6e944040,
 799	0x917c, 0xffffffff, 0x00030002,
 800	0x9198, 0xffffffff, 0x00030002,
 801	0x915c, 0xffffffff, 0x00010000,
 802	0x3f90, 0xffff0000, 0xff000000,
 803	0x9178, 0xffffffff, 0x00070000,
 804	0x9194, 0xffffffff, 0x00070000,
 805	0x9148, 0xffff0000, 0xff000000,
 806	0x9190, 0xffffffff, 0x00090008,
 807	0x91ac, 0xffffffff, 0x00090008,
 808	0x3f94, 0xffff0000, 0xff000000,
 809	0x914c, 0xffff0000, 0xff000000,
 810	0x929c, 0xffffffff, 0x00000001,
 811	0x8a18, 0xffffffff, 0x00000100,
 812	0x8b28, 0xffffffff, 0x00000100,
 813	0x9144, 0xffffffff, 0x00000100,
 814	0x9b7c, 0xffffffff, 0x00000000,
 815	0x8030, 0xffffffff, 0x0000100a,
 816	0x8a14, 0xffffffff, 0x00000001,
 817	0x8b24, 0xffffffff, 0x00ff0fff,
 818	0x8b10, 0xffffffff, 0x00000000,
 819	0x28a4c, 0x06000000, 0x06000000,
 820	0x4d8, 0xffffffff, 0x00000100,
 821	0x913c, 0xffff000f, 0x0100000a,
 822	0x960c, 0xffffffff, 0x54763210,
 823	0x88c4, 0xffffffff, 0x000000c2,
 824	0x88d4, 0xffffffff, 0x00000010,
 825	0x8974, 0xffffffff, 0x00000000,
 826	0xc78, 0x00000080, 0x00000080,
 827	0x5e78, 0xffffffff, 0x001000f0,
 828	0xd02c, 0xffffffff, 0x08421000,
 829	0xa008, 0xffffffff, 0x00010000,
 830	0x8d00, 0xffffffff, 0x100e4848,
 831	0x8d04, 0xffffffff, 0x00164745,
 832	0x8c00, 0xffffffff, 0xe4000003,
 833	0x8cf0, 0x1fffffff, 0x08e00410,
 834	0x28350, 0xffffffff, 0x00000000,
 835	0x9508, 0xffffffff, 0x00000002,
 836	0x900c, 0xffffffff, 0x0017071f,
 837	0x8c18, 0xffffffff, 0x10101060,
 838	0x8c1c, 0xffffffff, 0x00001010
 839};
 840
 841static const u32 barts_golden_registers[] =
 842{
 843	0x5eb4, 0xffffffff, 0x00000002,
 844	0x5e78, 0x8f311ff1, 0x001000f0,
 845	0x3f90, 0xffff0000, 0xff000000,
 846	0x9148, 0xffff0000, 0xff000000,
 847	0x3f94, 0xffff0000, 0xff000000,
 848	0x914c, 0xffff0000, 0xff000000,
 849	0xc78, 0x00000080, 0x00000080,
 850	0xbd4, 0x70073777, 0x00010001,
 851	0xd02c, 0xbfffff1f, 0x08421000,
 852	0xd0b8, 0x03773777, 0x02011003,
 853	0x5bc0, 0x00200000, 0x50100000,
 854	0x98f8, 0x33773777, 0x02011003,
 855	0x98fc, 0xffffffff, 0x76543210,
 856	0x7030, 0x31000311, 0x00000011,
 857	0x2f48, 0x00000007, 0x02011003,
 858	0x6b28, 0x00000010, 0x00000012,
 859	0x7728, 0x00000010, 0x00000012,
 860	0x10328, 0x00000010, 0x00000012,
 861	0x10f28, 0x00000010, 0x00000012,
 862	0x11b28, 0x00000010, 0x00000012,
 863	0x12728, 0x00000010, 0x00000012,
 864	0x240c, 0x000007ff, 0x00000380,
 865	0x8a14, 0xf000001f, 0x00000007,
 866	0x8b24, 0x3fff3fff, 0x00ff0fff,
 867	0x8b10, 0x0000ff0f, 0x00000000,
 868	0x28a4c, 0x07ffffff, 0x06000000,
 869	0x10c, 0x00000001, 0x00010003,
 870	0xa02c, 0xffffffff, 0x0000009b,
 871	0x913c, 0x0000000f, 0x0100000a,
 872	0x8d00, 0xffff7f7f, 0x100e4848,
 873	0x8d04, 0x00ffffff, 0x00164745,
 874	0x8c00, 0xfffc0003, 0xe4000003,
 875	0x8c04, 0xf8ff00ff, 0x40600060,
 876	0x8c08, 0x00ff00ff, 0x001c001c,
 877	0x8cf0, 0x1fff1fff, 0x08e00620,
 878	0x8c20, 0x0fff0fff, 0x00800080,
 879	0x8c24, 0x0fff0fff, 0x00800080,
 880	0x8c18, 0xffffffff, 0x20202078,
 881	0x8c1c, 0x0000ffff, 0x00001010,
 882	0x28350, 0x00000f01, 0x00000000,
 883	0x9508, 0x3700001f, 0x00000002,
 884	0x960c, 0xffffffff, 0x54763210,
 885	0x88c4, 0x001f3ae3, 0x000000c2,
 886	0x88d4, 0x0000001f, 0x00000010,
 887	0x8974, 0xffffffff, 0x00000000
 888};
 889
 890static const u32 turks_golden_registers[] =
 891{
 892	0x5eb4, 0xffffffff, 0x00000002,
 893	0x5e78, 0x8f311ff1, 0x001000f0,
 894	0x8c8, 0x00003000, 0x00001070,
 895	0x8cc, 0x000fffff, 0x00040035,
 896	0x3f90, 0xffff0000, 0xfff00000,
 897	0x9148, 0xffff0000, 0xfff00000,
 898	0x3f94, 0xffff0000, 0xfff00000,
 899	0x914c, 0xffff0000, 0xfff00000,
 900	0xc78, 0x00000080, 0x00000080,
 901	0xbd4, 0x00073007, 0x00010002,
 902	0xd02c, 0xbfffff1f, 0x08421000,
 903	0xd0b8, 0x03773777, 0x02010002,
 904	0x5bc0, 0x00200000, 0x50100000,
 905	0x98f8, 0x33773777, 0x00010002,
 906	0x98fc, 0xffffffff, 0x33221100,
 907	0x7030, 0x31000311, 0x00000011,
 908	0x2f48, 0x33773777, 0x00010002,
 909	0x6b28, 0x00000010, 0x00000012,
 910	0x7728, 0x00000010, 0x00000012,
 911	0x10328, 0x00000010, 0x00000012,
 912	0x10f28, 0x00000010, 0x00000012,
 913	0x11b28, 0x00000010, 0x00000012,
 914	0x12728, 0x00000010, 0x00000012,
 915	0x240c, 0x000007ff, 0x00000380,
 916	0x8a14, 0xf000001f, 0x00000007,
 917	0x8b24, 0x3fff3fff, 0x00ff0fff,
 918	0x8b10, 0x0000ff0f, 0x00000000,
 919	0x28a4c, 0x07ffffff, 0x06000000,
 920	0x10c, 0x00000001, 0x00010003,
 921	0xa02c, 0xffffffff, 0x0000009b,
 922	0x913c, 0x0000000f, 0x0100000a,
 923	0x8d00, 0xffff7f7f, 0x100e4848,
 924	0x8d04, 0x00ffffff, 0x00164745,
 925	0x8c00, 0xfffc0003, 0xe4000003,
 926	0x8c04, 0xf8ff00ff, 0x40600060,
 927	0x8c08, 0x00ff00ff, 0x001c001c,
 928	0x8cf0, 0x1fff1fff, 0x08e00410,
 929	0x8c20, 0x0fff0fff, 0x00800080,
 930	0x8c24, 0x0fff0fff, 0x00800080,
 931	0x8c18, 0xffffffff, 0x20202078,
 932	0x8c1c, 0x0000ffff, 0x00001010,
 933	0x28350, 0x00000f01, 0x00000000,
 934	0x9508, 0x3700001f, 0x00000002,
 935	0x960c, 0xffffffff, 0x54763210,
 936	0x88c4, 0x001f3ae3, 0x000000c2,
 937	0x88d4, 0x0000001f, 0x00000010,
 938	0x8974, 0xffffffff, 0x00000000
 939};
 940
 941static const u32 caicos_golden_registers[] =
 942{
 943	0x5eb4, 0xffffffff, 0x00000002,
 944	0x5e78, 0x8f311ff1, 0x001000f0,
 945	0x8c8, 0x00003420, 0x00001450,
 946	0x8cc, 0x000fffff, 0x00040035,
 947	0x3f90, 0xffff0000, 0xfffc0000,
 948	0x9148, 0xffff0000, 0xfffc0000,
 949	0x3f94, 0xffff0000, 0xfffc0000,
 950	0x914c, 0xffff0000, 0xfffc0000,
 951	0xc78, 0x00000080, 0x00000080,
 952	0xbd4, 0x00073007, 0x00010001,
 953	0xd02c, 0xbfffff1f, 0x08421000,
 954	0xd0b8, 0x03773777, 0x02010001,
 955	0x5bc0, 0x00200000, 0x50100000,
 956	0x98f8, 0x33773777, 0x02010001,
 957	0x98fc, 0xffffffff, 0x33221100,
 958	0x7030, 0x31000311, 0x00000011,
 959	0x2f48, 0x33773777, 0x02010001,
 960	0x6b28, 0x00000010, 0x00000012,
 961	0x7728, 0x00000010, 0x00000012,
 962	0x10328, 0x00000010, 0x00000012,
 963	0x10f28, 0x00000010, 0x00000012,
 964	0x11b28, 0x00000010, 0x00000012,
 965	0x12728, 0x00000010, 0x00000012,
 966	0x240c, 0x000007ff, 0x00000380,
 967	0x8a14, 0xf000001f, 0x00000001,
 968	0x8b24, 0x3fff3fff, 0x00ff0fff,
 969	0x8b10, 0x0000ff0f, 0x00000000,
 970	0x28a4c, 0x07ffffff, 0x06000000,
 971	0x10c, 0x00000001, 0x00010003,
 972	0xa02c, 0xffffffff, 0x0000009b,
 973	0x913c, 0x0000000f, 0x0100000a,
 974	0x8d00, 0xffff7f7f, 0x100e4848,
 975	0x8d04, 0x00ffffff, 0x00164745,
 976	0x8c00, 0xfffc0003, 0xe4000003,
 977	0x8c04, 0xf8ff00ff, 0x40600060,
 978	0x8c08, 0x00ff00ff, 0x001c001c,
 979	0x8cf0, 0x1fff1fff, 0x08e00410,
 980	0x8c20, 0x0fff0fff, 0x00800080,
 981	0x8c24, 0x0fff0fff, 0x00800080,
 982	0x8c18, 0xffffffff, 0x20202078,
 983	0x8c1c, 0x0000ffff, 0x00001010,
 984	0x28350, 0x00000f01, 0x00000000,
 985	0x9508, 0x3700001f, 0x00000002,
 986	0x960c, 0xffffffff, 0x54763210,
 987	0x88c4, 0x001f3ae3, 0x000000c2,
 988	0x88d4, 0x0000001f, 0x00000010,
 989	0x8974, 0xffffffff, 0x00000000
 990};
 991
 992static void evergreen_init_golden_registers(struct radeon_device *rdev)
 993{
 994	switch (rdev->family) {
 995	case CHIP_CYPRESS:
 996	case CHIP_HEMLOCK:
 997		radeon_program_register_sequence(rdev,
 998						 evergreen_golden_registers,
 999						 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1000		radeon_program_register_sequence(rdev,
1001						 evergreen_golden_registers2,
1002						 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1003		radeon_program_register_sequence(rdev,
1004						 cypress_mgcg_init,
1005						 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1006		break;
1007	case CHIP_JUNIPER:
1008		radeon_program_register_sequence(rdev,
1009						 evergreen_golden_registers,
1010						 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1011		radeon_program_register_sequence(rdev,
1012						 evergreen_golden_registers2,
1013						 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1014		radeon_program_register_sequence(rdev,
1015						 juniper_mgcg_init,
1016						 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1017		break;
1018	case CHIP_REDWOOD:
1019		radeon_program_register_sequence(rdev,
1020						 evergreen_golden_registers,
1021						 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1022		radeon_program_register_sequence(rdev,
1023						 evergreen_golden_registers2,
1024						 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1025		radeon_program_register_sequence(rdev,
1026						 redwood_mgcg_init,
1027						 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1028		break;
1029	case CHIP_CEDAR:
1030		radeon_program_register_sequence(rdev,
1031						 cedar_golden_registers,
1032						 (const u32)ARRAY_SIZE(cedar_golden_registers));
1033		radeon_program_register_sequence(rdev,
1034						 evergreen_golden_registers2,
1035						 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1036		radeon_program_register_sequence(rdev,
1037						 cedar_mgcg_init,
1038						 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1039		break;
1040	case CHIP_PALM:
1041		radeon_program_register_sequence(rdev,
1042						 wrestler_golden_registers,
1043						 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1044		break;
1045	case CHIP_SUMO:
1046		radeon_program_register_sequence(rdev,
1047						 supersumo_golden_registers,
1048						 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1049		break;
1050	case CHIP_SUMO2:
1051		radeon_program_register_sequence(rdev,
1052						 supersumo_golden_registers,
1053						 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1054		radeon_program_register_sequence(rdev,
1055						 sumo_golden_registers,
1056						 (const u32)ARRAY_SIZE(sumo_golden_registers));
1057		break;
1058	case CHIP_BARTS:
1059		radeon_program_register_sequence(rdev,
1060						 barts_golden_registers,
1061						 (const u32)ARRAY_SIZE(barts_golden_registers));
1062		break;
1063	case CHIP_TURKS:
1064		radeon_program_register_sequence(rdev,
1065						 turks_golden_registers,
1066						 (const u32)ARRAY_SIZE(turks_golden_registers));
1067		break;
1068	case CHIP_CAICOS:
1069		radeon_program_register_sequence(rdev,
1070						 caicos_golden_registers,
1071						 (const u32)ARRAY_SIZE(caicos_golden_registers));
1072		break;
1073	default:
1074		break;
1075	}
1076}
1077
1078/**
1079 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1080 *
1081 * @rdev: radeon_device pointer
1082 * @reg: register offset in bytes
1083 * @val: register value
1084 *
1085 * Returns 0 for success or -EINVAL for an invalid register
1086 *
1087 */
1088int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1089					u32 reg, u32 *val)
1090{
1091	switch (reg) {
1092	case GRBM_STATUS:
1093	case GRBM_STATUS_SE0:
1094	case GRBM_STATUS_SE1:
1095	case SRBM_STATUS:
1096	case SRBM_STATUS2:
1097	case DMA_STATUS_REG:
1098	case UVD_STATUS:
1099		*val = RREG32(reg);
1100		return 0;
1101	default:
1102		return -EINVAL;
1103	}
1104}
1105
1106void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1107			     unsigned *bankh, unsigned *mtaspect,
1108			     unsigned *tile_split)
1109{
1110	*bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1111	*bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1112	*mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1113	*tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1114	switch (*bankw) {
1115	default:
1116	case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1117	case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1118	case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1119	case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1120	}
1121	switch (*bankh) {
1122	default:
1123	case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1124	case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1125	case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1126	case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1127	}
1128	switch (*mtaspect) {
1129	default:
1130	case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1131	case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1132	case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1133	case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1134	}
1135}
1136
1137static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1138			      u32 cntl_reg, u32 status_reg)
1139{
1140	int r, i;
1141	struct atom_clock_dividers dividers;
1142
1143	r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1144					   clock, false, &dividers);
1145	if (r)
1146		return r;
1147
1148	WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1149
1150	for (i = 0; i < 100; i++) {
1151		if (RREG32(status_reg) & DCLK_STATUS)
1152			break;
1153		mdelay(10);
1154	}
1155	if (i == 100)
1156		return -ETIMEDOUT;
1157
1158	return 0;
1159}
1160
1161int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1162{
1163	int r = 0;
1164	u32 cg_scratch = RREG32(CG_SCRATCH1);
1165
1166	r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1167	if (r)
1168		goto done;
1169	cg_scratch &= 0xffff0000;
1170	cg_scratch |= vclk / 100; /* Mhz */
1171
1172	r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1173	if (r)
1174		goto done;
1175	cg_scratch &= 0x0000ffff;
1176	cg_scratch |= (dclk / 100) << 16; /* Mhz */
1177
1178done:
1179	WREG32(CG_SCRATCH1, cg_scratch);
1180
1181	return r;
1182}
1183
1184int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1185{
1186	/* start off with something large */
1187	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1188	int r;
1189
1190	/* bypass vclk and dclk with bclk */
1191	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1192		VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1193		~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1194
1195	/* put PLL in bypass mode */
1196	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1197
1198	if (!vclk || !dclk) {
1199		/* keep the Bypass mode, put PLL to sleep */
1200		WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1201		return 0;
1202	}
1203
1204	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1205					  16384, 0x03FFFFFF, 0, 128, 5,
1206					  &fb_div, &vclk_div, &dclk_div);
1207	if (r)
1208		return r;
1209
1210	/* set VCO_MODE to 1 */
1211	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1212
1213	/* toggle UPLL_SLEEP to 1 then back to 0 */
1214	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1215	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1216
1217	/* deassert UPLL_RESET */
1218	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1219
1220	mdelay(1);
1221
1222	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1223	if (r)
1224		return r;
1225
1226	/* assert UPLL_RESET again */
1227	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1228
1229	/* disable spread spectrum. */
1230	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1231
1232	/* set feedback divider */
1233	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1234
1235	/* set ref divider to 0 */
1236	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1237
1238	if (fb_div < 307200)
1239		WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1240	else
1241		WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1242
1243	/* set PDIV_A and PDIV_B */
1244	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1245		UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1246		~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1247
1248	/* give the PLL some time to settle */
1249	mdelay(15);
1250
1251	/* deassert PLL_RESET */
1252	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1253
1254	mdelay(15);
1255
1256	/* switch from bypass mode to normal mode */
1257	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1258
1259	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1260	if (r)
1261		return r;
1262
1263	/* switch VCLK and DCLK selection */
1264	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1265		VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1266		~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1267
1268	mdelay(100);
1269
1270	return 0;
1271}
1272
1273void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1274{
1275	int readrq;
1276	u16 v;
1277
1278	readrq = pcie_get_readrq(rdev->pdev);
1279	v = ffs(readrq) - 8;
1280	/* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1281	 * to avoid hangs or perfomance issues
1282	 */
1283	if ((v == 0) || (v == 6) || (v == 7))
1284		pcie_set_readrq(rdev->pdev, 512);
1285}
1286
1287void dce4_program_fmt(struct drm_encoder *encoder)
1288{
1289	struct drm_device *dev = encoder->dev;
1290	struct radeon_device *rdev = dev->dev_private;
1291	struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1292	struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1293	struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1294	int bpc = 0;
1295	u32 tmp = 0;
1296	enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1297
1298	if (connector) {
1299		struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1300		bpc = radeon_get_monitor_bpc(connector);
1301		dither = radeon_connector->dither;
1302	}
1303
1304	/* LVDS/eDP FMT is set up by atom */
1305	if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1306		return;
1307
1308	/* not needed for analog */
1309	if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1310	    (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1311		return;
1312
1313	if (bpc == 0)
1314		return;
1315
1316	switch (bpc) {
1317	case 6:
1318		if (dither == RADEON_FMT_DITHER_ENABLE)
1319			/* XXX sort out optimal dither settings */
1320			tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1321				FMT_SPATIAL_DITHER_EN);
1322		else
1323			tmp |= FMT_TRUNCATE_EN;
1324		break;
1325	case 8:
1326		if (dither == RADEON_FMT_DITHER_ENABLE)
1327			/* XXX sort out optimal dither settings */
1328			tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1329				FMT_RGB_RANDOM_ENABLE |
1330				FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1331		else
1332			tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1333		break;
1334	case 10:
1335	default:
1336		/* not needed */
1337		break;
1338	}
1339
1340	WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1341}
1342
1343static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1344{
1345	if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1346		return true;
1347	else
1348		return false;
1349}
1350
1351static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1352{
1353	u32 pos1, pos2;
1354
1355	pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1356	pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1357
1358	if (pos1 != pos2)
1359		return true;
1360	else
1361		return false;
1362}
1363
1364/**
1365 * dce4_wait_for_vblank - vblank wait asic callback.
1366 *
1367 * @rdev: radeon_device pointer
1368 * @crtc: crtc to wait for vblank on
1369 *
1370 * Wait for vblank on the requested crtc (evergreen+).
1371 */
1372void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1373{
1374	unsigned i = 0;
1375
1376	if (crtc >= rdev->num_crtc)
1377		return;
1378
1379	if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1380		return;
1381
1382	/* depending on when we hit vblank, we may be close to active; if so,
1383	 * wait for another frame.
1384	 */
1385	while (dce4_is_in_vblank(rdev, crtc)) {
1386		if (i++ % 100 == 0) {
1387			if (!dce4_is_counter_moving(rdev, crtc))
1388				break;
1389		}
1390	}
1391
1392	while (!dce4_is_in_vblank(rdev, crtc)) {
1393		if (i++ % 100 == 0) {
1394			if (!dce4_is_counter_moving(rdev, crtc))
1395				break;
1396		}
1397	}
1398}
1399
1400/**
1401 * evergreen_page_flip - pageflip callback.
1402 *
1403 * @rdev: radeon_device pointer
1404 * @crtc_id: crtc to cleanup pageflip on
1405 * @crtc_base: new address of the crtc (GPU MC address)
1406 *
1407 * Triggers the actual pageflip by updating the primary
1408 * surface base address (evergreen+).
1409 */
1410void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1411{
1412	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 
 
1413
1414	/* update the scanout addresses */
1415	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1416	       upper_32_bits(crtc_base));
1417	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1418	       (u32)crtc_base);
1419	/* post the write */
1420	RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
 
 
 
 
 
 
1421}
1422
1423/**
1424 * evergreen_page_flip_pending - check if page flip is still pending
1425 *
1426 * @rdev: radeon_device pointer
1427 * @crtc_id: crtc to check
 
1428 *
 
 
 
 
1429 * Returns the current update pending status.
1430 */
1431bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1432{
1433	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1434
1435	/* Return current update_pending status: */
1436	return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1437		EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1438}
1439
1440/* get temperature in millidegrees */
1441int evergreen_get_temp(struct radeon_device *rdev)
1442{
1443	u32 temp, toffset;
1444	int actual_temp = 0;
1445
1446	if (rdev->family == CHIP_JUNIPER) {
1447		toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1448			TOFFSET_SHIFT;
1449		temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1450			TS0_ADC_DOUT_SHIFT;
1451
1452		if (toffset & 0x100)
1453			actual_temp = temp / 2 - (0x200 - toffset);
1454		else
1455			actual_temp = temp / 2 + toffset;
1456
1457		actual_temp = actual_temp * 1000;
1458
1459	} else {
1460		temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1461			ASIC_T_SHIFT;
1462
1463		if (temp & 0x400)
1464			actual_temp = -256;
1465		else if (temp & 0x200)
1466			actual_temp = 255;
1467		else if (temp & 0x100) {
1468			actual_temp = temp & 0x1ff;
1469			actual_temp |= ~0x1ff;
1470		} else
1471			actual_temp = temp & 0xff;
1472
1473		actual_temp = (actual_temp * 1000) / 2;
1474	}
1475
1476	return actual_temp;
1477}
1478
1479int sumo_get_temp(struct radeon_device *rdev)
1480{
1481	u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1482	int actual_temp = temp - 49;
1483
1484	return actual_temp * 1000;
1485}
1486
1487/**
1488 * sumo_pm_init_profile - Initialize power profiles callback.
1489 *
1490 * @rdev: radeon_device pointer
1491 *
1492 * Initialize the power states used in profile mode
1493 * (sumo, trinity, SI).
1494 * Used for profile mode only.
1495 */
1496void sumo_pm_init_profile(struct radeon_device *rdev)
1497{
1498	int idx;
1499
1500	/* default */
1501	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1502	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1503	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1504	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1505
1506	/* low,mid sh/mh */
1507	if (rdev->flags & RADEON_IS_MOBILITY)
1508		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1509	else
1510		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1511
1512	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1513	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1514	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1515	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1516
1517	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1518	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1519	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1520	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1521
1522	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1523	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1524	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1525	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1526
1527	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1528	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1529	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1530	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1531
1532	/* high sh/mh */
1533	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1534	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1535	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1536	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1537	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1538		rdev->pm.power_state[idx].num_clock_modes - 1;
1539
1540	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1541	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1542	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1543	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1544		rdev->pm.power_state[idx].num_clock_modes - 1;
1545}
1546
1547/**
1548 * btc_pm_init_profile - Initialize power profiles callback.
1549 *
1550 * @rdev: radeon_device pointer
1551 *
1552 * Initialize the power states used in profile mode
1553 * (BTC, cayman).
1554 * Used for profile mode only.
1555 */
1556void btc_pm_init_profile(struct radeon_device *rdev)
1557{
1558	int idx;
1559
1560	/* default */
1561	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1562	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1563	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1564	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1565	/* starting with BTC, there is one state that is used for both
1566	 * MH and SH.  Difference is that we always use the high clock index for
1567	 * mclk.
1568	 */
1569	if (rdev->flags & RADEON_IS_MOBILITY)
1570		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1571	else
1572		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1573	/* low sh */
1574	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1575	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1576	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1577	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1578	/* mid sh */
1579	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1580	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1581	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1582	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1583	/* high sh */
1584	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1585	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1586	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1587	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1588	/* low mh */
1589	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1590	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1591	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1592	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1593	/* mid mh */
1594	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1595	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1596	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1597	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1598	/* high mh */
1599	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1600	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1601	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1602	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1603}
1604
1605/**
1606 * evergreen_pm_misc - set additional pm hw parameters callback.
1607 *
1608 * @rdev: radeon_device pointer
1609 *
1610 * Set non-clock parameters associated with a power state
1611 * (voltage, etc.) (evergreen+).
1612 */
1613void evergreen_pm_misc(struct radeon_device *rdev)
1614{
1615	int req_ps_idx = rdev->pm.requested_power_state_index;
1616	int req_cm_idx = rdev->pm.requested_clock_mode_index;
1617	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1618	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1619
1620	if (voltage->type == VOLTAGE_SW) {
1621		/* 0xff0x are flags rather then an actual voltage */
1622		if ((voltage->voltage & 0xff00) == 0xff00)
1623			return;
1624		if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1625			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1626			rdev->pm.current_vddc = voltage->voltage;
1627			DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1628		}
1629
1630		/* starting with BTC, there is one state that is used for both
1631		 * MH and SH.  Difference is that we always use the high clock index for
1632		 * mclk and vddci.
1633		 */
1634		if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1635		    (rdev->family >= CHIP_BARTS) &&
1636		    rdev->pm.active_crtc_count &&
1637		    ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1638		     (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1639			voltage = &rdev->pm.power_state[req_ps_idx].
1640				clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1641
1642		/* 0xff0x are flags rather then an actual voltage */
1643		if ((voltage->vddci & 0xff00) == 0xff00)
1644			return;
1645		if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1646			radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1647			rdev->pm.current_vddci = voltage->vddci;
1648			DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1649		}
1650	}
1651}
1652
1653/**
1654 * evergreen_pm_prepare - pre-power state change callback.
1655 *
1656 * @rdev: radeon_device pointer
1657 *
1658 * Prepare for a power state change (evergreen+).
1659 */
1660void evergreen_pm_prepare(struct radeon_device *rdev)
1661{
1662	struct drm_device *ddev = rdev->ddev;
1663	struct drm_crtc *crtc;
1664	struct radeon_crtc *radeon_crtc;
1665	u32 tmp;
1666
1667	/* disable any active CRTCs */
1668	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1669		radeon_crtc = to_radeon_crtc(crtc);
1670		if (radeon_crtc->enabled) {
1671			tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1672			tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1673			WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1674		}
1675	}
1676}
1677
1678/**
1679 * evergreen_pm_finish - post-power state change callback.
1680 *
1681 * @rdev: radeon_device pointer
1682 *
1683 * Clean up after a power state change (evergreen+).
1684 */
1685void evergreen_pm_finish(struct radeon_device *rdev)
1686{
1687	struct drm_device *ddev = rdev->ddev;
1688	struct drm_crtc *crtc;
1689	struct radeon_crtc *radeon_crtc;
1690	u32 tmp;
1691
1692	/* enable any active CRTCs */
1693	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1694		radeon_crtc = to_radeon_crtc(crtc);
1695		if (radeon_crtc->enabled) {
1696			tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1697			tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1698			WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1699		}
1700	}
1701}
1702
1703/**
1704 * evergreen_hpd_sense - hpd sense callback.
1705 *
1706 * @rdev: radeon_device pointer
1707 * @hpd: hpd (hotplug detect) pin
1708 *
1709 * Checks if a digital monitor is connected (evergreen+).
1710 * Returns true if connected, false if not connected.
1711 */
1712bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1713{
1714	bool connected = false;
1715
1716	switch (hpd) {
1717	case RADEON_HPD_1:
1718		if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1719			connected = true;
1720		break;
1721	case RADEON_HPD_2:
1722		if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1723			connected = true;
1724		break;
1725	case RADEON_HPD_3:
1726		if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1727			connected = true;
1728		break;
1729	case RADEON_HPD_4:
1730		if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1731			connected = true;
1732		break;
1733	case RADEON_HPD_5:
1734		if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1735			connected = true;
1736		break;
1737	case RADEON_HPD_6:
1738		if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1739			connected = true;
1740		break;
1741	default:
1742		break;
1743	}
1744
1745	return connected;
1746}
1747
1748/**
1749 * evergreen_hpd_set_polarity - hpd set polarity callback.
1750 *
1751 * @rdev: radeon_device pointer
1752 * @hpd: hpd (hotplug detect) pin
1753 *
1754 * Set the polarity of the hpd pin (evergreen+).
1755 */
1756void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1757				enum radeon_hpd_id hpd)
1758{
1759	u32 tmp;
1760	bool connected = evergreen_hpd_sense(rdev, hpd);
1761
1762	switch (hpd) {
1763	case RADEON_HPD_1:
1764		tmp = RREG32(DC_HPD1_INT_CONTROL);
1765		if (connected)
1766			tmp &= ~DC_HPDx_INT_POLARITY;
1767		else
1768			tmp |= DC_HPDx_INT_POLARITY;
1769		WREG32(DC_HPD1_INT_CONTROL, tmp);
1770		break;
1771	case RADEON_HPD_2:
1772		tmp = RREG32(DC_HPD2_INT_CONTROL);
1773		if (connected)
1774			tmp &= ~DC_HPDx_INT_POLARITY;
1775		else
1776			tmp |= DC_HPDx_INT_POLARITY;
1777		WREG32(DC_HPD2_INT_CONTROL, tmp);
1778		break;
1779	case RADEON_HPD_3:
1780		tmp = RREG32(DC_HPD3_INT_CONTROL);
1781		if (connected)
1782			tmp &= ~DC_HPDx_INT_POLARITY;
1783		else
1784			tmp |= DC_HPDx_INT_POLARITY;
1785		WREG32(DC_HPD3_INT_CONTROL, tmp);
1786		break;
1787	case RADEON_HPD_4:
1788		tmp = RREG32(DC_HPD4_INT_CONTROL);
1789		if (connected)
1790			tmp &= ~DC_HPDx_INT_POLARITY;
1791		else
1792			tmp |= DC_HPDx_INT_POLARITY;
1793		WREG32(DC_HPD4_INT_CONTROL, tmp);
1794		break;
1795	case RADEON_HPD_5:
1796		tmp = RREG32(DC_HPD5_INT_CONTROL);
1797		if (connected)
1798			tmp &= ~DC_HPDx_INT_POLARITY;
1799		else
1800			tmp |= DC_HPDx_INT_POLARITY;
1801		WREG32(DC_HPD5_INT_CONTROL, tmp);
1802			break;
1803	case RADEON_HPD_6:
1804		tmp = RREG32(DC_HPD6_INT_CONTROL);
1805		if (connected)
1806			tmp &= ~DC_HPDx_INT_POLARITY;
1807		else
1808			tmp |= DC_HPDx_INT_POLARITY;
1809		WREG32(DC_HPD6_INT_CONTROL, tmp);
1810		break;
1811	default:
1812		break;
1813	}
1814}
1815
1816/**
1817 * evergreen_hpd_init - hpd setup callback.
1818 *
1819 * @rdev: radeon_device pointer
1820 *
1821 * Setup the hpd pins used by the card (evergreen+).
1822 * Enable the pin, set the polarity, and enable the hpd interrupts.
1823 */
1824void evergreen_hpd_init(struct radeon_device *rdev)
1825{
1826	struct drm_device *dev = rdev->ddev;
1827	struct drm_connector *connector;
1828	unsigned enabled = 0;
1829	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1830		DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1831
1832	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1833		struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1834
1835		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1836		    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1837			/* don't try to enable hpd on eDP or LVDS avoid breaking the
1838			 * aux dp channel on imac and help (but not completely fix)
1839			 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1840			 * also avoid interrupt storms during dpms.
1841			 */
1842			continue;
1843		}
1844		switch (radeon_connector->hpd.hpd) {
1845		case RADEON_HPD_1:
1846			WREG32(DC_HPD1_CONTROL, tmp);
1847			break;
1848		case RADEON_HPD_2:
1849			WREG32(DC_HPD2_CONTROL, tmp);
1850			break;
1851		case RADEON_HPD_3:
1852			WREG32(DC_HPD3_CONTROL, tmp);
1853			break;
1854		case RADEON_HPD_4:
1855			WREG32(DC_HPD4_CONTROL, tmp);
1856			break;
1857		case RADEON_HPD_5:
1858			WREG32(DC_HPD5_CONTROL, tmp);
1859			break;
1860		case RADEON_HPD_6:
1861			WREG32(DC_HPD6_CONTROL, tmp);
1862			break;
1863		default:
1864			break;
1865		}
1866		radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1867		enabled |= 1 << radeon_connector->hpd.hpd;
1868	}
1869	radeon_irq_kms_enable_hpd(rdev, enabled);
1870}
1871
1872/**
1873 * evergreen_hpd_fini - hpd tear down callback.
1874 *
1875 * @rdev: radeon_device pointer
1876 *
1877 * Tear down the hpd pins used by the card (evergreen+).
1878 * Disable the hpd interrupts.
1879 */
1880void evergreen_hpd_fini(struct radeon_device *rdev)
1881{
1882	struct drm_device *dev = rdev->ddev;
1883	struct drm_connector *connector;
1884	unsigned disabled = 0;
1885
1886	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1887		struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1888		switch (radeon_connector->hpd.hpd) {
1889		case RADEON_HPD_1:
1890			WREG32(DC_HPD1_CONTROL, 0);
1891			break;
1892		case RADEON_HPD_2:
1893			WREG32(DC_HPD2_CONTROL, 0);
1894			break;
1895		case RADEON_HPD_3:
1896			WREG32(DC_HPD3_CONTROL, 0);
1897			break;
1898		case RADEON_HPD_4:
1899			WREG32(DC_HPD4_CONTROL, 0);
1900			break;
1901		case RADEON_HPD_5:
1902			WREG32(DC_HPD5_CONTROL, 0);
1903			break;
1904		case RADEON_HPD_6:
1905			WREG32(DC_HPD6_CONTROL, 0);
1906			break;
1907		default:
1908			break;
1909		}
1910		disabled |= 1 << radeon_connector->hpd.hpd;
1911	}
1912	radeon_irq_kms_disable_hpd(rdev, disabled);
1913}
1914
1915/* watermark setup */
1916
1917static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1918					struct radeon_crtc *radeon_crtc,
1919					struct drm_display_mode *mode,
1920					struct drm_display_mode *other_mode)
1921{
1922	u32 tmp, buffer_alloc, i;
1923	u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1924	/*
1925	 * Line Buffer Setup
1926	 * There are 3 line buffers, each one shared by 2 display controllers.
1927	 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1928	 * the display controllers.  The paritioning is done via one of four
1929	 * preset allocations specified in bits 2:0:
1930	 * first display controller
1931	 *  0 - first half of lb (3840 * 2)
1932	 *  1 - first 3/4 of lb (5760 * 2)
1933	 *  2 - whole lb (7680 * 2), other crtc must be disabled
1934	 *  3 - first 1/4 of lb (1920 * 2)
1935	 * second display controller
1936	 *  4 - second half of lb (3840 * 2)
1937	 *  5 - second 3/4 of lb (5760 * 2)
1938	 *  6 - whole lb (7680 * 2), other crtc must be disabled
1939	 *  7 - last 1/4 of lb (1920 * 2)
1940	 */
1941	/* this can get tricky if we have two large displays on a paired group
1942	 * of crtcs.  Ideally for multiple large displays we'd assign them to
1943	 * non-linked crtcs for maximum line buffer allocation.
1944	 */
1945	if (radeon_crtc->base.enabled && mode) {
1946		if (other_mode) {
1947			tmp = 0; /* 1/2 */
1948			buffer_alloc = 1;
1949		} else {
1950			tmp = 2; /* whole */
1951			buffer_alloc = 2;
1952		}
1953	} else {
1954		tmp = 0;
1955		buffer_alloc = 0;
1956	}
1957
1958	/* second controller of the pair uses second half of the lb */
1959	if (radeon_crtc->crtc_id % 2)
1960		tmp += 4;
1961	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1962
1963	if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1964		WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1965		       DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1966		for (i = 0; i < rdev->usec_timeout; i++) {
1967			if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1968			    DMIF_BUFFERS_ALLOCATED_COMPLETED)
1969				break;
1970			udelay(1);
1971		}
1972	}
1973
1974	if (radeon_crtc->base.enabled && mode) {
1975		switch (tmp) {
1976		case 0:
1977		case 4:
1978		default:
1979			if (ASIC_IS_DCE5(rdev))
1980				return 4096 * 2;
1981			else
1982				return 3840 * 2;
1983		case 1:
1984		case 5:
1985			if (ASIC_IS_DCE5(rdev))
1986				return 6144 * 2;
1987			else
1988				return 5760 * 2;
1989		case 2:
1990		case 6:
1991			if (ASIC_IS_DCE5(rdev))
1992				return 8192 * 2;
1993			else
1994				return 7680 * 2;
1995		case 3:
1996		case 7:
1997			if (ASIC_IS_DCE5(rdev))
1998				return 2048 * 2;
1999			else
2000				return 1920 * 2;
2001		}
2002	}
2003
2004	/* controller not enabled, so no lb used */
2005	return 0;
2006}
2007
2008u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
2009{
2010	u32 tmp = RREG32(MC_SHARED_CHMAP);
2011
2012	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2013	case 0:
2014	default:
2015		return 1;
2016	case 1:
2017		return 2;
2018	case 2:
2019		return 4;
2020	case 3:
2021		return 8;
2022	}
2023}
2024
2025struct evergreen_wm_params {
2026	u32 dram_channels; /* number of dram channels */
2027	u32 yclk;          /* bandwidth per dram data pin in kHz */
2028	u32 sclk;          /* engine clock in kHz */
2029	u32 disp_clk;      /* display clock in kHz */
2030	u32 src_width;     /* viewport width */
2031	u32 active_time;   /* active display time in ns */
2032	u32 blank_time;    /* blank time in ns */
2033	bool interlaced;    /* mode is interlaced */
2034	fixed20_12 vsc;    /* vertical scale ratio */
2035	u32 num_heads;     /* number of active crtcs */
2036	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2037	u32 lb_size;       /* line buffer allocated to pipe */
2038	u32 vtaps;         /* vertical scaler taps */
2039};
2040
2041static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
2042{
2043	/* Calculate DRAM Bandwidth and the part allocated to display. */
2044	fixed20_12 dram_efficiency; /* 0.7 */
2045	fixed20_12 yclk, dram_channels, bandwidth;
2046	fixed20_12 a;
2047
2048	a.full = dfixed_const(1000);
2049	yclk.full = dfixed_const(wm->yclk);
2050	yclk.full = dfixed_div(yclk, a);
2051	dram_channels.full = dfixed_const(wm->dram_channels * 4);
2052	a.full = dfixed_const(10);
2053	dram_efficiency.full = dfixed_const(7);
2054	dram_efficiency.full = dfixed_div(dram_efficiency, a);
2055	bandwidth.full = dfixed_mul(dram_channels, yclk);
2056	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2057
2058	return dfixed_trunc(bandwidth);
2059}
2060
2061static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2062{
2063	/* Calculate DRAM Bandwidth and the part allocated to display. */
2064	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2065	fixed20_12 yclk, dram_channels, bandwidth;
2066	fixed20_12 a;
2067
2068	a.full = dfixed_const(1000);
2069	yclk.full = dfixed_const(wm->yclk);
2070	yclk.full = dfixed_div(yclk, a);
2071	dram_channels.full = dfixed_const(wm->dram_channels * 4);
2072	a.full = dfixed_const(10);
2073	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2074	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2075	bandwidth.full = dfixed_mul(dram_channels, yclk);
2076	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2077
2078	return dfixed_trunc(bandwidth);
2079}
2080
2081static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2082{
2083	/* Calculate the display Data return Bandwidth */
2084	fixed20_12 return_efficiency; /* 0.8 */
2085	fixed20_12 sclk, bandwidth;
2086	fixed20_12 a;
2087
2088	a.full = dfixed_const(1000);
2089	sclk.full = dfixed_const(wm->sclk);
2090	sclk.full = dfixed_div(sclk, a);
2091	a.full = dfixed_const(10);
2092	return_efficiency.full = dfixed_const(8);
2093	return_efficiency.full = dfixed_div(return_efficiency, a);
2094	a.full = dfixed_const(32);
2095	bandwidth.full = dfixed_mul(a, sclk);
2096	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2097
2098	return dfixed_trunc(bandwidth);
2099}
2100
2101static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2102{
2103	/* Calculate the DMIF Request Bandwidth */
2104	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2105	fixed20_12 disp_clk, bandwidth;
2106	fixed20_12 a;
2107
2108	a.full = dfixed_const(1000);
2109	disp_clk.full = dfixed_const(wm->disp_clk);
2110	disp_clk.full = dfixed_div(disp_clk, a);
2111	a.full = dfixed_const(10);
2112	disp_clk_request_efficiency.full = dfixed_const(8);
2113	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2114	a.full = dfixed_const(32);
2115	bandwidth.full = dfixed_mul(a, disp_clk);
2116	bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2117
2118	return dfixed_trunc(bandwidth);
2119}
2120
2121static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2122{
2123	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2124	u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2125	u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2126	u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2127
2128	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2129}
2130
2131static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2132{
2133	/* Calculate the display mode Average Bandwidth
2134	 * DisplayMode should contain the source and destination dimensions,
2135	 * timing, etc.
2136	 */
2137	fixed20_12 bpp;
2138	fixed20_12 line_time;
2139	fixed20_12 src_width;
2140	fixed20_12 bandwidth;
2141	fixed20_12 a;
2142
2143	a.full = dfixed_const(1000);
2144	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2145	line_time.full = dfixed_div(line_time, a);
2146	bpp.full = dfixed_const(wm->bytes_per_pixel);
2147	src_width.full = dfixed_const(wm->src_width);
2148	bandwidth.full = dfixed_mul(src_width, bpp);
2149	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2150	bandwidth.full = dfixed_div(bandwidth, line_time);
2151
2152	return dfixed_trunc(bandwidth);
2153}
2154
2155static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2156{
2157	/* First calcualte the latency in ns */
2158	u32 mc_latency = 2000; /* 2000 ns. */
2159	u32 available_bandwidth = evergreen_available_bandwidth(wm);
2160	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2161	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2162	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2163	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2164		(wm->num_heads * cursor_line_pair_return_time);
2165	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2166	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2167	fixed20_12 a, b, c;
2168
2169	if (wm->num_heads == 0)
2170		return 0;
2171
2172	a.full = dfixed_const(2);
2173	b.full = dfixed_const(1);
2174	if ((wm->vsc.full > a.full) ||
2175	    ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2176	    (wm->vtaps >= 5) ||
2177	    ((wm->vsc.full >= a.full) && wm->interlaced))
2178		max_src_lines_per_dst_line = 4;
2179	else
2180		max_src_lines_per_dst_line = 2;
2181
2182	a.full = dfixed_const(available_bandwidth);
2183	b.full = dfixed_const(wm->num_heads);
2184	a.full = dfixed_div(a, b);
2185
2186	b.full = dfixed_const(1000);
2187	c.full = dfixed_const(wm->disp_clk);
2188	b.full = dfixed_div(c, b);
2189	c.full = dfixed_const(wm->bytes_per_pixel);
2190	b.full = dfixed_mul(b, c);
2191
2192	lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2193
2194	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2195	b.full = dfixed_const(1000);
2196	c.full = dfixed_const(lb_fill_bw);
2197	b.full = dfixed_div(c, b);
2198	a.full = dfixed_div(a, b);
2199	line_fill_time = dfixed_trunc(a);
2200
2201	if (line_fill_time < wm->active_time)
2202		return latency;
2203	else
2204		return latency + (line_fill_time - wm->active_time);
2205
2206}
2207
2208static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2209{
2210	if (evergreen_average_bandwidth(wm) <=
2211	    (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2212		return true;
2213	else
2214		return false;
2215};
2216
2217static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2218{
2219	if (evergreen_average_bandwidth(wm) <=
2220	    (evergreen_available_bandwidth(wm) / wm->num_heads))
2221		return true;
2222	else
2223		return false;
2224};
2225
2226static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2227{
2228	u32 lb_partitions = wm->lb_size / wm->src_width;
2229	u32 line_time = wm->active_time + wm->blank_time;
2230	u32 latency_tolerant_lines;
2231	u32 latency_hiding;
2232	fixed20_12 a;
2233
2234	a.full = dfixed_const(1);
2235	if (wm->vsc.full > a.full)
2236		latency_tolerant_lines = 1;
2237	else {
2238		if (lb_partitions <= (wm->vtaps + 1))
2239			latency_tolerant_lines = 1;
2240		else
2241			latency_tolerant_lines = 2;
2242	}
2243
2244	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2245
2246	if (evergreen_latency_watermark(wm) <= latency_hiding)
2247		return true;
2248	else
2249		return false;
2250}
2251
2252static void evergreen_program_watermarks(struct radeon_device *rdev,
2253					 struct radeon_crtc *radeon_crtc,
2254					 u32 lb_size, u32 num_heads)
2255{
2256	struct drm_display_mode *mode = &radeon_crtc->base.mode;
2257	struct evergreen_wm_params wm_low, wm_high;
2258	u32 dram_channels;
2259	u32 pixel_period;
2260	u32 line_time = 0;
2261	u32 latency_watermark_a = 0, latency_watermark_b = 0;
2262	u32 priority_a_mark = 0, priority_b_mark = 0;
2263	u32 priority_a_cnt = PRIORITY_OFF;
2264	u32 priority_b_cnt = PRIORITY_OFF;
2265	u32 pipe_offset = radeon_crtc->crtc_id * 16;
2266	u32 tmp, arb_control3;
2267	fixed20_12 a, b, c;
2268
2269	if (radeon_crtc->base.enabled && num_heads && mode) {
2270		pixel_period = 1000000 / (u32)mode->clock;
2271		line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2272		priority_a_cnt = 0;
2273		priority_b_cnt = 0;
2274		dram_channels = evergreen_get_number_of_dram_channels(rdev);
2275
2276		/* watermark for high clocks */
2277		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2278			wm_high.yclk =
2279				radeon_dpm_get_mclk(rdev, false) * 10;
2280			wm_high.sclk =
2281				radeon_dpm_get_sclk(rdev, false) * 10;
2282		} else {
2283			wm_high.yclk = rdev->pm.current_mclk * 10;
2284			wm_high.sclk = rdev->pm.current_sclk * 10;
2285		}
2286
2287		wm_high.disp_clk = mode->clock;
2288		wm_high.src_width = mode->crtc_hdisplay;
2289		wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2290		wm_high.blank_time = line_time - wm_high.active_time;
2291		wm_high.interlaced = false;
2292		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2293			wm_high.interlaced = true;
2294		wm_high.vsc = radeon_crtc->vsc;
2295		wm_high.vtaps = 1;
2296		if (radeon_crtc->rmx_type != RMX_OFF)
2297			wm_high.vtaps = 2;
2298		wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2299		wm_high.lb_size = lb_size;
2300		wm_high.dram_channels = dram_channels;
2301		wm_high.num_heads = num_heads;
2302
2303		/* watermark for low clocks */
2304		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2305			wm_low.yclk =
2306				radeon_dpm_get_mclk(rdev, true) * 10;
2307			wm_low.sclk =
2308				radeon_dpm_get_sclk(rdev, true) * 10;
2309		} else {
2310			wm_low.yclk = rdev->pm.current_mclk * 10;
2311			wm_low.sclk = rdev->pm.current_sclk * 10;
2312		}
2313
2314		wm_low.disp_clk = mode->clock;
2315		wm_low.src_width = mode->crtc_hdisplay;
2316		wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2317		wm_low.blank_time = line_time - wm_low.active_time;
2318		wm_low.interlaced = false;
2319		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2320			wm_low.interlaced = true;
2321		wm_low.vsc = radeon_crtc->vsc;
2322		wm_low.vtaps = 1;
2323		if (radeon_crtc->rmx_type != RMX_OFF)
2324			wm_low.vtaps = 2;
2325		wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2326		wm_low.lb_size = lb_size;
2327		wm_low.dram_channels = dram_channels;
2328		wm_low.num_heads = num_heads;
2329
2330		/* set for high clocks */
2331		latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2332		/* set for low clocks */
2333		latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2334
2335		/* possibly force display priority to high */
2336		/* should really do this at mode validation time... */
2337		if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2338		    !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2339		    !evergreen_check_latency_hiding(&wm_high) ||
2340		    (rdev->disp_priority == 2)) {
2341			DRM_DEBUG_KMS("force priority a to high\n");
2342			priority_a_cnt |= PRIORITY_ALWAYS_ON;
2343		}
2344		if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2345		    !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2346		    !evergreen_check_latency_hiding(&wm_low) ||
2347		    (rdev->disp_priority == 2)) {
2348			DRM_DEBUG_KMS("force priority b to high\n");
2349			priority_b_cnt |= PRIORITY_ALWAYS_ON;
2350		}
2351
2352		a.full = dfixed_const(1000);
2353		b.full = dfixed_const(mode->clock);
2354		b.full = dfixed_div(b, a);
2355		c.full = dfixed_const(latency_watermark_a);
2356		c.full = dfixed_mul(c, b);
2357		c.full = dfixed_mul(c, radeon_crtc->hsc);
2358		c.full = dfixed_div(c, a);
2359		a.full = dfixed_const(16);
2360		c.full = dfixed_div(c, a);
2361		priority_a_mark = dfixed_trunc(c);
2362		priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2363
2364		a.full = dfixed_const(1000);
2365		b.full = dfixed_const(mode->clock);
2366		b.full = dfixed_div(b, a);
2367		c.full = dfixed_const(latency_watermark_b);
2368		c.full = dfixed_mul(c, b);
2369		c.full = dfixed_mul(c, radeon_crtc->hsc);
2370		c.full = dfixed_div(c, a);
2371		a.full = dfixed_const(16);
2372		c.full = dfixed_div(c, a);
2373		priority_b_mark = dfixed_trunc(c);
2374		priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2375
2376		/* Save number of lines the linebuffer leads before the scanout */
2377		radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2378	}
2379
2380	/* select wm A */
2381	arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2382	tmp = arb_control3;
2383	tmp &= ~LATENCY_WATERMARK_MASK(3);
2384	tmp |= LATENCY_WATERMARK_MASK(1);
2385	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2386	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2387	       (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2388		LATENCY_HIGH_WATERMARK(line_time)));
2389	/* select wm B */
2390	tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2391	tmp &= ~LATENCY_WATERMARK_MASK(3);
2392	tmp |= LATENCY_WATERMARK_MASK(2);
2393	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2394	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2395	       (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2396		LATENCY_HIGH_WATERMARK(line_time)));
2397	/* restore original selection */
2398	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2399
2400	/* write the priority marks */
2401	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2402	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2403
2404	/* save values for DPM */
2405	radeon_crtc->line_time = line_time;
2406	radeon_crtc->wm_high = latency_watermark_a;
2407	radeon_crtc->wm_low = latency_watermark_b;
2408}
2409
2410/**
2411 * evergreen_bandwidth_update - update display watermarks callback.
2412 *
2413 * @rdev: radeon_device pointer
2414 *
2415 * Update the display watermarks based on the requested mode(s)
2416 * (evergreen+).
2417 */
2418void evergreen_bandwidth_update(struct radeon_device *rdev)
2419{
2420	struct drm_display_mode *mode0 = NULL;
2421	struct drm_display_mode *mode1 = NULL;
2422	u32 num_heads = 0, lb_size;
2423	int i;
2424
2425	if (!rdev->mode_info.mode_config_initialized)
2426		return;
2427
2428	radeon_update_display_priority(rdev);
2429
2430	for (i = 0; i < rdev->num_crtc; i++) {
2431		if (rdev->mode_info.crtcs[i]->base.enabled)
2432			num_heads++;
2433	}
2434	for (i = 0; i < rdev->num_crtc; i += 2) {
2435		mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2436		mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2437		lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2438		evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2439		lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2440		evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2441	}
2442}
2443
2444/**
2445 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2446 *
2447 * @rdev: radeon_device pointer
2448 *
2449 * Wait for the MC (memory controller) to be idle.
2450 * (evergreen+).
2451 * Returns 0 if the MC is idle, -1 if not.
2452 */
2453int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2454{
2455	unsigned i;
2456	u32 tmp;
2457
2458	for (i = 0; i < rdev->usec_timeout; i++) {
2459		/* read MC_STATUS */
2460		tmp = RREG32(SRBM_STATUS) & 0x1F00;
2461		if (!tmp)
2462			return 0;
2463		udelay(1);
2464	}
2465	return -1;
2466}
2467
2468/*
2469 * GART
2470 */
2471void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2472{
2473	unsigned i;
2474	u32 tmp;
2475
2476	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2477
2478	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2479	for (i = 0; i < rdev->usec_timeout; i++) {
2480		/* read MC_STATUS */
2481		tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2482		tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2483		if (tmp == 2) {
2484			printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2485			return;
2486		}
2487		if (tmp) {
2488			return;
2489		}
2490		udelay(1);
2491	}
2492}
2493
2494static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2495{
2496	u32 tmp;
2497	int r;
2498
2499	if (rdev->gart.robj == NULL) {
2500		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2501		return -EINVAL;
2502	}
2503	r = radeon_gart_table_vram_pin(rdev);
2504	if (r)
2505		return r;
 
2506	/* Setup L2 cache */
2507	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2508				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2509				EFFECTIVE_L2_QUEUE_SIZE(7));
2510	WREG32(VM_L2_CNTL2, 0);
2511	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2512	/* Setup TLB control */
2513	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2514		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2515		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2516		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2517	if (rdev->flags & RADEON_IS_IGP) {
2518		WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2519		WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2520		WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2521	} else {
2522		WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2523		WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2524		WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2525		if ((rdev->family == CHIP_JUNIPER) ||
2526		    (rdev->family == CHIP_CYPRESS) ||
2527		    (rdev->family == CHIP_HEMLOCK) ||
2528		    (rdev->family == CHIP_BARTS))
2529			WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2530	}
2531	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2532	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2533	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2534	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2535	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2536	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2537	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2538	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2539				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2540	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2541			(u32)(rdev->dummy_page.addr >> 12));
2542	WREG32(VM_CONTEXT1_CNTL, 0);
2543
2544	evergreen_pcie_gart_tlb_flush(rdev);
2545	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2546		 (unsigned)(rdev->mc.gtt_size >> 20),
2547		 (unsigned long long)rdev->gart.table_addr);
2548	rdev->gart.ready = true;
2549	return 0;
2550}
2551
2552static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2553{
2554	u32 tmp;
2555
2556	/* Disable all tables */
2557	WREG32(VM_CONTEXT0_CNTL, 0);
2558	WREG32(VM_CONTEXT1_CNTL, 0);
2559
2560	/* Setup L2 cache */
2561	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2562				EFFECTIVE_L2_QUEUE_SIZE(7));
2563	WREG32(VM_L2_CNTL2, 0);
2564	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2565	/* Setup TLB control */
2566	tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2567	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2568	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2569	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2570	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2571	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2572	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2573	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2574	radeon_gart_table_vram_unpin(rdev);
2575}
2576
2577static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2578{
2579	evergreen_pcie_gart_disable(rdev);
2580	radeon_gart_table_vram_free(rdev);
2581	radeon_gart_fini(rdev);
2582}
2583
2584
2585static void evergreen_agp_enable(struct radeon_device *rdev)
2586{
2587	u32 tmp;
2588
2589	/* Setup L2 cache */
2590	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2591				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2592				EFFECTIVE_L2_QUEUE_SIZE(7));
2593	WREG32(VM_L2_CNTL2, 0);
2594	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2595	/* Setup TLB control */
2596	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2597		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2598		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2599		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2600	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2601	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2602	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2603	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2604	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2605	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2606	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2607	WREG32(VM_CONTEXT0_CNTL, 0);
2608	WREG32(VM_CONTEXT1_CNTL, 0);
2609}
2610
2611static const unsigned ni_dig_offsets[] =
2612{
2613	NI_DIG0_REGISTER_OFFSET,
2614	NI_DIG1_REGISTER_OFFSET,
2615	NI_DIG2_REGISTER_OFFSET,
2616	NI_DIG3_REGISTER_OFFSET,
2617	NI_DIG4_REGISTER_OFFSET,
2618	NI_DIG5_REGISTER_OFFSET
2619};
2620
2621static const unsigned ni_tx_offsets[] =
2622{
2623	NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2624	NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2625	NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2626	NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2627	NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2628	NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2629};
2630
2631static const unsigned evergreen_dp_offsets[] =
2632{
2633	EVERGREEN_DP0_REGISTER_OFFSET,
2634	EVERGREEN_DP1_REGISTER_OFFSET,
2635	EVERGREEN_DP2_REGISTER_OFFSET,
2636	EVERGREEN_DP3_REGISTER_OFFSET,
2637	EVERGREEN_DP4_REGISTER_OFFSET,
2638	EVERGREEN_DP5_REGISTER_OFFSET
2639};
2640
2641
2642/*
2643 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2644 * We go from crtc to connector and it is not relible  since it
2645 * should be an opposite direction .If crtc is enable then
2646 * find the dig_fe which selects this crtc and insure that it enable.
2647 * if such dig_fe is found then find dig_be which selects found dig_be and
2648 * insure that it enable and in DP_SST mode.
2649 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2650 * from dp symbols clocks .
2651 */
2652static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2653					       unsigned crtc_id, unsigned *ret_dig_fe)
2654{
2655	unsigned i;
2656	unsigned dig_fe;
2657	unsigned dig_be;
2658	unsigned dig_en_be;
2659	unsigned uniphy_pll;
2660	unsigned digs_fe_selected;
2661	unsigned dig_be_mode;
2662	unsigned dig_fe_mask;
2663	bool is_enabled = false;
2664	bool found_crtc = false;
2665
2666	/* loop through all running dig_fe to find selected crtc */
2667	for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2668		dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2669		if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2670		    crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2671			/* found running pipe */
2672			found_crtc = true;
2673			dig_fe_mask = 1 << i;
2674			dig_fe = i;
2675			break;
2676		}
2677	}
2678
2679	if (found_crtc) {
2680		/* loop through all running dig_be to find selected dig_fe */
2681		for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2682			dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2683			/* if dig_fe_selected by dig_be? */
2684			digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2685			dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2686			if (dig_fe_mask &  digs_fe_selected &&
2687			    /* if dig_be in sst mode? */
2688			    dig_be_mode == NI_DIG_BE_DPSST) {
2689				dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2690						   ni_dig_offsets[i]);
2691				uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2692						    ni_tx_offsets[i]);
2693				/* dig_be enable and tx is running */
2694				if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2695				    dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2696				    uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2697					is_enabled = true;
2698					*ret_dig_fe = dig_fe;
2699					break;
2700				}
2701			}
2702		}
2703	}
2704
2705	return is_enabled;
2706}
2707
2708/*
2709 * Blank dig when in dp sst mode
2710 * Dig ignores crtc timing
2711 */
2712static void evergreen_blank_dp_output(struct radeon_device *rdev,
2713				      unsigned dig_fe)
2714{
2715	unsigned stream_ctrl;
2716	unsigned fifo_ctrl;
2717	unsigned counter = 0;
2718
2719	if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2720		DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2721		return;
2722	}
2723
2724	stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2725			     evergreen_dp_offsets[dig_fe]);
2726	if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2727		DRM_ERROR("dig %d , should be enable\n", dig_fe);
2728		return;
2729	}
2730
2731	stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2732	WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2733	       evergreen_dp_offsets[dig_fe], stream_ctrl);
2734
2735	stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2736			     evergreen_dp_offsets[dig_fe]);
2737	while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2738		msleep(1);
2739		counter++;
2740		stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2741				     evergreen_dp_offsets[dig_fe]);
2742	}
2743	if (counter >= 32 )
2744		DRM_ERROR("counter exceeds %d\n", counter);
2745
2746	fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2747	fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2748	WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2749
2750}
2751
2752void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2753{
2754	u32 crtc_enabled, tmp, frame_count, blackout;
2755	int i, j;
2756	unsigned dig_fe;
2757
2758	if (!ASIC_IS_NODCE(rdev)) {
2759		save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2760		save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2761
2762		/* disable VGA render */
2763		WREG32(VGA_RENDER_CONTROL, 0);
2764	}
2765	/* blank the display controllers */
2766	for (i = 0; i < rdev->num_crtc; i++) {
2767		crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2768		if (crtc_enabled) {
2769			save->crtc_enabled[i] = true;
2770			if (ASIC_IS_DCE6(rdev)) {
2771				tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2772				if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2773					radeon_wait_for_vblank(rdev, i);
2774					WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2775					tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2776					WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2777					WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2778				}
2779			} else {
2780				tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2781				if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2782					radeon_wait_for_vblank(rdev, i);
2783					WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2784					tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2785					WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2786					WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2787				}
2788			}
2789			/* wait for the next frame */
2790			frame_count = radeon_get_vblank_counter(rdev, i);
2791			for (j = 0; j < rdev->usec_timeout; j++) {
2792				if (radeon_get_vblank_counter(rdev, i) != frame_count)
2793					break;
2794				udelay(1);
2795			}
2796			/*we should disable dig if it drives dp sst*/
2797			/*but we are in radeon_device_init and the topology is unknown*/
2798			/*and it is available after radeon_modeset_init*/
2799			/*the following method radeon_atom_encoder_dpms_dig*/
2800			/*does the job if we initialize it properly*/
2801			/*for now we do it this manually*/
2802			/**/
2803			if (ASIC_IS_DCE5(rdev) &&
2804			    evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2805				evergreen_blank_dp_output(rdev, dig_fe);
2806			/*we could remove 6 lines below*/
2807			/* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2808			WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2809			tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2810			tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2811			WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2812			WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2813			save->crtc_enabled[i] = false;
2814			/* ***** */
2815		} else {
2816			save->crtc_enabled[i] = false;
2817		}
2818	}
2819
2820	radeon_mc_wait_for_idle(rdev);
2821
2822	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2823	if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2824		/* Block CPU access */
2825		WREG32(BIF_FB_EN, 0);
2826		/* blackout the MC */
2827		blackout &= ~BLACKOUT_MODE_MASK;
2828		WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2829	}
2830	/* wait for the MC to settle */
2831	udelay(100);
2832
2833	/* lock double buffered regs */
2834	for (i = 0; i < rdev->num_crtc; i++) {
2835		if (save->crtc_enabled[i]) {
2836			tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2837			if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2838				tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2839				WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2840			}
2841			tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2842			if (!(tmp & 1)) {
2843				tmp |= 1;
2844				WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2845			}
2846		}
2847	}
2848}
2849
2850void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2851{
2852	u32 tmp, frame_count;
2853	int i, j;
2854
2855	/* update crtc base addresses */
2856	for (i = 0; i < rdev->num_crtc; i++) {
2857		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2858		       upper_32_bits(rdev->mc.vram_start));
2859		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2860		       upper_32_bits(rdev->mc.vram_start));
2861		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2862		       (u32)rdev->mc.vram_start);
2863		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2864		       (u32)rdev->mc.vram_start);
2865	}
2866
2867	if (!ASIC_IS_NODCE(rdev)) {
2868		WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2869		WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2870	}
2871
2872	/* unlock regs and wait for update */
2873	for (i = 0; i < rdev->num_crtc; i++) {
2874		if (save->crtc_enabled[i]) {
2875			tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2876			if ((tmp & 0x7) != 3) {
2877				tmp &= ~0x7;
2878				tmp |= 0x3;
2879				WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2880			}
2881			tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2882			if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2883				tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2884				WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2885			}
2886			tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2887			if (tmp & 1) {
2888				tmp &= ~1;
2889				WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2890			}
2891			for (j = 0; j < rdev->usec_timeout; j++) {
2892				tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2893				if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2894					break;
2895				udelay(1);
2896			}
2897		}
2898	}
2899
2900	/* unblackout the MC */
2901	tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2902	tmp &= ~BLACKOUT_MODE_MASK;
2903	WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2904	/* allow CPU access */
2905	WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2906
2907	for (i = 0; i < rdev->num_crtc; i++) {
2908		if (save->crtc_enabled[i]) {
2909			if (ASIC_IS_DCE6(rdev)) {
2910				tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2911				tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2912				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2913				WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2914				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2915			} else {
2916				tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2917				tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2918				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2919				WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2920				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2921			}
2922			/* wait for the next frame */
2923			frame_count = radeon_get_vblank_counter(rdev, i);
2924			for (j = 0; j < rdev->usec_timeout; j++) {
2925				if (radeon_get_vblank_counter(rdev, i) != frame_count)
2926					break;
2927				udelay(1);
2928			}
2929		}
2930	}
2931	if (!ASIC_IS_NODCE(rdev)) {
2932		/* Unlock vga access */
2933		WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2934		mdelay(1);
2935		WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2936	}
2937}
2938
2939void evergreen_mc_program(struct radeon_device *rdev)
2940{
2941	struct evergreen_mc_save save;
2942	u32 tmp;
2943	int i, j;
2944
2945	/* Initialize HDP */
2946	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2947		WREG32((0x2c14 + j), 0x00000000);
2948		WREG32((0x2c18 + j), 0x00000000);
2949		WREG32((0x2c1c + j), 0x00000000);
2950		WREG32((0x2c20 + j), 0x00000000);
2951		WREG32((0x2c24 + j), 0x00000000);
2952	}
2953	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2954
2955	evergreen_mc_stop(rdev, &save);
2956	if (evergreen_mc_wait_for_idle(rdev)) {
2957		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2958	}
2959	/* Lockout access through VGA aperture*/
2960	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2961	/* Update configuration */
2962	if (rdev->flags & RADEON_IS_AGP) {
2963		if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2964			/* VRAM before AGP */
2965			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2966				rdev->mc.vram_start >> 12);
2967			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2968				rdev->mc.gtt_end >> 12);
2969		} else {
2970			/* VRAM after AGP */
2971			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2972				rdev->mc.gtt_start >> 12);
2973			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2974				rdev->mc.vram_end >> 12);
2975		}
2976	} else {
2977		WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2978			rdev->mc.vram_start >> 12);
2979		WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2980			rdev->mc.vram_end >> 12);
2981	}
2982	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2983	/* llano/ontario only */
2984	if ((rdev->family == CHIP_PALM) ||
2985	    (rdev->family == CHIP_SUMO) ||
2986	    (rdev->family == CHIP_SUMO2)) {
2987		tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2988		tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2989		tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2990		WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2991	}
2992	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2993	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2994	WREG32(MC_VM_FB_LOCATION, tmp);
2995	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2996	WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2997	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2998	if (rdev->flags & RADEON_IS_AGP) {
2999		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
3000		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
3001		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
3002	} else {
3003		WREG32(MC_VM_AGP_BASE, 0);
3004		WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3005		WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3006	}
3007	if (evergreen_mc_wait_for_idle(rdev)) {
3008		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3009	}
3010	evergreen_mc_resume(rdev, &save);
3011	/* we need to own VRAM, so turn off the VGA renderer here
3012	 * to stop it overwriting our objects */
3013	rv515_vga_render_disable(rdev);
3014}
3015
3016/*
3017 * CP.
3018 */
3019void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3020{
3021	struct radeon_ring *ring = &rdev->ring[ib->ring];
3022	u32 next_rptr;
3023
3024	/* set to DX10/11 mode */
3025	radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
3026	radeon_ring_write(ring, 1);
3027
3028	if (ring->rptr_save_reg) {
3029		next_rptr = ring->wptr + 3 + 4;
3030		radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3031		radeon_ring_write(ring, ((ring->rptr_save_reg - 
3032					  PACKET3_SET_CONFIG_REG_START) >> 2));
3033		radeon_ring_write(ring, next_rptr);
3034	} else if (rdev->wb.enabled) {
3035		next_rptr = ring->wptr + 5 + 4;
3036		radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
3037		radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3038		radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
3039		radeon_ring_write(ring, next_rptr);
3040		radeon_ring_write(ring, 0);
3041	}
3042
3043	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3044	radeon_ring_write(ring,
3045#ifdef __BIG_ENDIAN
3046			  (2 << 0) |
3047#endif
3048			  (ib->gpu_addr & 0xFFFFFFFC));
3049	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
3050	radeon_ring_write(ring, ib->length_dw);
3051}
3052
3053
3054static int evergreen_cp_load_microcode(struct radeon_device *rdev)
3055{
3056	const __be32 *fw_data;
3057	int i;
3058
3059	if (!rdev->me_fw || !rdev->pfp_fw)
3060		return -EINVAL;
3061
3062	r700_cp_stop(rdev);
3063	WREG32(CP_RB_CNTL,
3064#ifdef __BIG_ENDIAN
3065	       BUF_SWAP_32BIT |
3066#endif
3067	       RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
3068
3069	fw_data = (const __be32 *)rdev->pfp_fw->data;
3070	WREG32(CP_PFP_UCODE_ADDR, 0);
3071	for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
3072		WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3073	WREG32(CP_PFP_UCODE_ADDR, 0);
3074
3075	fw_data = (const __be32 *)rdev->me_fw->data;
3076	WREG32(CP_ME_RAM_WADDR, 0);
3077	for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
3078		WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3079
3080	WREG32(CP_PFP_UCODE_ADDR, 0);
3081	WREG32(CP_ME_RAM_WADDR, 0);
3082	WREG32(CP_ME_RAM_RADDR, 0);
3083	return 0;
3084}
3085
3086static int evergreen_cp_start(struct radeon_device *rdev)
3087{
3088	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3089	int r, i;
3090	uint32_t cp_me;
3091
3092	r = radeon_ring_lock(rdev, ring, 7);
3093	if (r) {
3094		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3095		return r;
3096	}
3097	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3098	radeon_ring_write(ring, 0x1);
3099	radeon_ring_write(ring, 0x0);
3100	radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3101	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3102	radeon_ring_write(ring, 0);
3103	radeon_ring_write(ring, 0);
3104	radeon_ring_unlock_commit(rdev, ring, false);
3105
3106	cp_me = 0xff;
3107	WREG32(CP_ME_CNTL, cp_me);
3108
3109	r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3110	if (r) {
3111		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3112		return r;
3113	}
3114
3115	/* setup clear context state */
3116	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3117	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3118
3119	for (i = 0; i < evergreen_default_size; i++)
3120		radeon_ring_write(ring, evergreen_default_state[i]);
3121
3122	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3123	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3124
3125	/* set clear context state */
3126	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3127	radeon_ring_write(ring, 0);
3128
3129	/* SQ_VTX_BASE_VTX_LOC */
3130	radeon_ring_write(ring, 0xc0026f00);
3131	radeon_ring_write(ring, 0x00000000);
3132	radeon_ring_write(ring, 0x00000000);
3133	radeon_ring_write(ring, 0x00000000);
3134
3135	/* Clear consts */
3136	radeon_ring_write(ring, 0xc0036f00);
3137	radeon_ring_write(ring, 0x00000bc4);
3138	radeon_ring_write(ring, 0xffffffff);
3139	radeon_ring_write(ring, 0xffffffff);
3140	radeon_ring_write(ring, 0xffffffff);
3141
3142	radeon_ring_write(ring, 0xc0026900);
3143	radeon_ring_write(ring, 0x00000316);
3144	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3145	radeon_ring_write(ring, 0x00000010); /*  */
3146
3147	radeon_ring_unlock_commit(rdev, ring, false);
3148
3149	return 0;
3150}
3151
3152static int evergreen_cp_resume(struct radeon_device *rdev)
3153{
3154	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3155	u32 tmp;
3156	u32 rb_bufsz;
3157	int r;
3158
3159	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3160	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3161				 SOFT_RESET_PA |
3162				 SOFT_RESET_SH |
3163				 SOFT_RESET_VGT |
3164				 SOFT_RESET_SPI |
3165				 SOFT_RESET_SX));
3166	RREG32(GRBM_SOFT_RESET);
3167	mdelay(15);
3168	WREG32(GRBM_SOFT_RESET, 0);
3169	RREG32(GRBM_SOFT_RESET);
3170
3171	/* Set ring buffer size */
3172	rb_bufsz = order_base_2(ring->ring_size / 8);
3173	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3174#ifdef __BIG_ENDIAN
3175	tmp |= BUF_SWAP_32BIT;
3176#endif
3177	WREG32(CP_RB_CNTL, tmp);
3178	WREG32(CP_SEM_WAIT_TIMER, 0x0);
3179	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3180
3181	/* Set the write pointer delay */
3182	WREG32(CP_RB_WPTR_DELAY, 0);
3183
3184	/* Initialize the ring buffer's read and write pointers */
3185	WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3186	WREG32(CP_RB_RPTR_WR, 0);
3187	ring->wptr = 0;
3188	WREG32(CP_RB_WPTR, ring->wptr);
3189
3190	/* set the wb address whether it's enabled or not */
3191	WREG32(CP_RB_RPTR_ADDR,
3192	       ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3193	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3194	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3195
3196	if (rdev->wb.enabled)
3197		WREG32(SCRATCH_UMSK, 0xff);
3198	else {
3199		tmp |= RB_NO_UPDATE;
3200		WREG32(SCRATCH_UMSK, 0);
3201	}
3202
3203	mdelay(1);
3204	WREG32(CP_RB_CNTL, tmp);
3205
3206	WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3207	WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3208
3209	evergreen_cp_start(rdev);
3210	ring->ready = true;
3211	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3212	if (r) {
3213		ring->ready = false;
3214		return r;
3215	}
3216	return 0;
3217}
3218
3219/*
3220 * Core functions
3221 */
3222static void evergreen_gpu_init(struct radeon_device *rdev)
3223{
3224	u32 gb_addr_config;
3225	u32 mc_shared_chmap, mc_arb_ramcfg;
3226	u32 sx_debug_1;
3227	u32 smx_dc_ctl0;
3228	u32 sq_config;
3229	u32 sq_lds_resource_mgmt;
3230	u32 sq_gpr_resource_mgmt_1;
3231	u32 sq_gpr_resource_mgmt_2;
3232	u32 sq_gpr_resource_mgmt_3;
3233	u32 sq_thread_resource_mgmt;
3234	u32 sq_thread_resource_mgmt_2;
3235	u32 sq_stack_resource_mgmt_1;
3236	u32 sq_stack_resource_mgmt_2;
3237	u32 sq_stack_resource_mgmt_3;
3238	u32 vgt_cache_invalidation;
3239	u32 hdp_host_path_cntl, tmp;
3240	u32 disabled_rb_mask;
3241	int i, j, ps_thread_count;
3242
3243	switch (rdev->family) {
3244	case CHIP_CYPRESS:
3245	case CHIP_HEMLOCK:
3246		rdev->config.evergreen.num_ses = 2;
3247		rdev->config.evergreen.max_pipes = 4;
3248		rdev->config.evergreen.max_tile_pipes = 8;
3249		rdev->config.evergreen.max_simds = 10;
3250		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3251		rdev->config.evergreen.max_gprs = 256;
3252		rdev->config.evergreen.max_threads = 248;
3253		rdev->config.evergreen.max_gs_threads = 32;
3254		rdev->config.evergreen.max_stack_entries = 512;
3255		rdev->config.evergreen.sx_num_of_sets = 4;
3256		rdev->config.evergreen.sx_max_export_size = 256;
3257		rdev->config.evergreen.sx_max_export_pos_size = 64;
3258		rdev->config.evergreen.sx_max_export_smx_size = 192;
3259		rdev->config.evergreen.max_hw_contexts = 8;
3260		rdev->config.evergreen.sq_num_cf_insts = 2;
3261
3262		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3263		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3264		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3265		gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3266		break;
3267	case CHIP_JUNIPER:
3268		rdev->config.evergreen.num_ses = 1;
3269		rdev->config.evergreen.max_pipes = 4;
3270		rdev->config.evergreen.max_tile_pipes = 4;
3271		rdev->config.evergreen.max_simds = 10;
3272		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3273		rdev->config.evergreen.max_gprs = 256;
3274		rdev->config.evergreen.max_threads = 248;
3275		rdev->config.evergreen.max_gs_threads = 32;
3276		rdev->config.evergreen.max_stack_entries = 512;
3277		rdev->config.evergreen.sx_num_of_sets = 4;
3278		rdev->config.evergreen.sx_max_export_size = 256;
3279		rdev->config.evergreen.sx_max_export_pos_size = 64;
3280		rdev->config.evergreen.sx_max_export_smx_size = 192;
3281		rdev->config.evergreen.max_hw_contexts = 8;
3282		rdev->config.evergreen.sq_num_cf_insts = 2;
3283
3284		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3285		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3286		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3287		gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3288		break;
3289	case CHIP_REDWOOD:
3290		rdev->config.evergreen.num_ses = 1;
3291		rdev->config.evergreen.max_pipes = 4;
3292		rdev->config.evergreen.max_tile_pipes = 4;
3293		rdev->config.evergreen.max_simds = 5;
3294		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3295		rdev->config.evergreen.max_gprs = 256;
3296		rdev->config.evergreen.max_threads = 248;
3297		rdev->config.evergreen.max_gs_threads = 32;
3298		rdev->config.evergreen.max_stack_entries = 256;
3299		rdev->config.evergreen.sx_num_of_sets = 4;
3300		rdev->config.evergreen.sx_max_export_size = 256;
3301		rdev->config.evergreen.sx_max_export_pos_size = 64;
3302		rdev->config.evergreen.sx_max_export_smx_size = 192;
3303		rdev->config.evergreen.max_hw_contexts = 8;
3304		rdev->config.evergreen.sq_num_cf_insts = 2;
3305
3306		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3307		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3308		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3309		gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3310		break;
3311	case CHIP_CEDAR:
3312	default:
3313		rdev->config.evergreen.num_ses = 1;
3314		rdev->config.evergreen.max_pipes = 2;
3315		rdev->config.evergreen.max_tile_pipes = 2;
3316		rdev->config.evergreen.max_simds = 2;
3317		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3318		rdev->config.evergreen.max_gprs = 256;
3319		rdev->config.evergreen.max_threads = 192;
3320		rdev->config.evergreen.max_gs_threads = 16;
3321		rdev->config.evergreen.max_stack_entries = 256;
3322		rdev->config.evergreen.sx_num_of_sets = 4;
3323		rdev->config.evergreen.sx_max_export_size = 128;
3324		rdev->config.evergreen.sx_max_export_pos_size = 32;
3325		rdev->config.evergreen.sx_max_export_smx_size = 96;
3326		rdev->config.evergreen.max_hw_contexts = 4;
3327		rdev->config.evergreen.sq_num_cf_insts = 1;
3328
3329		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3330		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3331		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3332		gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3333		break;
3334	case CHIP_PALM:
3335		rdev->config.evergreen.num_ses = 1;
3336		rdev->config.evergreen.max_pipes = 2;
3337		rdev->config.evergreen.max_tile_pipes = 2;
3338		rdev->config.evergreen.max_simds = 2;
3339		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3340		rdev->config.evergreen.max_gprs = 256;
3341		rdev->config.evergreen.max_threads = 192;
3342		rdev->config.evergreen.max_gs_threads = 16;
3343		rdev->config.evergreen.max_stack_entries = 256;
3344		rdev->config.evergreen.sx_num_of_sets = 4;
3345		rdev->config.evergreen.sx_max_export_size = 128;
3346		rdev->config.evergreen.sx_max_export_pos_size = 32;
3347		rdev->config.evergreen.sx_max_export_smx_size = 96;
3348		rdev->config.evergreen.max_hw_contexts = 4;
3349		rdev->config.evergreen.sq_num_cf_insts = 1;
3350
3351		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3352		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3353		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3354		gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3355		break;
3356	case CHIP_SUMO:
3357		rdev->config.evergreen.num_ses = 1;
3358		rdev->config.evergreen.max_pipes = 4;
3359		rdev->config.evergreen.max_tile_pipes = 4;
3360		if (rdev->pdev->device == 0x9648)
3361			rdev->config.evergreen.max_simds = 3;
3362		else if ((rdev->pdev->device == 0x9647) ||
3363			 (rdev->pdev->device == 0x964a))
3364			rdev->config.evergreen.max_simds = 4;
3365		else
3366			rdev->config.evergreen.max_simds = 5;
3367		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3368		rdev->config.evergreen.max_gprs = 256;
3369		rdev->config.evergreen.max_threads = 248;
3370		rdev->config.evergreen.max_gs_threads = 32;
3371		rdev->config.evergreen.max_stack_entries = 256;
3372		rdev->config.evergreen.sx_num_of_sets = 4;
3373		rdev->config.evergreen.sx_max_export_size = 256;
3374		rdev->config.evergreen.sx_max_export_pos_size = 64;
3375		rdev->config.evergreen.sx_max_export_smx_size = 192;
3376		rdev->config.evergreen.max_hw_contexts = 8;
3377		rdev->config.evergreen.sq_num_cf_insts = 2;
3378
3379		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3380		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3381		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3382		gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3383		break;
3384	case CHIP_SUMO2:
3385		rdev->config.evergreen.num_ses = 1;
3386		rdev->config.evergreen.max_pipes = 4;
3387		rdev->config.evergreen.max_tile_pipes = 4;
3388		rdev->config.evergreen.max_simds = 2;
3389		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3390		rdev->config.evergreen.max_gprs = 256;
3391		rdev->config.evergreen.max_threads = 248;
3392		rdev->config.evergreen.max_gs_threads = 32;
3393		rdev->config.evergreen.max_stack_entries = 512;
3394		rdev->config.evergreen.sx_num_of_sets = 4;
3395		rdev->config.evergreen.sx_max_export_size = 256;
3396		rdev->config.evergreen.sx_max_export_pos_size = 64;
3397		rdev->config.evergreen.sx_max_export_smx_size = 192;
3398		rdev->config.evergreen.max_hw_contexts = 4;
3399		rdev->config.evergreen.sq_num_cf_insts = 2;
3400
3401		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3402		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3403		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3404		gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3405		break;
3406	case CHIP_BARTS:
3407		rdev->config.evergreen.num_ses = 2;
3408		rdev->config.evergreen.max_pipes = 4;
3409		rdev->config.evergreen.max_tile_pipes = 8;
3410		rdev->config.evergreen.max_simds = 7;
3411		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3412		rdev->config.evergreen.max_gprs = 256;
3413		rdev->config.evergreen.max_threads = 248;
3414		rdev->config.evergreen.max_gs_threads = 32;
3415		rdev->config.evergreen.max_stack_entries = 512;
3416		rdev->config.evergreen.sx_num_of_sets = 4;
3417		rdev->config.evergreen.sx_max_export_size = 256;
3418		rdev->config.evergreen.sx_max_export_pos_size = 64;
3419		rdev->config.evergreen.sx_max_export_smx_size = 192;
3420		rdev->config.evergreen.max_hw_contexts = 8;
3421		rdev->config.evergreen.sq_num_cf_insts = 2;
3422
3423		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3424		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3425		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3426		gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3427		break;
3428	case CHIP_TURKS:
3429		rdev->config.evergreen.num_ses = 1;
3430		rdev->config.evergreen.max_pipes = 4;
3431		rdev->config.evergreen.max_tile_pipes = 4;
3432		rdev->config.evergreen.max_simds = 6;
3433		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3434		rdev->config.evergreen.max_gprs = 256;
3435		rdev->config.evergreen.max_threads = 248;
3436		rdev->config.evergreen.max_gs_threads = 32;
3437		rdev->config.evergreen.max_stack_entries = 256;
3438		rdev->config.evergreen.sx_num_of_sets = 4;
3439		rdev->config.evergreen.sx_max_export_size = 256;
3440		rdev->config.evergreen.sx_max_export_pos_size = 64;
3441		rdev->config.evergreen.sx_max_export_smx_size = 192;
3442		rdev->config.evergreen.max_hw_contexts = 8;
3443		rdev->config.evergreen.sq_num_cf_insts = 2;
3444
3445		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3446		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3447		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3448		gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3449		break;
3450	case CHIP_CAICOS:
3451		rdev->config.evergreen.num_ses = 1;
3452		rdev->config.evergreen.max_pipes = 2;
3453		rdev->config.evergreen.max_tile_pipes = 2;
3454		rdev->config.evergreen.max_simds = 2;
3455		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3456		rdev->config.evergreen.max_gprs = 256;
3457		rdev->config.evergreen.max_threads = 192;
3458		rdev->config.evergreen.max_gs_threads = 16;
3459		rdev->config.evergreen.max_stack_entries = 256;
3460		rdev->config.evergreen.sx_num_of_sets = 4;
3461		rdev->config.evergreen.sx_max_export_size = 128;
3462		rdev->config.evergreen.sx_max_export_pos_size = 32;
3463		rdev->config.evergreen.sx_max_export_smx_size = 96;
3464		rdev->config.evergreen.max_hw_contexts = 4;
3465		rdev->config.evergreen.sq_num_cf_insts = 1;
3466
3467		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3468		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3469		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3470		gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3471		break;
3472	}
3473
3474	/* Initialize HDP */
3475	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3476		WREG32((0x2c14 + j), 0x00000000);
3477		WREG32((0x2c18 + j), 0x00000000);
3478		WREG32((0x2c1c + j), 0x00000000);
3479		WREG32((0x2c20 + j), 0x00000000);
3480		WREG32((0x2c24 + j), 0x00000000);
3481	}
3482
3483	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3484	WREG32(SRBM_INT_CNTL, 0x1);
3485	WREG32(SRBM_INT_ACK, 0x1);
3486
3487	evergreen_fix_pci_max_read_req_size(rdev);
3488
3489	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3490	if ((rdev->family == CHIP_PALM) ||
3491	    (rdev->family == CHIP_SUMO) ||
3492	    (rdev->family == CHIP_SUMO2))
3493		mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3494	else
3495		mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3496
3497	/* setup tiling info dword.  gb_addr_config is not adequate since it does
3498	 * not have bank info, so create a custom tiling dword.
3499	 * bits 3:0   num_pipes
3500	 * bits 7:4   num_banks
3501	 * bits 11:8  group_size
3502	 * bits 15:12 row_size
3503	 */
3504	rdev->config.evergreen.tile_config = 0;
3505	switch (rdev->config.evergreen.max_tile_pipes) {
3506	case 1:
3507	default:
3508		rdev->config.evergreen.tile_config |= (0 << 0);
3509		break;
3510	case 2:
3511		rdev->config.evergreen.tile_config |= (1 << 0);
3512		break;
3513	case 4:
3514		rdev->config.evergreen.tile_config |= (2 << 0);
3515		break;
3516	case 8:
3517		rdev->config.evergreen.tile_config |= (3 << 0);
3518		break;
3519	}
3520	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3521	if (rdev->flags & RADEON_IS_IGP)
3522		rdev->config.evergreen.tile_config |= 1 << 4;
3523	else {
3524		switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3525		case 0: /* four banks */
3526			rdev->config.evergreen.tile_config |= 0 << 4;
3527			break;
3528		case 1: /* eight banks */
3529			rdev->config.evergreen.tile_config |= 1 << 4;
3530			break;
3531		case 2: /* sixteen banks */
3532		default:
3533			rdev->config.evergreen.tile_config |= 2 << 4;
3534			break;
3535		}
3536	}
3537	rdev->config.evergreen.tile_config |= 0 << 8;
3538	rdev->config.evergreen.tile_config |=
3539		((gb_addr_config & 0x30000000) >> 28) << 12;
3540
 
 
3541	if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3542		u32 efuse_straps_4;
3543		u32 efuse_straps_3;
3544
3545		efuse_straps_4 = RREG32_RCU(0x204);
3546		efuse_straps_3 = RREG32_RCU(0x203);
3547		tmp = (((efuse_straps_4 & 0xf) << 4) |
3548		      ((efuse_straps_3 & 0xf0000000) >> 28));
3549	} else {
3550		tmp = 0;
3551		for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3552			u32 rb_disable_bitmap;
3553
3554			WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3555			WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3556			rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3557			tmp <<= 4;
3558			tmp |= rb_disable_bitmap;
3559		}
3560	}
3561	/* enabled rb are just the one not disabled :) */
3562	disabled_rb_mask = tmp;
3563	tmp = 0;
3564	for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3565		tmp |= (1 << i);
3566	/* if all the backends are disabled, fix it up here */
3567	if ((disabled_rb_mask & tmp) == tmp) {
3568		for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3569			disabled_rb_mask &= ~(1 << i);
3570	}
3571
3572	for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3573		u32 simd_disable_bitmap;
3574
3575		WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3576		WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3577		simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3578		simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3579		tmp <<= 16;
3580		tmp |= simd_disable_bitmap;
3581	}
3582	rdev->config.evergreen.active_simds = hweight32(~tmp);
3583
3584	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3585	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3586
3587	WREG32(GB_ADDR_CONFIG, gb_addr_config);
3588	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3589	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3590	WREG32(DMA_TILING_CONFIG, gb_addr_config);
3591	WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3592	WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3593	WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3594
3595	if ((rdev->config.evergreen.max_backends == 1) &&
3596	    (rdev->flags & RADEON_IS_IGP)) {
3597		if ((disabled_rb_mask & 3) == 1) {
3598			/* RB0 disabled, RB1 enabled */
3599			tmp = 0x11111111;
3600		} else {
3601			/* RB1 disabled, RB0 enabled */
3602			tmp = 0x00000000;
3603		}
3604	} else {
3605		tmp = gb_addr_config & NUM_PIPES_MASK;
3606		tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3607						EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3608	}
3609	WREG32(GB_BACKEND_MAP, tmp);
3610
3611	WREG32(CGTS_SYS_TCC_DISABLE, 0);
3612	WREG32(CGTS_TCC_DISABLE, 0);
3613	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3614	WREG32(CGTS_USER_TCC_DISABLE, 0);
3615
3616	/* set HW defaults for 3D engine */
3617	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3618				     ROQ_IB2_START(0x2b)));
3619
3620	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3621
3622	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3623			     SYNC_GRADIENT |
3624			     SYNC_WALKER |
3625			     SYNC_ALIGNER));
3626
3627	sx_debug_1 = RREG32(SX_DEBUG_1);
3628	sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3629	WREG32(SX_DEBUG_1, sx_debug_1);
3630
3631
3632	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3633	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3634	smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3635	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3636
3637	if (rdev->family <= CHIP_SUMO2)
3638		WREG32(SMX_SAR_CTL0, 0x00010000);
3639
3640	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3641					POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3642					SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3643
3644	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3645				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3646				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3647
3648	WREG32(VGT_NUM_INSTANCES, 1);
3649	WREG32(SPI_CONFIG_CNTL, 0);
3650	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3651	WREG32(CP_PERFMON_CNTL, 0);
3652
3653	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3654				  FETCH_FIFO_HIWATER(0x4) |
3655				  DONE_FIFO_HIWATER(0xe0) |
3656				  ALU_UPDATE_FIFO_HIWATER(0x8)));
3657
3658	sq_config = RREG32(SQ_CONFIG);
3659	sq_config &= ~(PS_PRIO(3) |
3660		       VS_PRIO(3) |
3661		       GS_PRIO(3) |
3662		       ES_PRIO(3));
3663	sq_config |= (VC_ENABLE |
3664		      EXPORT_SRC_C |
3665		      PS_PRIO(0) |
3666		      VS_PRIO(1) |
3667		      GS_PRIO(2) |
3668		      ES_PRIO(3));
3669
3670	switch (rdev->family) {
3671	case CHIP_CEDAR:
3672	case CHIP_PALM:
3673	case CHIP_SUMO:
3674	case CHIP_SUMO2:
3675	case CHIP_CAICOS:
3676		/* no vertex cache */
3677		sq_config &= ~VC_ENABLE;
3678		break;
3679	default:
3680		break;
3681	}
3682
3683	sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3684
3685	sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3686	sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3687	sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3688	sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3689	sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3690	sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3691	sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3692
3693	switch (rdev->family) {
3694	case CHIP_CEDAR:
3695	case CHIP_PALM:
3696	case CHIP_SUMO:
3697	case CHIP_SUMO2:
3698		ps_thread_count = 96;
3699		break;
3700	default:
3701		ps_thread_count = 128;
3702		break;
3703	}
3704
3705	sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3706	sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3707	sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3708	sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3709	sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3710	sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3711
3712	sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3713	sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3714	sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3715	sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3716	sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3717	sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3718
3719	WREG32(SQ_CONFIG, sq_config);
3720	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3721	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3722	WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3723	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3724	WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3725	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3726	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3727	WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3728	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3729	WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3730
3731	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3732					  FORCE_EOV_MAX_REZ_CNT(255)));
3733
3734	switch (rdev->family) {
3735	case CHIP_CEDAR:
3736	case CHIP_PALM:
3737	case CHIP_SUMO:
3738	case CHIP_SUMO2:
3739	case CHIP_CAICOS:
3740		vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3741		break;
3742	default:
3743		vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3744		break;
3745	}
3746	vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3747	WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3748
3749	WREG32(VGT_GS_VERTEX_REUSE, 16);
3750	WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3751	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3752
3753	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3754	WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3755
3756	WREG32(CB_PERF_CTR0_SEL_0, 0);
3757	WREG32(CB_PERF_CTR0_SEL_1, 0);
3758	WREG32(CB_PERF_CTR1_SEL_0, 0);
3759	WREG32(CB_PERF_CTR1_SEL_1, 0);
3760	WREG32(CB_PERF_CTR2_SEL_0, 0);
3761	WREG32(CB_PERF_CTR2_SEL_1, 0);
3762	WREG32(CB_PERF_CTR3_SEL_0, 0);
3763	WREG32(CB_PERF_CTR3_SEL_1, 0);
3764
3765	/* clear render buffer base addresses */
3766	WREG32(CB_COLOR0_BASE, 0);
3767	WREG32(CB_COLOR1_BASE, 0);
3768	WREG32(CB_COLOR2_BASE, 0);
3769	WREG32(CB_COLOR3_BASE, 0);
3770	WREG32(CB_COLOR4_BASE, 0);
3771	WREG32(CB_COLOR5_BASE, 0);
3772	WREG32(CB_COLOR6_BASE, 0);
3773	WREG32(CB_COLOR7_BASE, 0);
3774	WREG32(CB_COLOR8_BASE, 0);
3775	WREG32(CB_COLOR9_BASE, 0);
3776	WREG32(CB_COLOR10_BASE, 0);
3777	WREG32(CB_COLOR11_BASE, 0);
3778
3779	/* set the shader const cache sizes to 0 */
3780	for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3781		WREG32(i, 0);
3782	for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3783		WREG32(i, 0);
3784
3785	tmp = RREG32(HDP_MISC_CNTL);
3786	tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3787	WREG32(HDP_MISC_CNTL, tmp);
3788
3789	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3790	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3791
3792	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3793
3794	udelay(50);
3795
3796}
3797
3798int evergreen_mc_init(struct radeon_device *rdev)
3799{
3800	u32 tmp;
3801	int chansize, numchan;
3802
3803	/* Get VRAM informations */
3804	rdev->mc.vram_is_ddr = true;
3805	if ((rdev->family == CHIP_PALM) ||
3806	    (rdev->family == CHIP_SUMO) ||
3807	    (rdev->family == CHIP_SUMO2))
3808		tmp = RREG32(FUS_MC_ARB_RAMCFG);
3809	else
3810		tmp = RREG32(MC_ARB_RAMCFG);
3811	if (tmp & CHANSIZE_OVERRIDE) {
3812		chansize = 16;
3813	} else if (tmp & CHANSIZE_MASK) {
3814		chansize = 64;
3815	} else {
3816		chansize = 32;
3817	}
3818	tmp = RREG32(MC_SHARED_CHMAP);
3819	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3820	case 0:
3821	default:
3822		numchan = 1;
3823		break;
3824	case 1:
3825		numchan = 2;
3826		break;
3827	case 2:
3828		numchan = 4;
3829		break;
3830	case 3:
3831		numchan = 8;
3832		break;
3833	}
3834	rdev->mc.vram_width = numchan * chansize;
3835	/* Could aper size report 0 ? */
3836	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3837	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3838	/* Setup GPU memory space */
3839	if ((rdev->family == CHIP_PALM) ||
3840	    (rdev->family == CHIP_SUMO) ||
3841	    (rdev->family == CHIP_SUMO2)) {
3842		/* size in bytes on fusion */
3843		rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3844		rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3845	} else {
3846		/* size in MB on evergreen/cayman/tn */
3847		rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3848		rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3849	}
3850	rdev->mc.visible_vram_size = rdev->mc.aper_size;
3851	r700_vram_gtt_location(rdev, &rdev->mc);
3852	radeon_update_bandwidth_info(rdev);
3853
3854	return 0;
3855}
3856
3857void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3858{
3859	dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3860		RREG32(GRBM_STATUS));
3861	dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3862		RREG32(GRBM_STATUS_SE0));
3863	dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3864		RREG32(GRBM_STATUS_SE1));
3865	dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3866		RREG32(SRBM_STATUS));
3867	dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3868		RREG32(SRBM_STATUS2));
3869	dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3870		RREG32(CP_STALLED_STAT1));
3871	dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3872		RREG32(CP_STALLED_STAT2));
3873	dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3874		RREG32(CP_BUSY_STAT));
3875	dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3876		RREG32(CP_STAT));
3877	dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3878		RREG32(DMA_STATUS_REG));
3879	if (rdev->family >= CHIP_CAYMAN) {
3880		dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3881			 RREG32(DMA_STATUS_REG + 0x800));
3882	}
3883}
3884
3885bool evergreen_is_display_hung(struct radeon_device *rdev)
3886{
3887	u32 crtc_hung = 0;
3888	u32 crtc_status[6];
3889	u32 i, j, tmp;
3890
3891	for (i = 0; i < rdev->num_crtc; i++) {
3892		if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3893			crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3894			crtc_hung |= (1 << i);
3895		}
3896	}
3897
3898	for (j = 0; j < 10; j++) {
3899		for (i = 0; i < rdev->num_crtc; i++) {
3900			if (crtc_hung & (1 << i)) {
3901				tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3902				if (tmp != crtc_status[i])
3903					crtc_hung &= ~(1 << i);
3904			}
3905		}
3906		if (crtc_hung == 0)
3907			return false;
3908		udelay(100);
3909	}
3910
3911	return true;
3912}
3913
3914u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3915{
3916	u32 reset_mask = 0;
3917	u32 tmp;
3918
3919	/* GRBM_STATUS */
3920	tmp = RREG32(GRBM_STATUS);
3921	if (tmp & (PA_BUSY | SC_BUSY |
3922		   SH_BUSY | SX_BUSY |
3923		   TA_BUSY | VGT_BUSY |
3924		   DB_BUSY | CB_BUSY |
3925		   SPI_BUSY | VGT_BUSY_NO_DMA))
3926		reset_mask |= RADEON_RESET_GFX;
3927
3928	if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3929		   CP_BUSY | CP_COHERENCY_BUSY))
3930		reset_mask |= RADEON_RESET_CP;
3931
3932	if (tmp & GRBM_EE_BUSY)
3933		reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3934
3935	/* DMA_STATUS_REG */
3936	tmp = RREG32(DMA_STATUS_REG);
3937	if (!(tmp & DMA_IDLE))
3938		reset_mask |= RADEON_RESET_DMA;
3939
3940	/* SRBM_STATUS2 */
3941	tmp = RREG32(SRBM_STATUS2);
3942	if (tmp & DMA_BUSY)
3943		reset_mask |= RADEON_RESET_DMA;
3944
3945	/* SRBM_STATUS */
3946	tmp = RREG32(SRBM_STATUS);
3947	if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3948		reset_mask |= RADEON_RESET_RLC;
3949
3950	if (tmp & IH_BUSY)
3951		reset_mask |= RADEON_RESET_IH;
3952
3953	if (tmp & SEM_BUSY)
3954		reset_mask |= RADEON_RESET_SEM;
3955
3956	if (tmp & GRBM_RQ_PENDING)
3957		reset_mask |= RADEON_RESET_GRBM;
3958
3959	if (tmp & VMC_BUSY)
3960		reset_mask |= RADEON_RESET_VMC;
3961
3962	if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3963		   MCC_BUSY | MCD_BUSY))
3964		reset_mask |= RADEON_RESET_MC;
3965
3966	if (evergreen_is_display_hung(rdev))
3967		reset_mask |= RADEON_RESET_DISPLAY;
3968
3969	/* VM_L2_STATUS */
3970	tmp = RREG32(VM_L2_STATUS);
3971	if (tmp & L2_BUSY)
3972		reset_mask |= RADEON_RESET_VMC;
3973
3974	/* Skip MC reset as it's mostly likely not hung, just busy */
3975	if (reset_mask & RADEON_RESET_MC) {
3976		DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3977		reset_mask &= ~RADEON_RESET_MC;
3978	}
3979
3980	return reset_mask;
3981}
3982
3983static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3984{
3985	struct evergreen_mc_save save;
3986	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3987	u32 tmp;
3988
3989	if (reset_mask == 0)
3990		return;
3991
3992	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3993
3994	evergreen_print_gpu_status_regs(rdev);
3995
3996	/* Disable CP parsing/prefetching */
3997	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3998
3999	if (reset_mask & RADEON_RESET_DMA) {
4000		/* Disable DMA */
4001		tmp = RREG32(DMA_RB_CNTL);
4002		tmp &= ~DMA_RB_ENABLE;
4003		WREG32(DMA_RB_CNTL, tmp);
4004	}
4005
4006	udelay(50);
4007
4008	evergreen_mc_stop(rdev, &save);
4009	if (evergreen_mc_wait_for_idle(rdev)) {
4010		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4011	}
4012
4013	if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
4014		grbm_soft_reset |= SOFT_RESET_DB |
4015			SOFT_RESET_CB |
4016			SOFT_RESET_PA |
4017			SOFT_RESET_SC |
4018			SOFT_RESET_SPI |
4019			SOFT_RESET_SX |
4020			SOFT_RESET_SH |
4021			SOFT_RESET_TC |
4022			SOFT_RESET_TA |
4023			SOFT_RESET_VC |
4024			SOFT_RESET_VGT;
4025	}
4026
4027	if (reset_mask & RADEON_RESET_CP) {
4028		grbm_soft_reset |= SOFT_RESET_CP |
4029			SOFT_RESET_VGT;
4030
4031		srbm_soft_reset |= SOFT_RESET_GRBM;
4032	}
4033
4034	if (reset_mask & RADEON_RESET_DMA)
4035		srbm_soft_reset |= SOFT_RESET_DMA;
4036
4037	if (reset_mask & RADEON_RESET_DISPLAY)
4038		srbm_soft_reset |= SOFT_RESET_DC;
4039
4040	if (reset_mask & RADEON_RESET_RLC)
4041		srbm_soft_reset |= SOFT_RESET_RLC;
4042
4043	if (reset_mask & RADEON_RESET_SEM)
4044		srbm_soft_reset |= SOFT_RESET_SEM;
4045
4046	if (reset_mask & RADEON_RESET_IH)
4047		srbm_soft_reset |= SOFT_RESET_IH;
4048
4049	if (reset_mask & RADEON_RESET_GRBM)
4050		srbm_soft_reset |= SOFT_RESET_GRBM;
4051
4052	if (reset_mask & RADEON_RESET_VMC)
4053		srbm_soft_reset |= SOFT_RESET_VMC;
4054
4055	if (!(rdev->flags & RADEON_IS_IGP)) {
4056		if (reset_mask & RADEON_RESET_MC)
4057			srbm_soft_reset |= SOFT_RESET_MC;
4058	}
4059
4060	if (grbm_soft_reset) {
4061		tmp = RREG32(GRBM_SOFT_RESET);
4062		tmp |= grbm_soft_reset;
4063		dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4064		WREG32(GRBM_SOFT_RESET, tmp);
4065		tmp = RREG32(GRBM_SOFT_RESET);
4066
4067		udelay(50);
4068
4069		tmp &= ~grbm_soft_reset;
4070		WREG32(GRBM_SOFT_RESET, tmp);
4071		tmp = RREG32(GRBM_SOFT_RESET);
4072	}
4073
4074	if (srbm_soft_reset) {
4075		tmp = RREG32(SRBM_SOFT_RESET);
4076		tmp |= srbm_soft_reset;
4077		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4078		WREG32(SRBM_SOFT_RESET, tmp);
4079		tmp = RREG32(SRBM_SOFT_RESET);
4080
4081		udelay(50);
4082
4083		tmp &= ~srbm_soft_reset;
4084		WREG32(SRBM_SOFT_RESET, tmp);
4085		tmp = RREG32(SRBM_SOFT_RESET);
4086	}
4087
4088	/* Wait a little for things to settle down */
4089	udelay(50);
4090
4091	evergreen_mc_resume(rdev, &save);
4092	udelay(50);
4093
4094	evergreen_print_gpu_status_regs(rdev);
4095}
4096
4097void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4098{
4099	struct evergreen_mc_save save;
4100	u32 tmp, i;
4101
4102	dev_info(rdev->dev, "GPU pci config reset\n");
4103
4104	/* disable dpm? */
4105
4106	/* Disable CP parsing/prefetching */
4107	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4108	udelay(50);
4109	/* Disable DMA */
4110	tmp = RREG32(DMA_RB_CNTL);
4111	tmp &= ~DMA_RB_ENABLE;
4112	WREG32(DMA_RB_CNTL, tmp);
4113	/* XXX other engines? */
4114
4115	/* halt the rlc */
4116	r600_rlc_stop(rdev);
4117
4118	udelay(50);
4119
4120	/* set mclk/sclk to bypass */
4121	rv770_set_clk_bypass_mode(rdev);
4122	/* disable BM */
4123	pci_clear_master(rdev->pdev);
4124	/* disable mem access */
4125	evergreen_mc_stop(rdev, &save);
4126	if (evergreen_mc_wait_for_idle(rdev)) {
4127		dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4128	}
4129	/* reset */
4130	radeon_pci_config_reset(rdev);
4131	/* wait for asic to come out of reset */
4132	for (i = 0; i < rdev->usec_timeout; i++) {
4133		if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4134			break;
4135		udelay(1);
4136	}
4137}
4138
4139int evergreen_asic_reset(struct radeon_device *rdev)
4140{
4141	u32 reset_mask;
4142
4143	reset_mask = evergreen_gpu_check_soft_reset(rdev);
4144
4145	if (reset_mask)
4146		r600_set_bios_scratch_engine_hung(rdev, true);
4147
4148	/* try soft reset */
4149	evergreen_gpu_soft_reset(rdev, reset_mask);
4150
4151	reset_mask = evergreen_gpu_check_soft_reset(rdev);
4152
4153	/* try pci config reset */
4154	if (reset_mask && radeon_hard_reset)
4155		evergreen_gpu_pci_config_reset(rdev);
4156
4157	reset_mask = evergreen_gpu_check_soft_reset(rdev);
4158
4159	if (!reset_mask)
4160		r600_set_bios_scratch_engine_hung(rdev, false);
4161
4162	return 0;
4163}
4164
4165/**
4166 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4167 *
4168 * @rdev: radeon_device pointer
4169 * @ring: radeon_ring structure holding ring information
4170 *
4171 * Check if the GFX engine is locked up.
4172 * Returns true if the engine appears to be locked up, false if not.
4173 */
4174bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4175{
4176	u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4177
4178	if (!(reset_mask & (RADEON_RESET_GFX |
4179			    RADEON_RESET_COMPUTE |
4180			    RADEON_RESET_CP))) {
4181		radeon_ring_lockup_update(rdev, ring);
4182		return false;
4183	}
4184	return radeon_ring_test_lockup(rdev, ring);
4185}
4186
4187/*
4188 * RLC
4189 */
4190#define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
4191#define RLC_CLEAR_STATE_END_MARKER          0x00000001
4192
4193void sumo_rlc_fini(struct radeon_device *rdev)
4194{
4195	int r;
4196
4197	/* save restore block */
4198	if (rdev->rlc.save_restore_obj) {
4199		r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4200		if (unlikely(r != 0))
4201			dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4202		radeon_bo_unpin(rdev->rlc.save_restore_obj);
4203		radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4204
4205		radeon_bo_unref(&rdev->rlc.save_restore_obj);
4206		rdev->rlc.save_restore_obj = NULL;
4207	}
4208
4209	/* clear state block */
4210	if (rdev->rlc.clear_state_obj) {
4211		r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4212		if (unlikely(r != 0))
4213			dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4214		radeon_bo_unpin(rdev->rlc.clear_state_obj);
4215		radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4216
4217		radeon_bo_unref(&rdev->rlc.clear_state_obj);
4218		rdev->rlc.clear_state_obj = NULL;
4219	}
4220
4221	/* clear state block */
4222	if (rdev->rlc.cp_table_obj) {
4223		r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4224		if (unlikely(r != 0))
4225			dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4226		radeon_bo_unpin(rdev->rlc.cp_table_obj);
4227		radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4228
4229		radeon_bo_unref(&rdev->rlc.cp_table_obj);
4230		rdev->rlc.cp_table_obj = NULL;
4231	}
4232}
4233
4234#define CP_ME_TABLE_SIZE    96
4235
4236int sumo_rlc_init(struct radeon_device *rdev)
4237{
4238	const u32 *src_ptr;
4239	volatile u32 *dst_ptr;
4240	u32 dws, data, i, j, k, reg_num;
4241	u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4242	u64 reg_list_mc_addr;
4243	const struct cs_section_def *cs_data;
4244	int r;
4245
4246	src_ptr = rdev->rlc.reg_list;
4247	dws = rdev->rlc.reg_list_size;
4248	if (rdev->family >= CHIP_BONAIRE) {
4249		dws += (5 * 16) + 48 + 48 + 64;
4250	}
4251	cs_data = rdev->rlc.cs_data;
4252
4253	if (src_ptr) {
4254		/* save restore block */
4255		if (rdev->rlc.save_restore_obj == NULL) {
4256			r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4257					     RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4258					     NULL, &rdev->rlc.save_restore_obj);
4259			if (r) {
4260				dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4261				return r;
4262			}
4263		}
4264
4265		r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4266		if (unlikely(r != 0)) {
4267			sumo_rlc_fini(rdev);
4268			return r;
4269		}
4270		r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4271				  &rdev->rlc.save_restore_gpu_addr);
4272		if (r) {
4273			radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4274			dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4275			sumo_rlc_fini(rdev);
4276			return r;
4277		}
4278
4279		r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4280		if (r) {
4281			dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4282			sumo_rlc_fini(rdev);
4283			return r;
4284		}
4285		/* write the sr buffer */
4286		dst_ptr = rdev->rlc.sr_ptr;
4287		if (rdev->family >= CHIP_TAHITI) {
4288			/* SI */
4289			for (i = 0; i < rdev->rlc.reg_list_size; i++)
4290				dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4291		} else {
4292			/* ON/LN/TN */
4293			/* format:
4294			 * dw0: (reg2 << 16) | reg1
4295			 * dw1: reg1 save space
4296			 * dw2: reg2 save space
4297			 */
4298			for (i = 0; i < dws; i++) {
4299				data = src_ptr[i] >> 2;
4300				i++;
4301				if (i < dws)
4302					data |= (src_ptr[i] >> 2) << 16;
4303				j = (((i - 1) * 3) / 2);
4304				dst_ptr[j] = cpu_to_le32(data);
4305			}
4306			j = ((i * 3) / 2);
4307			dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4308		}
4309		radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4310		radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4311	}
4312
4313	if (cs_data) {
4314		/* clear state block */
4315		if (rdev->family >= CHIP_BONAIRE) {
4316			rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4317		} else if (rdev->family >= CHIP_TAHITI) {
4318			rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4319			dws = rdev->rlc.clear_state_size + (256 / 4);
4320		} else {
4321			reg_list_num = 0;
4322			dws = 0;
4323			for (i = 0; cs_data[i].section != NULL; i++) {
4324				for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4325					reg_list_num++;
4326					dws += cs_data[i].section[j].reg_count;
4327				}
4328			}
4329			reg_list_blk_index = (3 * reg_list_num + 2);
4330			dws += reg_list_blk_index;
4331			rdev->rlc.clear_state_size = dws;
4332		}
4333
4334		if (rdev->rlc.clear_state_obj == NULL) {
4335			r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4336					     RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4337					     NULL, &rdev->rlc.clear_state_obj);
4338			if (r) {
4339				dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4340				sumo_rlc_fini(rdev);
4341				return r;
4342			}
4343		}
4344		r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4345		if (unlikely(r != 0)) {
4346			sumo_rlc_fini(rdev);
4347			return r;
4348		}
4349		r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4350				  &rdev->rlc.clear_state_gpu_addr);
4351		if (r) {
4352			radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4353			dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4354			sumo_rlc_fini(rdev);
4355			return r;
4356		}
4357
4358		r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4359		if (r) {
4360			dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4361			sumo_rlc_fini(rdev);
4362			return r;
4363		}
4364		/* set up the cs buffer */
4365		dst_ptr = rdev->rlc.cs_ptr;
4366		if (rdev->family >= CHIP_BONAIRE) {
4367			cik_get_csb_buffer(rdev, dst_ptr);
4368		} else if (rdev->family >= CHIP_TAHITI) {
4369			reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4370			dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4371			dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4372			dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4373			si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4374		} else {
4375			reg_list_hdr_blk_index = 0;
4376			reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4377			data = upper_32_bits(reg_list_mc_addr);
4378			dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4379			reg_list_hdr_blk_index++;
4380			for (i = 0; cs_data[i].section != NULL; i++) {
4381				for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4382					reg_num = cs_data[i].section[j].reg_count;
4383					data = reg_list_mc_addr & 0xffffffff;
4384					dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4385					reg_list_hdr_blk_index++;
4386
4387					data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4388					dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4389					reg_list_hdr_blk_index++;
4390
4391					data = 0x08000000 | (reg_num * 4);
4392					dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4393					reg_list_hdr_blk_index++;
4394
4395					for (k = 0; k < reg_num; k++) {
4396						data = cs_data[i].section[j].extent[k];
4397						dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4398					}
4399					reg_list_mc_addr += reg_num * 4;
4400					reg_list_blk_index += reg_num;
4401				}
4402			}
4403			dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4404		}
4405		radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4406		radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4407	}
4408
4409	if (rdev->rlc.cp_table_size) {
4410		if (rdev->rlc.cp_table_obj == NULL) {
4411			r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4412					     PAGE_SIZE, true,
4413					     RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4414					     NULL, &rdev->rlc.cp_table_obj);
4415			if (r) {
4416				dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4417				sumo_rlc_fini(rdev);
4418				return r;
4419			}
4420		}
4421
4422		r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4423		if (unlikely(r != 0)) {
4424			dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4425			sumo_rlc_fini(rdev);
4426			return r;
4427		}
4428		r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4429				  &rdev->rlc.cp_table_gpu_addr);
4430		if (r) {
4431			radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4432			dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4433			sumo_rlc_fini(rdev);
4434			return r;
4435		}
4436		r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4437		if (r) {
4438			dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4439			sumo_rlc_fini(rdev);
4440			return r;
4441		}
4442
4443		cik_init_cp_pg_table(rdev);
4444
4445		radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4446		radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4447
4448	}
4449
4450	return 0;
4451}
4452
4453static void evergreen_rlc_start(struct radeon_device *rdev)
4454{
4455	u32 mask = RLC_ENABLE;
4456
4457	if (rdev->flags & RADEON_IS_IGP) {
4458		mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4459	}
4460
4461	WREG32(RLC_CNTL, mask);
4462}
4463
4464int evergreen_rlc_resume(struct radeon_device *rdev)
4465{
4466	u32 i;
4467	const __be32 *fw_data;
4468
4469	if (!rdev->rlc_fw)
4470		return -EINVAL;
4471
4472	r600_rlc_stop(rdev);
4473
4474	WREG32(RLC_HB_CNTL, 0);
4475
4476	if (rdev->flags & RADEON_IS_IGP) {
4477		if (rdev->family == CHIP_ARUBA) {
4478			u32 always_on_bitmap =
4479				3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4480			/* find out the number of active simds */
4481			u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4482			tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4483			tmp = hweight32(~tmp);
4484			if (tmp == rdev->config.cayman.max_simds_per_se) {
4485				WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4486				WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4487				WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4488				WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4489				WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4490			}
4491		} else {
4492			WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4493			WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4494		}
4495		WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4496		WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4497	} else {
4498		WREG32(RLC_HB_BASE, 0);
4499		WREG32(RLC_HB_RPTR, 0);
4500		WREG32(RLC_HB_WPTR, 0);
4501		WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4502		WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4503	}
4504	WREG32(RLC_MC_CNTL, 0);
4505	WREG32(RLC_UCODE_CNTL, 0);
4506
4507	fw_data = (const __be32 *)rdev->rlc_fw->data;
4508	if (rdev->family >= CHIP_ARUBA) {
4509		for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4510			WREG32(RLC_UCODE_ADDR, i);
4511			WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4512		}
4513	} else if (rdev->family >= CHIP_CAYMAN) {
4514		for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4515			WREG32(RLC_UCODE_ADDR, i);
4516			WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4517		}
4518	} else {
4519		for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4520			WREG32(RLC_UCODE_ADDR, i);
4521			WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4522		}
4523	}
4524	WREG32(RLC_UCODE_ADDR, 0);
4525
4526	evergreen_rlc_start(rdev);
4527
4528	return 0;
4529}
4530
4531/* Interrupts */
4532
4533u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4534{
4535	if (crtc >= rdev->num_crtc)
4536		return 0;
4537	else
4538		return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4539}
4540
4541void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4542{
4543	u32 tmp;
4544
4545	if (rdev->family >= CHIP_CAYMAN) {
4546		cayman_cp_int_cntl_setup(rdev, 0,
4547					 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4548		cayman_cp_int_cntl_setup(rdev, 1, 0);
4549		cayman_cp_int_cntl_setup(rdev, 2, 0);
4550		tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4551		WREG32(CAYMAN_DMA1_CNTL, tmp);
4552	} else
4553		WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4554	tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4555	WREG32(DMA_CNTL, tmp);
4556	WREG32(GRBM_INT_CNTL, 0);
4557	WREG32(SRBM_INT_CNTL, 0);
4558	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4559	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4560	if (rdev->num_crtc >= 4) {
4561		WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4562		WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4563	}
4564	if (rdev->num_crtc >= 6) {
4565		WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4566		WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4567	}
4568
4569	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4570	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4571	if (rdev->num_crtc >= 4) {
4572		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4573		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4574	}
4575	if (rdev->num_crtc >= 6) {
4576		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4577		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4578	}
4579
4580	/* only one DAC on DCE5 */
4581	if (!ASIC_IS_DCE5(rdev))
4582		WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4583	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4584
4585	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4586	WREG32(DC_HPD1_INT_CONTROL, tmp);
4587	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4588	WREG32(DC_HPD2_INT_CONTROL, tmp);
4589	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4590	WREG32(DC_HPD3_INT_CONTROL, tmp);
4591	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4592	WREG32(DC_HPD4_INT_CONTROL, tmp);
4593	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4594	WREG32(DC_HPD5_INT_CONTROL, tmp);
4595	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4596	WREG32(DC_HPD6_INT_CONTROL, tmp);
4597
4598}
4599
4600int evergreen_irq_set(struct radeon_device *rdev)
4601{
4602	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4603	u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4604	u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4605	u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4606	u32 grbm_int_cntl = 0;
4607	u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4608	u32 dma_cntl, dma_cntl1 = 0;
4609	u32 thermal_int = 0;
4610
4611	if (!rdev->irq.installed) {
4612		WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4613		return -EINVAL;
4614	}
4615	/* don't enable anything if the ih is disabled */
4616	if (!rdev->ih.enabled) {
4617		r600_disable_interrupts(rdev);
4618		/* force the active interrupt state to all disabled */
4619		evergreen_disable_interrupt_state(rdev);
4620		return 0;
4621	}
4622
4623	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4624	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4625	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4626	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4627	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4628	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4629	if (rdev->family == CHIP_ARUBA)
4630		thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4631			~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4632	else
4633		thermal_int = RREG32(CG_THERMAL_INT) &
4634			~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4635
4636	afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4637	afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4638	afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4639	afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4640	afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4641	afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4642
4643	dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4644
4645	if (rdev->family >= CHIP_CAYMAN) {
4646		/* enable CP interrupts on all rings */
4647		if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4648			DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4649			cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4650		}
4651		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4652			DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4653			cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4654		}
4655		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4656			DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4657			cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4658		}
4659	} else {
4660		if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4661			DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4662			cp_int_cntl |= RB_INT_ENABLE;
4663			cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4664		}
4665	}
4666
4667	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4668		DRM_DEBUG("r600_irq_set: sw int dma\n");
4669		dma_cntl |= TRAP_ENABLE;
4670	}
4671
4672	if (rdev->family >= CHIP_CAYMAN) {
4673		dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4674		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4675			DRM_DEBUG("r600_irq_set: sw int dma1\n");
4676			dma_cntl1 |= TRAP_ENABLE;
4677		}
4678	}
4679
4680	if (rdev->irq.dpm_thermal) {
4681		DRM_DEBUG("dpm thermal\n");
4682		thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4683	}
4684
4685	if (rdev->irq.crtc_vblank_int[0] ||
4686	    atomic_read(&rdev->irq.pflip[0])) {
4687		DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4688		crtc1 |= VBLANK_INT_MASK;
4689	}
4690	if (rdev->irq.crtc_vblank_int[1] ||
4691	    atomic_read(&rdev->irq.pflip[1])) {
4692		DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4693		crtc2 |= VBLANK_INT_MASK;
4694	}
4695	if (rdev->irq.crtc_vblank_int[2] ||
4696	    atomic_read(&rdev->irq.pflip[2])) {
4697		DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4698		crtc3 |= VBLANK_INT_MASK;
4699	}
4700	if (rdev->irq.crtc_vblank_int[3] ||
4701	    atomic_read(&rdev->irq.pflip[3])) {
4702		DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4703		crtc4 |= VBLANK_INT_MASK;
4704	}
4705	if (rdev->irq.crtc_vblank_int[4] ||
4706	    atomic_read(&rdev->irq.pflip[4])) {
4707		DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4708		crtc5 |= VBLANK_INT_MASK;
4709	}
4710	if (rdev->irq.crtc_vblank_int[5] ||
4711	    atomic_read(&rdev->irq.pflip[5])) {
4712		DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4713		crtc6 |= VBLANK_INT_MASK;
4714	}
4715	if (rdev->irq.hpd[0]) {
4716		DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4717		hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4718	}
4719	if (rdev->irq.hpd[1]) {
4720		DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4721		hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4722	}
4723	if (rdev->irq.hpd[2]) {
4724		DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4725		hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4726	}
4727	if (rdev->irq.hpd[3]) {
4728		DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4729		hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4730	}
4731	if (rdev->irq.hpd[4]) {
4732		DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4733		hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4734	}
4735	if (rdev->irq.hpd[5]) {
4736		DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4737		hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4738	}
4739	if (rdev->irq.afmt[0]) {
4740		DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4741		afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4742	}
4743	if (rdev->irq.afmt[1]) {
4744		DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4745		afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4746	}
4747	if (rdev->irq.afmt[2]) {
4748		DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4749		afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4750	}
4751	if (rdev->irq.afmt[3]) {
4752		DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4753		afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4754	}
4755	if (rdev->irq.afmt[4]) {
4756		DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4757		afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4758	}
4759	if (rdev->irq.afmt[5]) {
4760		DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4761		afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4762	}
4763
4764	if (rdev->family >= CHIP_CAYMAN) {
4765		cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4766		cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4767		cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4768	} else
4769		WREG32(CP_INT_CNTL, cp_int_cntl);
4770
4771	WREG32(DMA_CNTL, dma_cntl);
4772
4773	if (rdev->family >= CHIP_CAYMAN)
4774		WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4775
4776	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4777
4778	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4779	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4780	if (rdev->num_crtc >= 4) {
4781		WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4782		WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4783	}
4784	if (rdev->num_crtc >= 6) {
4785		WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4786		WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4787	}
4788
4789	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4790	       GRPH_PFLIP_INT_MASK);
4791	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4792	       GRPH_PFLIP_INT_MASK);
4793	if (rdev->num_crtc >= 4) {
4794		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4795		       GRPH_PFLIP_INT_MASK);
4796		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4797		       GRPH_PFLIP_INT_MASK);
4798	}
4799	if (rdev->num_crtc >= 6) {
4800		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4801		       GRPH_PFLIP_INT_MASK);
4802		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4803		       GRPH_PFLIP_INT_MASK);
4804	}
4805
4806	WREG32(DC_HPD1_INT_CONTROL, hpd1);
4807	WREG32(DC_HPD2_INT_CONTROL, hpd2);
4808	WREG32(DC_HPD3_INT_CONTROL, hpd3);
4809	WREG32(DC_HPD4_INT_CONTROL, hpd4);
4810	WREG32(DC_HPD5_INT_CONTROL, hpd5);
4811	WREG32(DC_HPD6_INT_CONTROL, hpd6);
4812	if (rdev->family == CHIP_ARUBA)
4813		WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4814	else
4815		WREG32(CG_THERMAL_INT, thermal_int);
4816
4817	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4818	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4819	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4820	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4821	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4822	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4823
4824	/* posting read */
4825	RREG32(SRBM_STATUS);
4826
4827	return 0;
4828}
4829
4830static void evergreen_irq_ack(struct radeon_device *rdev)
4831{
4832	u32 tmp;
4833
4834	rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4835	rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4836	rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4837	rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4838	rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4839	rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4840	rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4841	rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4842	if (rdev->num_crtc >= 4) {
4843		rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4844		rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4845	}
4846	if (rdev->num_crtc >= 6) {
4847		rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4848		rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4849	}
4850
4851	rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4852	rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4853	rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4854	rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4855	rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4856	rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4857
4858	if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4859		WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4860	if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4861		WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4862	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4863		WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4864	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4865		WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4866	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4867		WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4868	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4869		WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4870
4871	if (rdev->num_crtc >= 4) {
4872		if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4873			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4874		if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4875			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4876		if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4877			WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4878		if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4879			WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4880		if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4881			WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4882		if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4883			WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4884	}
4885
4886	if (rdev->num_crtc >= 6) {
4887		if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4888			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4889		if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4890			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4891		if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4892			WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4893		if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4894			WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4895		if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4896			WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4897		if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4898			WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4899	}
4900
4901	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4902		tmp = RREG32(DC_HPD1_INT_CONTROL);
4903		tmp |= DC_HPDx_INT_ACK;
4904		WREG32(DC_HPD1_INT_CONTROL, tmp);
4905	}
4906	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4907		tmp = RREG32(DC_HPD2_INT_CONTROL);
4908		tmp |= DC_HPDx_INT_ACK;
4909		WREG32(DC_HPD2_INT_CONTROL, tmp);
4910	}
4911	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4912		tmp = RREG32(DC_HPD3_INT_CONTROL);
4913		tmp |= DC_HPDx_INT_ACK;
4914		WREG32(DC_HPD3_INT_CONTROL, tmp);
4915	}
4916	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4917		tmp = RREG32(DC_HPD4_INT_CONTROL);
4918		tmp |= DC_HPDx_INT_ACK;
4919		WREG32(DC_HPD4_INT_CONTROL, tmp);
4920	}
4921	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4922		tmp = RREG32(DC_HPD5_INT_CONTROL);
4923		tmp |= DC_HPDx_INT_ACK;
4924		WREG32(DC_HPD5_INT_CONTROL, tmp);
4925	}
4926	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4927		tmp = RREG32(DC_HPD5_INT_CONTROL);
4928		tmp |= DC_HPDx_INT_ACK;
4929		WREG32(DC_HPD6_INT_CONTROL, tmp);
4930	}
4931
4932	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
4933		tmp = RREG32(DC_HPD1_INT_CONTROL);
4934		tmp |= DC_HPDx_RX_INT_ACK;
4935		WREG32(DC_HPD1_INT_CONTROL, tmp);
4936	}
4937	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
4938		tmp = RREG32(DC_HPD2_INT_CONTROL);
4939		tmp |= DC_HPDx_RX_INT_ACK;
4940		WREG32(DC_HPD2_INT_CONTROL, tmp);
4941	}
4942	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
4943		tmp = RREG32(DC_HPD3_INT_CONTROL);
4944		tmp |= DC_HPDx_RX_INT_ACK;
4945		WREG32(DC_HPD3_INT_CONTROL, tmp);
4946	}
4947	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
4948		tmp = RREG32(DC_HPD4_INT_CONTROL);
4949		tmp |= DC_HPDx_RX_INT_ACK;
4950		WREG32(DC_HPD4_INT_CONTROL, tmp);
4951	}
4952	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
4953		tmp = RREG32(DC_HPD5_INT_CONTROL);
4954		tmp |= DC_HPDx_RX_INT_ACK;
4955		WREG32(DC_HPD5_INT_CONTROL, tmp);
4956	}
4957	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
4958		tmp = RREG32(DC_HPD5_INT_CONTROL);
4959		tmp |= DC_HPDx_RX_INT_ACK;
4960		WREG32(DC_HPD6_INT_CONTROL, tmp);
4961	}
4962
4963	if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4964		tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4965		tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4966		WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4967	}
4968	if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4969		tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4970		tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4971		WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4972	}
4973	if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4974		tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4975		tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4976		WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4977	}
4978	if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4979		tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4980		tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4981		WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4982	}
4983	if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4984		tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4985		tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4986		WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4987	}
4988	if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4989		tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4990		tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4991		WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4992	}
4993}
4994
4995static void evergreen_irq_disable(struct radeon_device *rdev)
4996{
4997	r600_disable_interrupts(rdev);
4998	/* Wait and acknowledge irq */
4999	mdelay(1);
5000	evergreen_irq_ack(rdev);
5001	evergreen_disable_interrupt_state(rdev);
5002}
5003
5004void evergreen_irq_suspend(struct radeon_device *rdev)
5005{
5006	evergreen_irq_disable(rdev);
5007	r600_rlc_stop(rdev);
5008}
5009
5010static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
5011{
5012	u32 wptr, tmp;
5013
5014	if (rdev->wb.enabled)
5015		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
5016	else
5017		wptr = RREG32(IH_RB_WPTR);
5018
5019	if (wptr & RB_OVERFLOW) {
5020		wptr &= ~RB_OVERFLOW;
5021		/* When a ring buffer overflow happen start parsing interrupt
5022		 * from the last not overwritten vector (wptr + 16). Hopefully
5023		 * this should allow us to catchup.
5024		 */
5025		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
5026			 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
5027		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
5028		tmp = RREG32(IH_RB_CNTL);
5029		tmp |= IH_WPTR_OVERFLOW_CLEAR;
5030		WREG32(IH_RB_CNTL, tmp);
5031	}
5032	return (wptr & rdev->ih.ptr_mask);
5033}
5034
5035int evergreen_irq_process(struct radeon_device *rdev)
5036{
5037	u32 wptr;
5038	u32 rptr;
5039	u32 src_id, src_data;
5040	u32 ring_index;
5041	bool queue_hotplug = false;
5042	bool queue_hdmi = false;
5043	bool queue_dp = false;
5044	bool queue_thermal = false;
5045	u32 status, addr;
5046
5047	if (!rdev->ih.enabled || rdev->shutdown)
5048		return IRQ_NONE;
5049
5050	wptr = evergreen_get_ih_wptr(rdev);
5051
5052restart_ih:
5053	/* is somebody else already processing irqs? */
5054	if (atomic_xchg(&rdev->ih.lock, 1))
5055		return IRQ_NONE;
5056
5057	rptr = rdev->ih.rptr;
5058	DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
5059
5060	/* Order reading of wptr vs. reading of IH ring data */
5061	rmb();
5062
5063	/* display interrupts */
5064	evergreen_irq_ack(rdev);
5065
5066	while (rptr != wptr) {
5067		/* wptr/rptr are in bytes! */
5068		ring_index = rptr / 4;
5069		src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
5070		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
5071
5072		switch (src_id) {
5073		case 1: /* D1 vblank/vline */
5074			switch (src_data) {
5075			case 0: /* D1 vblank */
5076				if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
5077					DRM_DEBUG("IH: D1 vblank - IH event w/o asserted irq bit?\n");
5078
5079				if (rdev->irq.crtc_vblank_int[0]) {
5080					drm_handle_vblank(rdev->ddev, 0);
5081					rdev->pm.vblank_sync = true;
5082					wake_up(&rdev->irq.vblank_queue);
 
 
 
5083				}
5084				if (atomic_read(&rdev->irq.pflip[0]))
5085					radeon_crtc_handle_vblank(rdev, 0);
5086				rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
5087				DRM_DEBUG("IH: D1 vblank\n");
5088
5089				break;
5090			case 1: /* D1 vline */
5091				if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
5092					DRM_DEBUG("IH: D1 vline - IH event w/o asserted irq bit?\n");
5093
5094				rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
5095				DRM_DEBUG("IH: D1 vline\n");
5096
5097				break;
5098			default:
5099				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5100				break;
5101			}
5102			break;
5103		case 2: /* D2 vblank/vline */
5104			switch (src_data) {
5105			case 0: /* D2 vblank */
5106				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
5107					DRM_DEBUG("IH: D2 vblank - IH event w/o asserted irq bit?\n");
5108
5109				if (rdev->irq.crtc_vblank_int[1]) {
5110					drm_handle_vblank(rdev->ddev, 1);
5111					rdev->pm.vblank_sync = true;
5112					wake_up(&rdev->irq.vblank_queue);
 
 
 
5113				}
5114				if (atomic_read(&rdev->irq.pflip[1]))
5115					radeon_crtc_handle_vblank(rdev, 1);
5116				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
5117				DRM_DEBUG("IH: D2 vblank\n");
5118
5119				break;
5120			case 1: /* D2 vline */
5121				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
5122					DRM_DEBUG("IH: D2 vline - IH event w/o asserted irq bit?\n");
5123
5124				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
5125				DRM_DEBUG("IH: D2 vline\n");
5126
5127				break;
5128			default:
5129				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5130				break;
5131			}
5132			break;
5133		case 3: /* D3 vblank/vline */
5134			switch (src_data) {
5135			case 0: /* D3 vblank */
5136				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
5137					DRM_DEBUG("IH: D3 vblank - IH event w/o asserted irq bit?\n");
5138
5139				if (rdev->irq.crtc_vblank_int[2]) {
5140					drm_handle_vblank(rdev->ddev, 2);
5141					rdev->pm.vblank_sync = true;
5142					wake_up(&rdev->irq.vblank_queue);
 
 
 
5143				}
5144				if (atomic_read(&rdev->irq.pflip[2]))
5145					radeon_crtc_handle_vblank(rdev, 2);
5146				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
5147				DRM_DEBUG("IH: D3 vblank\n");
5148
5149				break;
5150			case 1: /* D3 vline */
5151				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
5152					DRM_DEBUG("IH: D3 vline - IH event w/o asserted irq bit?\n");
5153
5154				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5155				DRM_DEBUG("IH: D3 vline\n");
5156
5157				break;
5158			default:
5159				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5160				break;
5161			}
5162			break;
5163		case 4: /* D4 vblank/vline */
5164			switch (src_data) {
5165			case 0: /* D4 vblank */
5166				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
5167					DRM_DEBUG("IH: D4 vblank - IH event w/o asserted irq bit?\n");
5168
5169				if (rdev->irq.crtc_vblank_int[3]) {
5170					drm_handle_vblank(rdev->ddev, 3);
5171					rdev->pm.vblank_sync = true;
5172					wake_up(&rdev->irq.vblank_queue);
 
 
 
5173				}
5174				if (atomic_read(&rdev->irq.pflip[3]))
5175					radeon_crtc_handle_vblank(rdev, 3);
5176				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5177				DRM_DEBUG("IH: D4 vblank\n");
5178
5179				break;
5180			case 1: /* D4 vline */
5181				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
5182					DRM_DEBUG("IH: D4 vline - IH event w/o asserted irq bit?\n");
5183
5184				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5185				DRM_DEBUG("IH: D4 vline\n");
5186
5187				break;
5188			default:
5189				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5190				break;
5191			}
5192			break;
5193		case 5: /* D5 vblank/vline */
5194			switch (src_data) {
5195			case 0: /* D5 vblank */
5196				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
5197					DRM_DEBUG("IH: D5 vblank - IH event w/o asserted irq bit?\n");
5198
5199				if (rdev->irq.crtc_vblank_int[4]) {
5200					drm_handle_vblank(rdev->ddev, 4);
5201					rdev->pm.vblank_sync = true;
5202					wake_up(&rdev->irq.vblank_queue);
 
 
 
5203				}
5204				if (atomic_read(&rdev->irq.pflip[4]))
5205					radeon_crtc_handle_vblank(rdev, 4);
5206				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5207				DRM_DEBUG("IH: D5 vblank\n");
5208
5209				break;
5210			case 1: /* D5 vline */
5211				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
5212					DRM_DEBUG("IH: D5 vline - IH event w/o asserted irq bit?\n");
5213
5214				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5215				DRM_DEBUG("IH: D5 vline\n");
5216
5217				break;
5218			default:
5219				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5220				break;
5221			}
5222			break;
5223		case 6: /* D6 vblank/vline */
5224			switch (src_data) {
5225			case 0: /* D6 vblank */
5226				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
5227					DRM_DEBUG("IH: D6 vblank - IH event w/o asserted irq bit?\n");
5228
5229				if (rdev->irq.crtc_vblank_int[5]) {
5230					drm_handle_vblank(rdev->ddev, 5);
5231					rdev->pm.vblank_sync = true;
5232					wake_up(&rdev->irq.vblank_queue);
 
 
 
5233				}
5234				if (atomic_read(&rdev->irq.pflip[5]))
5235					radeon_crtc_handle_vblank(rdev, 5);
5236				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5237				DRM_DEBUG("IH: D6 vblank\n");
5238
5239				break;
5240			case 1: /* D6 vline */
5241				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
5242					DRM_DEBUG("IH: D6 vline - IH event w/o asserted irq bit?\n");
5243
5244				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5245				DRM_DEBUG("IH: D6 vline\n");
5246
5247				break;
5248			default:
5249				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5250				break;
5251			}
5252			break;
5253		case 8: /* D1 page flip */
5254		case 10: /* D2 page flip */
5255		case 12: /* D3 page flip */
5256		case 14: /* D4 page flip */
5257		case 16: /* D5 page flip */
5258		case 18: /* D6 page flip */
5259			DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
5260			if (radeon_use_pflipirq > 0)
5261				radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
5262			break;
5263		case 42: /* HPD hotplug */
5264			switch (src_data) {
5265			case 0:
5266				if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
5267					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5268
5269				rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5270				queue_hotplug = true;
5271				DRM_DEBUG("IH: HPD1\n");
5272				break;
5273			case 1:
5274				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
5275					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5276
5277				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5278				queue_hotplug = true;
5279				DRM_DEBUG("IH: HPD2\n");
5280				break;
5281			case 2:
5282				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
5283					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5284
5285				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5286				queue_hotplug = true;
5287				DRM_DEBUG("IH: HPD3\n");
5288				break;
5289			case 3:
5290				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
5291					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5292
5293				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5294				queue_hotplug = true;
5295				DRM_DEBUG("IH: HPD4\n");
5296				break;
5297			case 4:
5298				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
5299					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5300
5301				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5302				queue_hotplug = true;
5303				DRM_DEBUG("IH: HPD5\n");
5304				break;
5305			case 5:
5306				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
5307					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5308
5309				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5310				queue_hotplug = true;
5311				DRM_DEBUG("IH: HPD6\n");
5312				break;
5313			case 6:
5314				if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
5315					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5316
5317				rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
5318				queue_dp = true;
5319				DRM_DEBUG("IH: HPD_RX 1\n");
5320				break;
5321			case 7:
5322				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
5323					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5324
5325				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
5326				queue_dp = true;
5327				DRM_DEBUG("IH: HPD_RX 2\n");
5328				break;
5329			case 8:
5330				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
5331					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5332
5333				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
5334				queue_dp = true;
5335				DRM_DEBUG("IH: HPD_RX 3\n");
5336				break;
5337			case 9:
5338				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
5339					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5340
5341				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
5342				queue_dp = true;
5343				DRM_DEBUG("IH: HPD_RX 4\n");
5344				break;
5345			case 10:
5346				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
5347					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5348
5349				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
5350				queue_dp = true;
5351				DRM_DEBUG("IH: HPD_RX 5\n");
5352				break;
5353			case 11:
5354				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
5355					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5356
5357				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
5358				queue_dp = true;
5359				DRM_DEBUG("IH: HPD_RX 6\n");
5360				break;
5361			default:
5362				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5363				break;
5364			}
5365			break;
5366		case 44: /* hdmi */
5367			switch (src_data) {
5368			case 0:
5369				if (!(rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG))
5370					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5371
5372				rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5373				queue_hdmi = true;
5374				DRM_DEBUG("IH: HDMI0\n");
5375				break;
5376			case 1:
5377				if (!(rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG))
5378					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5379
5380				rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5381				queue_hdmi = true;
5382				DRM_DEBUG("IH: HDMI1\n");
5383				break;
5384			case 2:
5385				if (!(rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG))
5386					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5387
5388				rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5389				queue_hdmi = true;
5390				DRM_DEBUG("IH: HDMI2\n");
5391				break;
5392			case 3:
5393				if (!(rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG))
5394					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5395
5396				rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5397				queue_hdmi = true;
5398				DRM_DEBUG("IH: HDMI3\n");
5399				break;
5400			case 4:
5401				if (!(rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG))
5402					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5403
5404				rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5405				queue_hdmi = true;
5406				DRM_DEBUG("IH: HDMI4\n");
5407				break;
5408			case 5:
5409				if (!(rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG))
5410					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5411
5412				rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5413				queue_hdmi = true;
5414				DRM_DEBUG("IH: HDMI5\n");
5415				break;
5416			default:
5417				DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5418				break;
5419			}
5420		case 96:
5421			DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
5422			WREG32(SRBM_INT_ACK, 0x1);
5423			break;
5424		case 124: /* UVD */
5425			DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5426			radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
5427			break;
5428		case 146:
5429		case 147:
5430			addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5431			status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
5432			/* reset addr and status */
5433			WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5434			if (addr == 0x0 && status == 0x0)
5435				break;
5436			dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5437			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
5438				addr);
5439			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5440				status);
5441			cayman_vm_decode_fault(rdev, status, addr);
 
 
5442			break;
5443		case 176: /* CP_INT in ring buffer */
5444		case 177: /* CP_INT in IB1 */
5445		case 178: /* CP_INT in IB2 */
5446			DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
5447			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5448			break;
5449		case 181: /* CP EOP event */
5450			DRM_DEBUG("IH: CP EOP\n");
5451			if (rdev->family >= CHIP_CAYMAN) {
5452				switch (src_data) {
5453				case 0:
5454					radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5455					break;
5456				case 1:
5457					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5458					break;
5459				case 2:
5460					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5461					break;
5462				}
5463			} else
5464				radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5465			break;
5466		case 224: /* DMA trap event */
5467			DRM_DEBUG("IH: DMA trap\n");
5468			radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5469			break;
5470		case 230: /* thermal low to high */
5471			DRM_DEBUG("IH: thermal low to high\n");
5472			rdev->pm.dpm.thermal.high_to_low = false;
5473			queue_thermal = true;
5474			break;
5475		case 231: /* thermal high to low */
5476			DRM_DEBUG("IH: thermal high to low\n");
5477			rdev->pm.dpm.thermal.high_to_low = true;
5478			queue_thermal = true;
5479			break;
5480		case 233: /* GUI IDLE */
5481			DRM_DEBUG("IH: GUI idle\n");
5482			break;
5483		case 244: /* DMA trap event */
5484			if (rdev->family >= CHIP_CAYMAN) {
5485				DRM_DEBUG("IH: DMA1 trap\n");
5486				radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5487			}
5488			break;
5489		default:
5490			DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5491			break;
5492		}
5493
5494		/* wptr/rptr are in bytes! */
5495		rptr += 16;
5496		rptr &= rdev->ih.ptr_mask;
5497		WREG32(IH_RB_RPTR, rptr);
5498	}
5499	if (queue_dp)
5500		schedule_work(&rdev->dp_work);
5501	if (queue_hotplug)
5502		schedule_delayed_work(&rdev->hotplug_work, 0);
5503	if (queue_hdmi)
5504		schedule_work(&rdev->audio_work);
5505	if (queue_thermal && rdev->pm.dpm_enabled)
5506		schedule_work(&rdev->pm.dpm.thermal.work);
5507	rdev->ih.rptr = rptr;
 
5508	atomic_set(&rdev->ih.lock, 0);
5509
5510	/* make sure wptr hasn't changed while processing */
5511	wptr = evergreen_get_ih_wptr(rdev);
5512	if (wptr != rptr)
5513		goto restart_ih;
5514
5515	return IRQ_HANDLED;
5516}
5517
5518static int evergreen_startup(struct radeon_device *rdev)
5519{
5520	struct radeon_ring *ring;
5521	int r;
5522
5523	/* enable pcie gen2 link */
5524	evergreen_pcie_gen2_enable(rdev);
5525	/* enable aspm */
5526	evergreen_program_aspm(rdev);
5527
5528	/* scratch needs to be initialized before MC */
5529	r = r600_vram_scratch_init(rdev);
5530	if (r)
5531		return r;
5532
5533	evergreen_mc_program(rdev);
5534
5535	if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5536		r = ni_mc_load_microcode(rdev);
5537		if (r) {
5538			DRM_ERROR("Failed to load MC firmware!\n");
5539			return r;
5540		}
5541	}
5542
5543	if (rdev->flags & RADEON_IS_AGP) {
5544		evergreen_agp_enable(rdev);
5545	} else {
5546		r = evergreen_pcie_gart_enable(rdev);
5547		if (r)
5548			return r;
5549	}
5550	evergreen_gpu_init(rdev);
5551
5552	/* allocate rlc buffers */
5553	if (rdev->flags & RADEON_IS_IGP) {
5554		rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5555		rdev->rlc.reg_list_size =
5556			(u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5557		rdev->rlc.cs_data = evergreen_cs_data;
5558		r = sumo_rlc_init(rdev);
5559		if (r) {
5560			DRM_ERROR("Failed to init rlc BOs!\n");
5561			return r;
5562		}
5563	}
5564
5565	/* allocate wb buffer */
5566	r = radeon_wb_init(rdev);
5567	if (r)
5568		return r;
5569
5570	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5571	if (r) {
5572		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5573		return r;
5574	}
5575
5576	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5577	if (r) {
5578		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5579		return r;
5580	}
5581
5582	r = uvd_v2_2_resume(rdev);
5583	if (!r) {
5584		r = radeon_fence_driver_start_ring(rdev,
5585						   R600_RING_TYPE_UVD_INDEX);
5586		if (r)
5587			dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5588	}
5589
5590	if (r)
5591		rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5592
5593	/* Enable IRQ */
5594	if (!rdev->irq.installed) {
5595		r = radeon_irq_kms_init(rdev);
5596		if (r)
5597			return r;
5598	}
5599
5600	r = r600_irq_init(rdev);
5601	if (r) {
5602		DRM_ERROR("radeon: IH init failed (%d).\n", r);
5603		radeon_irq_kms_fini(rdev);
5604		return r;
5605	}
5606	evergreen_irq_set(rdev);
5607
5608	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5609	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5610			     RADEON_CP_PACKET2);
5611	if (r)
5612		return r;
5613
5614	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5615	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5616			     DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5617	if (r)
5618		return r;
5619
5620	r = evergreen_cp_load_microcode(rdev);
5621	if (r)
5622		return r;
5623	r = evergreen_cp_resume(rdev);
5624	if (r)
5625		return r;
5626	r = r600_dma_resume(rdev);
5627	if (r)
5628		return r;
5629
5630	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5631	if (ring->ring_size) {
5632		r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5633				     RADEON_CP_PACKET2);
5634		if (!r)
5635			r = uvd_v1_0_init(rdev);
5636
5637		if (r)
5638			DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5639	}
5640
5641	r = radeon_ib_pool_init(rdev);
5642	if (r) {
5643		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5644		return r;
5645	}
5646
5647	r = radeon_audio_init(rdev);
5648	if (r) {
5649		DRM_ERROR("radeon: audio init failed\n");
5650		return r;
5651	}
5652
5653	return 0;
5654}
5655
5656int evergreen_resume(struct radeon_device *rdev)
5657{
5658	int r;
5659
5660	/* reset the asic, the gfx blocks are often in a bad state
5661	 * after the driver is unloaded or after a resume
5662	 */
5663	if (radeon_asic_reset(rdev))
5664		dev_warn(rdev->dev, "GPU reset failed !\n");
5665	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5666	 * posting will perform necessary task to bring back GPU into good
5667	 * shape.
5668	 */
5669	/* post card */
5670	atom_asic_init(rdev->mode_info.atom_context);
5671
5672	/* init golden registers */
5673	evergreen_init_golden_registers(rdev);
5674
5675	if (rdev->pm.pm_method == PM_METHOD_DPM)
5676		radeon_pm_resume(rdev);
5677
5678	rdev->accel_working = true;
5679	r = evergreen_startup(rdev);
5680	if (r) {
5681		DRM_ERROR("evergreen startup failed on resume\n");
5682		rdev->accel_working = false;
5683		return r;
5684	}
5685
5686	return r;
5687
5688}
5689
5690int evergreen_suspend(struct radeon_device *rdev)
5691{
5692	radeon_pm_suspend(rdev);
5693	radeon_audio_fini(rdev);
5694	uvd_v1_0_fini(rdev);
5695	radeon_uvd_suspend(rdev);
5696	r700_cp_stop(rdev);
5697	r600_dma_stop(rdev);
5698	evergreen_irq_suspend(rdev);
5699	radeon_wb_disable(rdev);
5700	evergreen_pcie_gart_disable(rdev);
5701
5702	return 0;
5703}
5704
5705/* Plan is to move initialization in that function and use
5706 * helper function so that radeon_device_init pretty much
5707 * do nothing more than calling asic specific function. This
5708 * should also allow to remove a bunch of callback function
5709 * like vram_info.
5710 */
5711int evergreen_init(struct radeon_device *rdev)
5712{
5713	int r;
5714
5715	/* Read BIOS */
5716	if (!radeon_get_bios(rdev)) {
5717		if (ASIC_IS_AVIVO(rdev))
5718			return -EINVAL;
5719	}
5720	/* Must be an ATOMBIOS */
5721	if (!rdev->is_atom_bios) {
5722		dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5723		return -EINVAL;
5724	}
5725	r = radeon_atombios_init(rdev);
5726	if (r)
5727		return r;
5728	/* reset the asic, the gfx blocks are often in a bad state
5729	 * after the driver is unloaded or after a resume
5730	 */
5731	if (radeon_asic_reset(rdev))
5732		dev_warn(rdev->dev, "GPU reset failed !\n");
5733	/* Post card if necessary */
5734	if (!radeon_card_posted(rdev)) {
5735		if (!rdev->bios) {
5736			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5737			return -EINVAL;
5738		}
5739		DRM_INFO("GPU not posted. posting now...\n");
5740		atom_asic_init(rdev->mode_info.atom_context);
5741	}
5742	/* init golden registers */
5743	evergreen_init_golden_registers(rdev);
5744	/* Initialize scratch registers */
5745	r600_scratch_init(rdev);
5746	/* Initialize surface registers */
5747	radeon_surface_init(rdev);
5748	/* Initialize clocks */
5749	radeon_get_clock_info(rdev->ddev);
5750	/* Fence driver */
5751	r = radeon_fence_driver_init(rdev);
5752	if (r)
5753		return r;
5754	/* initialize AGP */
5755	if (rdev->flags & RADEON_IS_AGP) {
5756		r = radeon_agp_init(rdev);
5757		if (r)
5758			radeon_agp_disable(rdev);
5759	}
5760	/* initialize memory controller */
5761	r = evergreen_mc_init(rdev);
5762	if (r)
5763		return r;
5764	/* Memory manager */
5765	r = radeon_bo_init(rdev);
5766	if (r)
5767		return r;
5768
5769	if (ASIC_IS_DCE5(rdev)) {
5770		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5771			r = ni_init_microcode(rdev);
5772			if (r) {
5773				DRM_ERROR("Failed to load firmware!\n");
5774				return r;
5775			}
5776		}
5777	} else {
5778		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5779			r = r600_init_microcode(rdev);
5780			if (r) {
5781				DRM_ERROR("Failed to load firmware!\n");
5782				return r;
5783			}
5784		}
5785	}
5786
5787	/* Initialize power management */
5788	radeon_pm_init(rdev);
5789
5790	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5791	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5792
5793	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5794	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5795
5796	r = radeon_uvd_init(rdev);
5797	if (!r) {
5798		rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5799		r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5800			       4096);
5801	}
5802
5803	rdev->ih.ring_obj = NULL;
5804	r600_ih_ring_init(rdev, 64 * 1024);
5805
5806	r = r600_pcie_gart_init(rdev);
5807	if (r)
5808		return r;
5809
5810	rdev->accel_working = true;
5811	r = evergreen_startup(rdev);
5812	if (r) {
5813		dev_err(rdev->dev, "disabling GPU acceleration\n");
5814		r700_cp_fini(rdev);
5815		r600_dma_fini(rdev);
5816		r600_irq_fini(rdev);
5817		if (rdev->flags & RADEON_IS_IGP)
5818			sumo_rlc_fini(rdev);
5819		radeon_wb_fini(rdev);
5820		radeon_ib_pool_fini(rdev);
5821		radeon_irq_kms_fini(rdev);
5822		evergreen_pcie_gart_fini(rdev);
5823		rdev->accel_working = false;
5824	}
5825
5826	/* Don't start up if the MC ucode is missing on BTC parts.
5827	 * The default clocks and voltages before the MC ucode
5828	 * is loaded are not suffient for advanced operations.
5829	 */
5830	if (ASIC_IS_DCE5(rdev)) {
5831		if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5832			DRM_ERROR("radeon: MC ucode required for NI+.\n");
5833			return -EINVAL;
5834		}
5835	}
5836
5837	return 0;
5838}
5839
5840void evergreen_fini(struct radeon_device *rdev)
5841{
5842	radeon_pm_fini(rdev);
5843	radeon_audio_fini(rdev);
5844	r700_cp_fini(rdev);
5845	r600_dma_fini(rdev);
5846	r600_irq_fini(rdev);
5847	if (rdev->flags & RADEON_IS_IGP)
5848		sumo_rlc_fini(rdev);
5849	radeon_wb_fini(rdev);
5850	radeon_ib_pool_fini(rdev);
5851	radeon_irq_kms_fini(rdev);
5852	uvd_v1_0_fini(rdev);
5853	radeon_uvd_fini(rdev);
5854	evergreen_pcie_gart_fini(rdev);
5855	r600_vram_scratch_fini(rdev);
5856	radeon_gem_fini(rdev);
5857	radeon_fence_driver_fini(rdev);
5858	radeon_agp_fini(rdev);
5859	radeon_bo_fini(rdev);
5860	radeon_atombios_fini(rdev);
5861	kfree(rdev->bios);
5862	rdev->bios = NULL;
5863}
5864
5865void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5866{
5867	u32 link_width_cntl, speed_cntl;
5868
5869	if (radeon_pcie_gen2 == 0)
5870		return;
5871
5872	if (rdev->flags & RADEON_IS_IGP)
5873		return;
5874
5875	if (!(rdev->flags & RADEON_IS_PCIE))
5876		return;
5877
5878	/* x2 cards have a special sequence */
5879	if (ASIC_IS_X2(rdev))
5880		return;
5881
5882	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5883		(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5884		return;
5885
5886	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5887	if (speed_cntl & LC_CURRENT_DATA_RATE) {
5888		DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5889		return;
5890	}
5891
5892	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5893
5894	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5895	    (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5896
5897		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5898		link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5899		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5900
5901		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5902		speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5903		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5904
5905		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5906		speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5907		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5908
5909		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5910		speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5911		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5912
5913		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5914		speed_cntl |= LC_GEN2_EN_STRAP;
5915		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5916
5917	} else {
5918		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5919		/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5920		if (1)
5921			link_width_cntl |= LC_UPCONFIGURE_DIS;
5922		else
5923			link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5924		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5925	}
5926}
5927
5928void evergreen_program_aspm(struct radeon_device *rdev)
5929{
5930	u32 data, orig;
5931	u32 pcie_lc_cntl, pcie_lc_cntl_old;
5932	bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5933	/* fusion_platform = true
5934	 * if the system is a fusion system
5935	 * (APU or DGPU in a fusion system).
5936	 * todo: check if the system is a fusion platform.
5937	 */
5938	bool fusion_platform = false;
5939
5940	if (radeon_aspm == 0)
5941		return;
5942
5943	if (!(rdev->flags & RADEON_IS_PCIE))
5944		return;
5945
5946	switch (rdev->family) {
5947	case CHIP_CYPRESS:
5948	case CHIP_HEMLOCK:
5949	case CHIP_JUNIPER:
5950	case CHIP_REDWOOD:
5951	case CHIP_CEDAR:
5952	case CHIP_SUMO:
5953	case CHIP_SUMO2:
5954	case CHIP_PALM:
5955	case CHIP_ARUBA:
5956		disable_l0s = true;
5957		break;
5958	default:
5959		disable_l0s = false;
5960		break;
5961	}
5962
5963	if (rdev->flags & RADEON_IS_IGP)
5964		fusion_platform = true; /* XXX also dGPUs in a fusion system */
5965
5966	data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5967	if (fusion_platform)
5968		data &= ~MULTI_PIF;
5969	else
5970		data |= MULTI_PIF;
5971	if (data != orig)
5972		WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5973
5974	data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5975	if (fusion_platform)
5976		data &= ~MULTI_PIF;
5977	else
5978		data |= MULTI_PIF;
5979	if (data != orig)
5980		WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5981
5982	pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5983	pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5984	if (!disable_l0s) {
5985		if (rdev->family >= CHIP_BARTS)
5986			pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5987		else
5988			pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5989	}
5990
5991	if (!disable_l1) {
5992		if (rdev->family >= CHIP_BARTS)
5993			pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5994		else
5995			pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5996
5997		if (!disable_plloff_in_l1) {
5998			data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5999			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6000			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6001			if (data != orig)
6002				WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6003
6004			data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6005			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6006			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6007			if (data != orig)
6008				WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6009
6010			data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6011			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6012			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6013			if (data != orig)
6014				WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6015
6016			data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6017			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6018			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6019			if (data != orig)
6020				WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6021
6022			if (rdev->family >= CHIP_BARTS) {
6023				data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6024				data &= ~PLL_RAMP_UP_TIME_0_MASK;
6025				data |= PLL_RAMP_UP_TIME_0(4);
6026				if (data != orig)
6027					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6028
6029				data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6030				data &= ~PLL_RAMP_UP_TIME_1_MASK;
6031				data |= PLL_RAMP_UP_TIME_1(4);
6032				if (data != orig)
6033					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6034
6035				data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6036				data &= ~PLL_RAMP_UP_TIME_0_MASK;
6037				data |= PLL_RAMP_UP_TIME_0(4);
6038				if (data != orig)
6039					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6040
6041				data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6042				data &= ~PLL_RAMP_UP_TIME_1_MASK;
6043				data |= PLL_RAMP_UP_TIME_1(4);
6044				if (data != orig)
6045					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6046			}
6047
6048			data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6049			data &= ~LC_DYN_LANES_PWR_STATE_MASK;
6050			data |= LC_DYN_LANES_PWR_STATE(3);
6051			if (data != orig)
6052				WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
6053
6054			if (rdev->family >= CHIP_BARTS) {
6055				data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
6056				data &= ~LS2_EXIT_TIME_MASK;
6057				data |= LS2_EXIT_TIME(1);
6058				if (data != orig)
6059					WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
6060
6061				data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
6062				data &= ~LS2_EXIT_TIME_MASK;
6063				data |= LS2_EXIT_TIME(1);
6064				if (data != orig)
6065					WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
6066			}
6067		}
6068	}
6069
6070	/* evergreen parts only */
6071	if (rdev->family < CHIP_BARTS)
6072		pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
6073
6074	if (pcie_lc_cntl != pcie_lc_cntl_old)
6075		WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
6076}