1/*
   2 * Copyright 2010 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24
  25#include <linux/firmware.h>
  26#include <linux/slab.h>
  27
  28#include <drm/drm_pci.h>
  29#include <drm/drm_vblank.h>
 
  30#include <drm/radeon_drm.h>
  31
  32#include "atom.h"
  33#include "avivod.h"
  34#include "evergreen_blit_shaders.h"
  35#include "evergreen_reg.h"
  36#include "evergreend.h"
  37#include "radeon.h"
  38#include "radeon_asic.h"
  39#include "radeon_audio.h"
  40#include "radeon_ucode.h"
  41
  42#define DC_HPDx_CONTROL(x)        (DC_HPD1_CONTROL     + (x * 0xc))
  43#define DC_HPDx_INT_CONTROL(x)    (DC_HPD1_INT_CONTROL + (x * 0xc))
  44#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS  + (x * 0xc))
  45
  46/*
  47 * Indirect registers accessor
  48 */
  49u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
  50{
  51	unsigned long flags;
  52	u32 r;
  53
  54	spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  55	WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  56	r = RREG32(EVERGREEN_CG_IND_DATA);
  57	spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  58	return r;
  59}
  60
  61void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  62{
  63	unsigned long flags;
  64
  65	spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  66	WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  67	WREG32(EVERGREEN_CG_IND_DATA, (v));
  68	spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  69}
  70
  71u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
  72{
  73	unsigned long flags;
  74	u32 r;
  75
  76	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  77	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  78	r = RREG32(EVERGREEN_PIF_PHY0_DATA);
  79	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  80	return r;
  81}
  82
  83void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  84{
  85	unsigned long flags;
  86
  87	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  88	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  89	WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
  90	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  91}
  92
  93u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
  94{
  95	unsigned long flags;
  96	u32 r;
  97
  98	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  99	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 100	r = RREG32(EVERGREEN_PIF_PHY1_DATA);
 101	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 102	return r;
 103}
 104
 105void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 106{
 107	unsigned long flags;
 108
 109	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 110	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 111	WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
 112	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 113}
 114
 115static const u32 crtc_offsets[6] =
 116{
 117	EVERGREEN_CRTC0_REGISTER_OFFSET,
 118	EVERGREEN_CRTC1_REGISTER_OFFSET,
 119	EVERGREEN_CRTC2_REGISTER_OFFSET,
 120	EVERGREEN_CRTC3_REGISTER_OFFSET,
 121	EVERGREEN_CRTC4_REGISTER_OFFSET,
 122	EVERGREEN_CRTC5_REGISTER_OFFSET
 123};
 124
 125#include "clearstate_evergreen.h"
 126
 127static const u32 sumo_rlc_save_restore_register_list[] =
 128{
 129	0x98fc,
 130	0x9830,
 131	0x9834,
 132	0x9838,
 133	0x9870,
 134	0x9874,
 135	0x8a14,
 136	0x8b24,
 137	0x8bcc,
 138	0x8b10,
 139	0x8d00,
 140	0x8d04,
 141	0x8c00,
 142	0x8c04,
 143	0x8c08,
 144	0x8c0c,
 145	0x8d8c,
 146	0x8c20,
 147	0x8c24,
 148	0x8c28,
 149	0x8c18,
 150	0x8c1c,
 151	0x8cf0,
 152	0x8e2c,
 153	0x8e38,
 154	0x8c30,
 155	0x9508,
 156	0x9688,
 157	0x9608,
 158	0x960c,
 159	0x9610,
 160	0x9614,
 161	0x88c4,
 162	0x88d4,
 163	0xa008,
 164	0x900c,
 165	0x9100,
 166	0x913c,
 167	0x98f8,
 168	0x98f4,
 169	0x9b7c,
 170	0x3f8c,
 171	0x8950,
 172	0x8954,
 173	0x8a18,
 174	0x8b28,
 175	0x9144,
 176	0x9148,
 177	0x914c,
 178	0x3f90,
 179	0x3f94,
 180	0x915c,
 181	0x9160,
 182	0x9178,
 183	0x917c,
 184	0x9180,
 185	0x918c,
 186	0x9190,
 187	0x9194,
 188	0x9198,
 189	0x919c,
 190	0x91a8,
 191	0x91ac,
 192	0x91b0,
 193	0x91b4,
 194	0x91b8,
 195	0x91c4,
 196	0x91c8,
 197	0x91cc,
 198	0x91d0,
 199	0x91d4,
 200	0x91e0,
 201	0x91e4,
 202	0x91ec,
 203	0x91f0,
 204	0x91f4,
 205	0x9200,
 206	0x9204,
 207	0x929c,
 208	0x9150,
 209	0x802c,
 210};
 211
 212static void evergreen_gpu_init(struct radeon_device *rdev);
 213void evergreen_fini(struct radeon_device *rdev);
 214void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 215void evergreen_program_aspm(struct radeon_device *rdev);
 216extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
 217				     int ring, u32 cp_int_cntl);
 218extern void cayman_vm_decode_fault(struct radeon_device *rdev,
 219				   u32 status, u32 addr);
 220void cik_init_cp_pg_table(struct radeon_device *rdev);
 221
 222extern u32 si_get_csb_size(struct radeon_device *rdev);
 223extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 224extern u32 cik_get_csb_size(struct radeon_device *rdev);
 225extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 226extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
 227
 228static const u32 evergreen_golden_registers[] =
 229{
 230	0x3f90, 0xffff0000, 0xff000000,
 231	0x9148, 0xffff0000, 0xff000000,
 232	0x3f94, 0xffff0000, 0xff000000,
 233	0x914c, 0xffff0000, 0xff000000,
 234	0x9b7c, 0xffffffff, 0x00000000,
 235	0x8a14, 0xffffffff, 0x00000007,
 236	0x8b10, 0xffffffff, 0x00000000,
 237	0x960c, 0xffffffff, 0x54763210,
 238	0x88c4, 0xffffffff, 0x000000c2,
 239	0x88d4, 0xffffffff, 0x00000010,
 240	0x8974, 0xffffffff, 0x00000000,
 241	0xc78, 0x00000080, 0x00000080,
 242	0x5eb4, 0xffffffff, 0x00000002,
 243	0x5e78, 0xffffffff, 0x001000f0,
 244	0x6104, 0x01000300, 0x00000000,
 245	0x5bc0, 0x00300000, 0x00000000,
 246	0x7030, 0xffffffff, 0x00000011,
 247	0x7c30, 0xffffffff, 0x00000011,
 248	0x10830, 0xffffffff, 0x00000011,
 249	0x11430, 0xffffffff, 0x00000011,
 250	0x12030, 0xffffffff, 0x00000011,
 251	0x12c30, 0xffffffff, 0x00000011,
 252	0xd02c, 0xffffffff, 0x08421000,
 253	0x240c, 0xffffffff, 0x00000380,
 254	0x8b24, 0xffffffff, 0x00ff0fff,
 255	0x28a4c, 0x06000000, 0x06000000,
 256	0x10c, 0x00000001, 0x00000001,
 257	0x8d00, 0xffffffff, 0x100e4848,
 258	0x8d04, 0xffffffff, 0x00164745,
 259	0x8c00, 0xffffffff, 0xe4000003,
 260	0x8c04, 0xffffffff, 0x40600060,
 261	0x8c08, 0xffffffff, 0x001c001c,
 262	0x8cf0, 0xffffffff, 0x08e00620,
 263	0x8c20, 0xffffffff, 0x00800080,
 264	0x8c24, 0xffffffff, 0x00800080,
 265	0x8c18, 0xffffffff, 0x20202078,
 266	0x8c1c, 0xffffffff, 0x00001010,
 267	0x28350, 0xffffffff, 0x00000000,
 268	0xa008, 0xffffffff, 0x00010000,
 269	0x5c4, 0xffffffff, 0x00000001,
 270	0x9508, 0xffffffff, 0x00000002,
 271	0x913c, 0x0000000f, 0x0000000a
 272};
 273
 274static const u32 evergreen_golden_registers2[] =
 275{
 276	0x2f4c, 0xffffffff, 0x00000000,
 277	0x54f4, 0xffffffff, 0x00000000,
 278	0x54f0, 0xffffffff, 0x00000000,
 279	0x5498, 0xffffffff, 0x00000000,
 280	0x549c, 0xffffffff, 0x00000000,
 281	0x5494, 0xffffffff, 0x00000000,
 282	0x53cc, 0xffffffff, 0x00000000,
 283	0x53c8, 0xffffffff, 0x00000000,
 284	0x53c4, 0xffffffff, 0x00000000,
 285	0x53c0, 0xffffffff, 0x00000000,
 286	0x53bc, 0xffffffff, 0x00000000,
 287	0x53b8, 0xffffffff, 0x00000000,
 288	0x53b4, 0xffffffff, 0x00000000,
 289	0x53b0, 0xffffffff, 0x00000000
 290};
 291
 292static const u32 cypress_mgcg_init[] =
 293{
 294	0x802c, 0xffffffff, 0xc0000000,
 295	0x5448, 0xffffffff, 0x00000100,
 296	0x55e4, 0xffffffff, 0x00000100,
 297	0x160c, 0xffffffff, 0x00000100,
 298	0x5644, 0xffffffff, 0x00000100,
 299	0xc164, 0xffffffff, 0x00000100,
 300	0x8a18, 0xffffffff, 0x00000100,
 301	0x897c, 0xffffffff, 0x06000100,
 302	0x8b28, 0xffffffff, 0x00000100,
 303	0x9144, 0xffffffff, 0x00000100,
 304	0x9a60, 0xffffffff, 0x00000100,
 305	0x9868, 0xffffffff, 0x00000100,
 306	0x8d58, 0xffffffff, 0x00000100,
 307	0x9510, 0xffffffff, 0x00000100,
 308	0x949c, 0xffffffff, 0x00000100,
 309	0x9654, 0xffffffff, 0x00000100,
 310	0x9030, 0xffffffff, 0x00000100,
 311	0x9034, 0xffffffff, 0x00000100,
 312	0x9038, 0xffffffff, 0x00000100,
 313	0x903c, 0xffffffff, 0x00000100,
 314	0x9040, 0xffffffff, 0x00000100,
 315	0xa200, 0xffffffff, 0x00000100,
 316	0xa204, 0xffffffff, 0x00000100,
 317	0xa208, 0xffffffff, 0x00000100,
 318	0xa20c, 0xffffffff, 0x00000100,
 319	0x971c, 0xffffffff, 0x00000100,
 320	0x977c, 0xffffffff, 0x00000100,
 321	0x3f80, 0xffffffff, 0x00000100,
 322	0xa210, 0xffffffff, 0x00000100,
 323	0xa214, 0xffffffff, 0x00000100,
 324	0x4d8, 0xffffffff, 0x00000100,
 325	0x9784, 0xffffffff, 0x00000100,
 326	0x9698, 0xffffffff, 0x00000100,
 327	0x4d4, 0xffffffff, 0x00000200,
 328	0x30cc, 0xffffffff, 0x00000100,
 329	0xd0c0, 0xffffffff, 0xff000100,
 330	0x802c, 0xffffffff, 0x40000000,
 331	0x915c, 0xffffffff, 0x00010000,
 332	0x9160, 0xffffffff, 0x00030002,
 333	0x9178, 0xffffffff, 0x00070000,
 334	0x917c, 0xffffffff, 0x00030002,
 335	0x9180, 0xffffffff, 0x00050004,
 336	0x918c, 0xffffffff, 0x00010006,
 337	0x9190, 0xffffffff, 0x00090008,
 338	0x9194, 0xffffffff, 0x00070000,
 339	0x9198, 0xffffffff, 0x00030002,
 340	0x919c, 0xffffffff, 0x00050004,
 341	0x91a8, 0xffffffff, 0x00010006,
 342	0x91ac, 0xffffffff, 0x00090008,
 343	0x91b0, 0xffffffff, 0x00070000,
 344	0x91b4, 0xffffffff, 0x00030002,
 345	0x91b8, 0xffffffff, 0x00050004,
 346	0x91c4, 0xffffffff, 0x00010006,
 347	0x91c8, 0xffffffff, 0x00090008,
 348	0x91cc, 0xffffffff, 0x00070000,
 349	0x91d0, 0xffffffff, 0x00030002,
 350	0x91d4, 0xffffffff, 0x00050004,
 351	0x91e0, 0xffffffff, 0x00010006,
 352	0x91e4, 0xffffffff, 0x00090008,
 353	0x91e8, 0xffffffff, 0x00000000,
 354	0x91ec, 0xffffffff, 0x00070000,
 355	0x91f0, 0xffffffff, 0x00030002,
 356	0x91f4, 0xffffffff, 0x00050004,
 357	0x9200, 0xffffffff, 0x00010006,
 358	0x9204, 0xffffffff, 0x00090008,
 359	0x9208, 0xffffffff, 0x00070000,
 360	0x920c, 0xffffffff, 0x00030002,
 361	0x9210, 0xffffffff, 0x00050004,
 362	0x921c, 0xffffffff, 0x00010006,
 363	0x9220, 0xffffffff, 0x00090008,
 364	0x9224, 0xffffffff, 0x00070000,
 365	0x9228, 0xffffffff, 0x00030002,
 366	0x922c, 0xffffffff, 0x00050004,
 367	0x9238, 0xffffffff, 0x00010006,
 368	0x923c, 0xffffffff, 0x00090008,
 369	0x9240, 0xffffffff, 0x00070000,
 370	0x9244, 0xffffffff, 0x00030002,
 371	0x9248, 0xffffffff, 0x00050004,
 372	0x9254, 0xffffffff, 0x00010006,
 373	0x9258, 0xffffffff, 0x00090008,
 374	0x925c, 0xffffffff, 0x00070000,
 375	0x9260, 0xffffffff, 0x00030002,
 376	0x9264, 0xffffffff, 0x00050004,
 377	0x9270, 0xffffffff, 0x00010006,
 378	0x9274, 0xffffffff, 0x00090008,
 379	0x9278, 0xffffffff, 0x00070000,
 380	0x927c, 0xffffffff, 0x00030002,
 381	0x9280, 0xffffffff, 0x00050004,
 382	0x928c, 0xffffffff, 0x00010006,
 383	0x9290, 0xffffffff, 0x00090008,
 384	0x9294, 0xffffffff, 0x00000000,
 385	0x929c, 0xffffffff, 0x00000001,
 386	0x802c, 0xffffffff, 0x40010000,
 387	0x915c, 0xffffffff, 0x00010000,
 388	0x9160, 0xffffffff, 0x00030002,
 389	0x9178, 0xffffffff, 0x00070000,
 390	0x917c, 0xffffffff, 0x00030002,
 391	0x9180, 0xffffffff, 0x00050004,
 392	0x918c, 0xffffffff, 0x00010006,
 393	0x9190, 0xffffffff, 0x00090008,
 394	0x9194, 0xffffffff, 0x00070000,
 395	0x9198, 0xffffffff, 0x00030002,
 396	0x919c, 0xffffffff, 0x00050004,
 397	0x91a8, 0xffffffff, 0x00010006,
 398	0x91ac, 0xffffffff, 0x00090008,
 399	0x91b0, 0xffffffff, 0x00070000,
 400	0x91b4, 0xffffffff, 0x00030002,
 401	0x91b8, 0xffffffff, 0x00050004,
 402	0x91c4, 0xffffffff, 0x00010006,
 403	0x91c8, 0xffffffff, 0x00090008,
 404	0x91cc, 0xffffffff, 0x00070000,
 405	0x91d0, 0xffffffff, 0x00030002,
 406	0x91d4, 0xffffffff, 0x00050004,
 407	0x91e0, 0xffffffff, 0x00010006,
 408	0x91e4, 0xffffffff, 0x00090008,
 409	0x91e8, 0xffffffff, 0x00000000,
 410	0x91ec, 0xffffffff, 0x00070000,
 411	0x91f0, 0xffffffff, 0x00030002,
 412	0x91f4, 0xffffffff, 0x00050004,
 413	0x9200, 0xffffffff, 0x00010006,
 414	0x9204, 0xffffffff, 0x00090008,
 415	0x9208, 0xffffffff, 0x00070000,
 416	0x920c, 0xffffffff, 0x00030002,
 417	0x9210, 0xffffffff, 0x00050004,
 418	0x921c, 0xffffffff, 0x00010006,
 419	0x9220, 0xffffffff, 0x00090008,
 420	0x9224, 0xffffffff, 0x00070000,
 421	0x9228, 0xffffffff, 0x00030002,
 422	0x922c, 0xffffffff, 0x00050004,
 423	0x9238, 0xffffffff, 0x00010006,
 424	0x923c, 0xffffffff, 0x00090008,
 425	0x9240, 0xffffffff, 0x00070000,
 426	0x9244, 0xffffffff, 0x00030002,
 427	0x9248, 0xffffffff, 0x00050004,
 428	0x9254, 0xffffffff, 0x00010006,
 429	0x9258, 0xffffffff, 0x00090008,
 430	0x925c, 0xffffffff, 0x00070000,
 431	0x9260, 0xffffffff, 0x00030002,
 432	0x9264, 0xffffffff, 0x00050004,
 433	0x9270, 0xffffffff, 0x00010006,
 434	0x9274, 0xffffffff, 0x00090008,
 435	0x9278, 0xffffffff, 0x00070000,
 436	0x927c, 0xffffffff, 0x00030002,
 437	0x9280, 0xffffffff, 0x00050004,
 438	0x928c, 0xffffffff, 0x00010006,
 439	0x9290, 0xffffffff, 0x00090008,
 440	0x9294, 0xffffffff, 0x00000000,
 441	0x929c, 0xffffffff, 0x00000001,
 442	0x802c, 0xffffffff, 0xc0000000
 443};
 444
 445static const u32 redwood_mgcg_init[] =
 446{
 447	0x802c, 0xffffffff, 0xc0000000,
 448	0x5448, 0xffffffff, 0x00000100,
 449	0x55e4, 0xffffffff, 0x00000100,
 450	0x160c, 0xffffffff, 0x00000100,
 451	0x5644, 0xffffffff, 0x00000100,
 452	0xc164, 0xffffffff, 0x00000100,
 453	0x8a18, 0xffffffff, 0x00000100,
 454	0x897c, 0xffffffff, 0x06000100,
 455	0x8b28, 0xffffffff, 0x00000100,
 456	0x9144, 0xffffffff, 0x00000100,
 457	0x9a60, 0xffffffff, 0x00000100,
 458	0x9868, 0xffffffff, 0x00000100,
 459	0x8d58, 0xffffffff, 0x00000100,
 460	0x9510, 0xffffffff, 0x00000100,
 461	0x949c, 0xffffffff, 0x00000100,
 462	0x9654, 0xffffffff, 0x00000100,
 463	0x9030, 0xffffffff, 0x00000100,
 464	0x9034, 0xffffffff, 0x00000100,
 465	0x9038, 0xffffffff, 0x00000100,
 466	0x903c, 0xffffffff, 0x00000100,
 467	0x9040, 0xffffffff, 0x00000100,
 468	0xa200, 0xffffffff, 0x00000100,
 469	0xa204, 0xffffffff, 0x00000100,
 470	0xa208, 0xffffffff, 0x00000100,
 471	0xa20c, 0xffffffff, 0x00000100,
 472	0x971c, 0xffffffff, 0x00000100,
 473	0x977c, 0xffffffff, 0x00000100,
 474	0x3f80, 0xffffffff, 0x00000100,
 475	0xa210, 0xffffffff, 0x00000100,
 476	0xa214, 0xffffffff, 0x00000100,
 477	0x4d8, 0xffffffff, 0x00000100,
 478	0x9784, 0xffffffff, 0x00000100,
 479	0x9698, 0xffffffff, 0x00000100,
 480	0x4d4, 0xffffffff, 0x00000200,
 481	0x30cc, 0xffffffff, 0x00000100,
 482	0xd0c0, 0xffffffff, 0xff000100,
 483	0x802c, 0xffffffff, 0x40000000,
 484	0x915c, 0xffffffff, 0x00010000,
 485	0x9160, 0xffffffff, 0x00030002,
 486	0x9178, 0xffffffff, 0x00070000,
 487	0x917c, 0xffffffff, 0x00030002,
 488	0x9180, 0xffffffff, 0x00050004,
 489	0x918c, 0xffffffff, 0x00010006,
 490	0x9190, 0xffffffff, 0x00090008,
 491	0x9194, 0xffffffff, 0x00070000,
 492	0x9198, 0xffffffff, 0x00030002,
 493	0x919c, 0xffffffff, 0x00050004,
 494	0x91a8, 0xffffffff, 0x00010006,
 495	0x91ac, 0xffffffff, 0x00090008,
 496	0x91b0, 0xffffffff, 0x00070000,
 497	0x91b4, 0xffffffff, 0x00030002,
 498	0x91b8, 0xffffffff, 0x00050004,
 499	0x91c4, 0xffffffff, 0x00010006,
 500	0x91c8, 0xffffffff, 0x00090008,
 501	0x91cc, 0xffffffff, 0x00070000,
 502	0x91d0, 0xffffffff, 0x00030002,
 503	0x91d4, 0xffffffff, 0x00050004,
 504	0x91e0, 0xffffffff, 0x00010006,
 505	0x91e4, 0xffffffff, 0x00090008,
 506	0x91e8, 0xffffffff, 0x00000000,
 507	0x91ec, 0xffffffff, 0x00070000,
 508	0x91f0, 0xffffffff, 0x00030002,
 509	0x91f4, 0xffffffff, 0x00050004,
 510	0x9200, 0xffffffff, 0x00010006,
 511	0x9204, 0xffffffff, 0x00090008,
 512	0x9294, 0xffffffff, 0x00000000,
 513	0x929c, 0xffffffff, 0x00000001,
 514	0x802c, 0xffffffff, 0xc0000000
 515};
 516
 517static const u32 cedar_golden_registers[] =
 518{
 519	0x3f90, 0xffff0000, 0xff000000,
 520	0x9148, 0xffff0000, 0xff000000,
 521	0x3f94, 0xffff0000, 0xff000000,
 522	0x914c, 0xffff0000, 0xff000000,
 523	0x9b7c, 0xffffffff, 0x00000000,
 524	0x8a14, 0xffffffff, 0x00000007,
 525	0x8b10, 0xffffffff, 0x00000000,
 526	0x960c, 0xffffffff, 0x54763210,
 527	0x88c4, 0xffffffff, 0x000000c2,
 528	0x88d4, 0xffffffff, 0x00000000,
 529	0x8974, 0xffffffff, 0x00000000,
 530	0xc78, 0x00000080, 0x00000080,
 531	0x5eb4, 0xffffffff, 0x00000002,
 532	0x5e78, 0xffffffff, 0x001000f0,
 533	0x6104, 0x01000300, 0x00000000,
 534	0x5bc0, 0x00300000, 0x00000000,
 535	0x7030, 0xffffffff, 0x00000011,
 536	0x7c30, 0xffffffff, 0x00000011,
 537	0x10830, 0xffffffff, 0x00000011,
 538	0x11430, 0xffffffff, 0x00000011,
 539	0xd02c, 0xffffffff, 0x08421000,
 540	0x240c, 0xffffffff, 0x00000380,
 541	0x8b24, 0xffffffff, 0x00ff0fff,
 542	0x28a4c, 0x06000000, 0x06000000,
 543	0x10c, 0x00000001, 0x00000001,
 544	0x8d00, 0xffffffff, 0x100e4848,
 545	0x8d04, 0xffffffff, 0x00164745,
 546	0x8c00, 0xffffffff, 0xe4000003,
 547	0x8c04, 0xffffffff, 0x40600060,
 548	0x8c08, 0xffffffff, 0x001c001c,
 549	0x8cf0, 0xffffffff, 0x08e00410,
 550	0x8c20, 0xffffffff, 0x00800080,
 551	0x8c24, 0xffffffff, 0x00800080,
 552	0x8c18, 0xffffffff, 0x20202078,
 553	0x8c1c, 0xffffffff, 0x00001010,
 554	0x28350, 0xffffffff, 0x00000000,
 555	0xa008, 0xffffffff, 0x00010000,
 556	0x5c4, 0xffffffff, 0x00000001,
 557	0x9508, 0xffffffff, 0x00000002
 558};
 559
 560static const u32 cedar_mgcg_init[] =
 561{
 562	0x802c, 0xffffffff, 0xc0000000,
 563	0x5448, 0xffffffff, 0x00000100,
 564	0x55e4, 0xffffffff, 0x00000100,
 565	0x160c, 0xffffffff, 0x00000100,
 566	0x5644, 0xffffffff, 0x00000100,
 567	0xc164, 0xffffffff, 0x00000100,
 568	0x8a18, 0xffffffff, 0x00000100,
 569	0x897c, 0xffffffff, 0x06000100,
 570	0x8b28, 0xffffffff, 0x00000100,
 571	0x9144, 0xffffffff, 0x00000100,
 572	0x9a60, 0xffffffff, 0x00000100,
 573	0x9868, 0xffffffff, 0x00000100,
 574	0x8d58, 0xffffffff, 0x00000100,
 575	0x9510, 0xffffffff, 0x00000100,
 576	0x949c, 0xffffffff, 0x00000100,
 577	0x9654, 0xffffffff, 0x00000100,
 578	0x9030, 0xffffffff, 0x00000100,
 579	0x9034, 0xffffffff, 0x00000100,
 580	0x9038, 0xffffffff, 0x00000100,
 581	0x903c, 0xffffffff, 0x00000100,
 582	0x9040, 0xffffffff, 0x00000100,
 583	0xa200, 0xffffffff, 0x00000100,
 584	0xa204, 0xffffffff, 0x00000100,
 585	0xa208, 0xffffffff, 0x00000100,
 586	0xa20c, 0xffffffff, 0x00000100,
 587	0x971c, 0xffffffff, 0x00000100,
 588	0x977c, 0xffffffff, 0x00000100,
 589	0x3f80, 0xffffffff, 0x00000100,
 590	0xa210, 0xffffffff, 0x00000100,
 591	0xa214, 0xffffffff, 0x00000100,
 592	0x4d8, 0xffffffff, 0x00000100,
 593	0x9784, 0xffffffff, 0x00000100,
 594	0x9698, 0xffffffff, 0x00000100,
 595	0x4d4, 0xffffffff, 0x00000200,
 596	0x30cc, 0xffffffff, 0x00000100,
 597	0xd0c0, 0xffffffff, 0xff000100,
 598	0x802c, 0xffffffff, 0x40000000,
 599	0x915c, 0xffffffff, 0x00010000,
 600	0x9178, 0xffffffff, 0x00050000,
 601	0x917c, 0xffffffff, 0x00030002,
 602	0x918c, 0xffffffff, 0x00010004,
 603	0x9190, 0xffffffff, 0x00070006,
 604	0x9194, 0xffffffff, 0x00050000,
 605	0x9198, 0xffffffff, 0x00030002,
 606	0x91a8, 0xffffffff, 0x00010004,
 607	0x91ac, 0xffffffff, 0x00070006,
 608	0x91e8, 0xffffffff, 0x00000000,
 609	0x9294, 0xffffffff, 0x00000000,
 610	0x929c, 0xffffffff, 0x00000001,
 611	0x802c, 0xffffffff, 0xc0000000
 612};
 613
 614static const u32 juniper_mgcg_init[] =
 615{
 616	0x802c, 0xffffffff, 0xc0000000,
 617	0x5448, 0xffffffff, 0x00000100,
 618	0x55e4, 0xffffffff, 0x00000100,
 619	0x160c, 0xffffffff, 0x00000100,
 620	0x5644, 0xffffffff, 0x00000100,
 621	0xc164, 0xffffffff, 0x00000100,
 622	0x8a18, 0xffffffff, 0x00000100,
 623	0x897c, 0xffffffff, 0x06000100,
 624	0x8b28, 0xffffffff, 0x00000100,
 625	0x9144, 0xffffffff, 0x00000100,
 626	0x9a60, 0xffffffff, 0x00000100,
 627	0x9868, 0xffffffff, 0x00000100,
 628	0x8d58, 0xffffffff, 0x00000100,
 629	0x9510, 0xffffffff, 0x00000100,
 630	0x949c, 0xffffffff, 0x00000100,
 631	0x9654, 0xffffffff, 0x00000100,
 632	0x9030, 0xffffffff, 0x00000100,
 633	0x9034, 0xffffffff, 0x00000100,
 634	0x9038, 0xffffffff, 0x00000100,
 635	0x903c, 0xffffffff, 0x00000100,
 636	0x9040, 0xffffffff, 0x00000100,
 637	0xa200, 0xffffffff, 0x00000100,
 638	0xa204, 0xffffffff, 0x00000100,
 639	0xa208, 0xffffffff, 0x00000100,
 640	0xa20c, 0xffffffff, 0x00000100,
 641	0x971c, 0xffffffff, 0x00000100,
 642	0xd0c0, 0xffffffff, 0xff000100,
 643	0x802c, 0xffffffff, 0x40000000,
 644	0x915c, 0xffffffff, 0x00010000,
 645	0x9160, 0xffffffff, 0x00030002,
 646	0x9178, 0xffffffff, 0x00070000,
 647	0x917c, 0xffffffff, 0x00030002,
 648	0x9180, 0xffffffff, 0x00050004,
 649	0x918c, 0xffffffff, 0x00010006,
 650	0x9190, 0xffffffff, 0x00090008,
 651	0x9194, 0xffffffff, 0x00070000,
 652	0x9198, 0xffffffff, 0x00030002,
 653	0x919c, 0xffffffff, 0x00050004,
 654	0x91a8, 0xffffffff, 0x00010006,
 655	0x91ac, 0xffffffff, 0x00090008,
 656	0x91b0, 0xffffffff, 0x00070000,
 657	0x91b4, 0xffffffff, 0x00030002,
 658	0x91b8, 0xffffffff, 0x00050004,
 659	0x91c4, 0xffffffff, 0x00010006,
 660	0x91c8, 0xffffffff, 0x00090008,
 661	0x91cc, 0xffffffff, 0x00070000,
 662	0x91d0, 0xffffffff, 0x00030002,
 663	0x91d4, 0xffffffff, 0x00050004,
 664	0x91e0, 0xffffffff, 0x00010006,
 665	0x91e4, 0xffffffff, 0x00090008,
 666	0x91e8, 0xffffffff, 0x00000000,
 667	0x91ec, 0xffffffff, 0x00070000,
 668	0x91f0, 0xffffffff, 0x00030002,
 669	0x91f4, 0xffffffff, 0x00050004,
 670	0x9200, 0xffffffff, 0x00010006,
 671	0x9204, 0xffffffff, 0x00090008,
 672	0x9208, 0xffffffff, 0x00070000,
 673	0x920c, 0xffffffff, 0x00030002,
 674	0x9210, 0xffffffff, 0x00050004,
 675	0x921c, 0xffffffff, 0x00010006,
 676	0x9220, 0xffffffff, 0x00090008,
 677	0x9224, 0xffffffff, 0x00070000,
 678	0x9228, 0xffffffff, 0x00030002,
 679	0x922c, 0xffffffff, 0x00050004,
 680	0x9238, 0xffffffff, 0x00010006,
 681	0x923c, 0xffffffff, 0x00090008,
 682	0x9240, 0xffffffff, 0x00070000,
 683	0x9244, 0xffffffff, 0x00030002,
 684	0x9248, 0xffffffff, 0x00050004,
 685	0x9254, 0xffffffff, 0x00010006,
 686	0x9258, 0xffffffff, 0x00090008,
 687	0x925c, 0xffffffff, 0x00070000,
 688	0x9260, 0xffffffff, 0x00030002,
 689	0x9264, 0xffffffff, 0x00050004,
 690	0x9270, 0xffffffff, 0x00010006,
 691	0x9274, 0xffffffff, 0x00090008,
 692	0x9278, 0xffffffff, 0x00070000,
 693	0x927c, 0xffffffff, 0x00030002,
 694	0x9280, 0xffffffff, 0x00050004,
 695	0x928c, 0xffffffff, 0x00010006,
 696	0x9290, 0xffffffff, 0x00090008,
 697	0x9294, 0xffffffff, 0x00000000,
 698	0x929c, 0xffffffff, 0x00000001,
 699	0x802c, 0xffffffff, 0xc0000000,
 700	0x977c, 0xffffffff, 0x00000100,
 701	0x3f80, 0xffffffff, 0x00000100,
 702	0xa210, 0xffffffff, 0x00000100,
 703	0xa214, 0xffffffff, 0x00000100,
 704	0x4d8, 0xffffffff, 0x00000100,
 705	0x9784, 0xffffffff, 0x00000100,
 706	0x9698, 0xffffffff, 0x00000100,
 707	0x4d4, 0xffffffff, 0x00000200,
 708	0x30cc, 0xffffffff, 0x00000100,
 709	0x802c, 0xffffffff, 0xc0000000
 710};
 711
 712static const u32 supersumo_golden_registers[] =
 713{
 714	0x5eb4, 0xffffffff, 0x00000002,
 715	0x5c4, 0xffffffff, 0x00000001,
 716	0x7030, 0xffffffff, 0x00000011,
 717	0x7c30, 0xffffffff, 0x00000011,
 718	0x6104, 0x01000300, 0x00000000,
 719	0x5bc0, 0x00300000, 0x00000000,
 720	0x8c04, 0xffffffff, 0x40600060,
 721	0x8c08, 0xffffffff, 0x001c001c,
 722	0x8c20, 0xffffffff, 0x00800080,
 723	0x8c24, 0xffffffff, 0x00800080,
 724	0x8c18, 0xffffffff, 0x20202078,
 725	0x8c1c, 0xffffffff, 0x00001010,
 726	0x918c, 0xffffffff, 0x00010006,
 727	0x91a8, 0xffffffff, 0x00010006,
 728	0x91c4, 0xffffffff, 0x00010006,
 729	0x91e0, 0xffffffff, 0x00010006,
 730	0x9200, 0xffffffff, 0x00010006,
 731	0x9150, 0xffffffff, 0x6e944040,
 732	0x917c, 0xffffffff, 0x00030002,
 733	0x9180, 0xffffffff, 0x00050004,
 734	0x9198, 0xffffffff, 0x00030002,
 735	0x919c, 0xffffffff, 0x00050004,
 736	0x91b4, 0xffffffff, 0x00030002,
 737	0x91b8, 0xffffffff, 0x00050004,
 738	0x91d0, 0xffffffff, 0x00030002,
 739	0x91d4, 0xffffffff, 0x00050004,
 740	0x91f0, 0xffffffff, 0x00030002,
 741	0x91f4, 0xffffffff, 0x00050004,
 742	0x915c, 0xffffffff, 0x00010000,
 743	0x9160, 0xffffffff, 0x00030002,
 744	0x3f90, 0xffff0000, 0xff000000,
 745	0x9178, 0xffffffff, 0x00070000,
 746	0x9194, 0xffffffff, 0x00070000,
 747	0x91b0, 0xffffffff, 0x00070000,
 748	0x91cc, 0xffffffff, 0x00070000,
 749	0x91ec, 0xffffffff, 0x00070000,
 750	0x9148, 0xffff0000, 0xff000000,
 751	0x9190, 0xffffffff, 0x00090008,
 752	0x91ac, 0xffffffff, 0x00090008,
 753	0x91c8, 0xffffffff, 0x00090008,
 754	0x91e4, 0xffffffff, 0x00090008,
 755	0x9204, 0xffffffff, 0x00090008,
 756	0x3f94, 0xffff0000, 0xff000000,
 757	0x914c, 0xffff0000, 0xff000000,
 758	0x929c, 0xffffffff, 0x00000001,
 759	0x8a18, 0xffffffff, 0x00000100,
 760	0x8b28, 0xffffffff, 0x00000100,
 761	0x9144, 0xffffffff, 0x00000100,
 762	0x5644, 0xffffffff, 0x00000100,
 763	0x9b7c, 0xffffffff, 0x00000000,
 764	0x8030, 0xffffffff, 0x0000100a,
 765	0x8a14, 0xffffffff, 0x00000007,
 766	0x8b24, 0xffffffff, 0x00ff0fff,
 767	0x8b10, 0xffffffff, 0x00000000,
 768	0x28a4c, 0x06000000, 0x06000000,
 769	0x4d8, 0xffffffff, 0x00000100,
 770	0x913c, 0xffff000f, 0x0100000a,
 771	0x960c, 0xffffffff, 0x54763210,
 772	0x88c4, 0xffffffff, 0x000000c2,
 773	0x88d4, 0xffffffff, 0x00000010,
 774	0x8974, 0xffffffff, 0x00000000,
 775	0xc78, 0x00000080, 0x00000080,
 776	0x5e78, 0xffffffff, 0x001000f0,
 777	0xd02c, 0xffffffff, 0x08421000,
 778	0xa008, 0xffffffff, 0x00010000,
 779	0x8d00, 0xffffffff, 0x100e4848,
 780	0x8d04, 0xffffffff, 0x00164745,
 781	0x8c00, 0xffffffff, 0xe4000003,
 782	0x8cf0, 0x1fffffff, 0x08e00620,
 783	0x28350, 0xffffffff, 0x00000000,
 784	0x9508, 0xffffffff, 0x00000002
 785};
 786
 787static const u32 sumo_golden_registers[] =
 788{
 789	0x900c, 0x00ffffff, 0x0017071f,
 790	0x8c18, 0xffffffff, 0x10101060,
 791	0x8c1c, 0xffffffff, 0x00001010,
 792	0x8c30, 0x0000000f, 0x00000005,
 793	0x9688, 0x0000000f, 0x00000007
 794};
 795
 796static const u32 wrestler_golden_registers[] =
 797{
 798	0x5eb4, 0xffffffff, 0x00000002,
 799	0x5c4, 0xffffffff, 0x00000001,
 800	0x7030, 0xffffffff, 0x00000011,
 801	0x7c30, 0xffffffff, 0x00000011,
 802	0x6104, 0x01000300, 0x00000000,
 803	0x5bc0, 0x00300000, 0x00000000,
 804	0x918c, 0xffffffff, 0x00010006,
 805	0x91a8, 0xffffffff, 0x00010006,
 806	0x9150, 0xffffffff, 0x6e944040,
 807	0x917c, 0xffffffff, 0x00030002,
 808	0x9198, 0xffffffff, 0x00030002,
 809	0x915c, 0xffffffff, 0x00010000,
 810	0x3f90, 0xffff0000, 0xff000000,
 811	0x9178, 0xffffffff, 0x00070000,
 812	0x9194, 0xffffffff, 0x00070000,
 813	0x9148, 0xffff0000, 0xff000000,
 814	0x9190, 0xffffffff, 0x00090008,
 815	0x91ac, 0xffffffff, 0x00090008,
 816	0x3f94, 0xffff0000, 0xff000000,
 817	0x914c, 0xffff0000, 0xff000000,
 818	0x929c, 0xffffffff, 0x00000001,
 819	0x8a18, 0xffffffff, 0x00000100,
 820	0x8b28, 0xffffffff, 0x00000100,
 821	0x9144, 0xffffffff, 0x00000100,
 822	0x9b7c, 0xffffffff, 0x00000000,
 823	0x8030, 0xffffffff, 0x0000100a,
 824	0x8a14, 0xffffffff, 0x00000001,
 825	0x8b24, 0xffffffff, 0x00ff0fff,
 826	0x8b10, 0xffffffff, 0x00000000,
 827	0x28a4c, 0x06000000, 0x06000000,
 828	0x4d8, 0xffffffff, 0x00000100,
 829	0x913c, 0xffff000f, 0x0100000a,
 830	0x960c, 0xffffffff, 0x54763210,
 831	0x88c4, 0xffffffff, 0x000000c2,
 832	0x88d4, 0xffffffff, 0x00000010,
 833	0x8974, 0xffffffff, 0x00000000,
 834	0xc78, 0x00000080, 0x00000080,
 835	0x5e78, 0xffffffff, 0x001000f0,
 836	0xd02c, 0xffffffff, 0x08421000,
 837	0xa008, 0xffffffff, 0x00010000,
 838	0x8d00, 0xffffffff, 0x100e4848,
 839	0x8d04, 0xffffffff, 0x00164745,
 840	0x8c00, 0xffffffff, 0xe4000003,
 841	0x8cf0, 0x1fffffff, 0x08e00410,
 842	0x28350, 0xffffffff, 0x00000000,
 843	0x9508, 0xffffffff, 0x00000002,
 844	0x900c, 0xffffffff, 0x0017071f,
 845	0x8c18, 0xffffffff, 0x10101060,
 846	0x8c1c, 0xffffffff, 0x00001010
 847};
 848
 849static const u32 barts_golden_registers[] =
 850{
 851	0x5eb4, 0xffffffff, 0x00000002,
 852	0x5e78, 0x8f311ff1, 0x001000f0,
 853	0x3f90, 0xffff0000, 0xff000000,
 854	0x9148, 0xffff0000, 0xff000000,
 855	0x3f94, 0xffff0000, 0xff000000,
 856	0x914c, 0xffff0000, 0xff000000,
 857	0xc78, 0x00000080, 0x00000080,
 858	0xbd4, 0x70073777, 0x00010001,
 859	0xd02c, 0xbfffff1f, 0x08421000,
 860	0xd0b8, 0x03773777, 0x02011003,
 861	0x5bc0, 0x00200000, 0x50100000,
 862	0x98f8, 0x33773777, 0x02011003,
 863	0x98fc, 0xffffffff, 0x76543210,
 864	0x7030, 0x31000311, 0x00000011,
 865	0x2f48, 0x00000007, 0x02011003,
 866	0x6b28, 0x00000010, 0x00000012,
 867	0x7728, 0x00000010, 0x00000012,
 868	0x10328, 0x00000010, 0x00000012,
 869	0x10f28, 0x00000010, 0x00000012,
 870	0x11b28, 0x00000010, 0x00000012,
 871	0x12728, 0x00000010, 0x00000012,
 872	0x240c, 0x000007ff, 0x00000380,
 873	0x8a14, 0xf000001f, 0x00000007,
 874	0x8b24, 0x3fff3fff, 0x00ff0fff,
 875	0x8b10, 0x0000ff0f, 0x00000000,
 876	0x28a4c, 0x07ffffff, 0x06000000,
 877	0x10c, 0x00000001, 0x00010003,
 878	0xa02c, 0xffffffff, 0x0000009b,
 879	0x913c, 0x0000000f, 0x0100000a,
 880	0x8d00, 0xffff7f7f, 0x100e4848,
 881	0x8d04, 0x00ffffff, 0x00164745,
 882	0x8c00, 0xfffc0003, 0xe4000003,
 883	0x8c04, 0xf8ff00ff, 0x40600060,
 884	0x8c08, 0x00ff00ff, 0x001c001c,
 885	0x8cf0, 0x1fff1fff, 0x08e00620,
 886	0x8c20, 0x0fff0fff, 0x00800080,
 887	0x8c24, 0x0fff0fff, 0x00800080,
 888	0x8c18, 0xffffffff, 0x20202078,
 889	0x8c1c, 0x0000ffff, 0x00001010,
 890	0x28350, 0x00000f01, 0x00000000,
 891	0x9508, 0x3700001f, 0x00000002,
 892	0x960c, 0xffffffff, 0x54763210,
 893	0x88c4, 0x001f3ae3, 0x000000c2,
 894	0x88d4, 0x0000001f, 0x00000010,
 895	0x8974, 0xffffffff, 0x00000000
 896};
 897
 898static const u32 turks_golden_registers[] =
 899{
 900	0x5eb4, 0xffffffff, 0x00000002,
 901	0x5e78, 0x8f311ff1, 0x001000f0,
 902	0x8c8, 0x00003000, 0x00001070,
 903	0x8cc, 0x000fffff, 0x00040035,
 904	0x3f90, 0xffff0000, 0xfff00000,
 905	0x9148, 0xffff0000, 0xfff00000,
 906	0x3f94, 0xffff0000, 0xfff00000,
 907	0x914c, 0xffff0000, 0xfff00000,
 908	0xc78, 0x00000080, 0x00000080,
 909	0xbd4, 0x00073007, 0x00010002,
 910	0xd02c, 0xbfffff1f, 0x08421000,
 911	0xd0b8, 0x03773777, 0x02010002,
 912	0x5bc0, 0x00200000, 0x50100000,
 913	0x98f8, 0x33773777, 0x00010002,
 914	0x98fc, 0xffffffff, 0x33221100,
 915	0x7030, 0x31000311, 0x00000011,
 916	0x2f48, 0x33773777, 0x00010002,
 917	0x6b28, 0x00000010, 0x00000012,
 918	0x7728, 0x00000010, 0x00000012,
 919	0x10328, 0x00000010, 0x00000012,
 920	0x10f28, 0x00000010, 0x00000012,
 921	0x11b28, 0x00000010, 0x00000012,
 922	0x12728, 0x00000010, 0x00000012,
 923	0x240c, 0x000007ff, 0x00000380,
 924	0x8a14, 0xf000001f, 0x00000007,
 925	0x8b24, 0x3fff3fff, 0x00ff0fff,
 926	0x8b10, 0x0000ff0f, 0x00000000,
 927	0x28a4c, 0x07ffffff, 0x06000000,
 928	0x10c, 0x00000001, 0x00010003,
 929	0xa02c, 0xffffffff, 0x0000009b,
 930	0x913c, 0x0000000f, 0x0100000a,
 931	0x8d00, 0xffff7f7f, 0x100e4848,
 932	0x8d04, 0x00ffffff, 0x00164745,
 933	0x8c00, 0xfffc0003, 0xe4000003,
 934	0x8c04, 0xf8ff00ff, 0x40600060,
 935	0x8c08, 0x00ff00ff, 0x001c001c,
 936	0x8cf0, 0x1fff1fff, 0x08e00410,
 937	0x8c20, 0x0fff0fff, 0x00800080,
 938	0x8c24, 0x0fff0fff, 0x00800080,
 939	0x8c18, 0xffffffff, 0x20202078,
 940	0x8c1c, 0x0000ffff, 0x00001010,
 941	0x28350, 0x00000f01, 0x00000000,
 942	0x9508, 0x3700001f, 0x00000002,
 943	0x960c, 0xffffffff, 0x54763210,
 944	0x88c4, 0x001f3ae3, 0x000000c2,
 945	0x88d4, 0x0000001f, 0x00000010,
 946	0x8974, 0xffffffff, 0x00000000
 947};
 948
 949static const u32 caicos_golden_registers[] =
 950{
 951	0x5eb4, 0xffffffff, 0x00000002,
 952	0x5e78, 0x8f311ff1, 0x001000f0,
 953	0x8c8, 0x00003420, 0x00001450,
 954	0x8cc, 0x000fffff, 0x00040035,
 955	0x3f90, 0xffff0000, 0xfffc0000,
 956	0x9148, 0xffff0000, 0xfffc0000,
 957	0x3f94, 0xffff0000, 0xfffc0000,
 958	0x914c, 0xffff0000, 0xfffc0000,
 959	0xc78, 0x00000080, 0x00000080,
 960	0xbd4, 0x00073007, 0x00010001,
 961	0xd02c, 0xbfffff1f, 0x08421000,
 962	0xd0b8, 0x03773777, 0x02010001,
 963	0x5bc0, 0x00200000, 0x50100000,
 964	0x98f8, 0x33773777, 0x02010001,
 965	0x98fc, 0xffffffff, 0x33221100,
 966	0x7030, 0x31000311, 0x00000011,
 967	0x2f48, 0x33773777, 0x02010001,
 968	0x6b28, 0x00000010, 0x00000012,
 969	0x7728, 0x00000010, 0x00000012,
 970	0x10328, 0x00000010, 0x00000012,
 971	0x10f28, 0x00000010, 0x00000012,
 972	0x11b28, 0x00000010, 0x00000012,
 973	0x12728, 0x00000010, 0x00000012,
 974	0x240c, 0x000007ff, 0x00000380,
 975	0x8a14, 0xf000001f, 0x00000001,
 976	0x8b24, 0x3fff3fff, 0x00ff0fff,
 977	0x8b10, 0x0000ff0f, 0x00000000,
 978	0x28a4c, 0x07ffffff, 0x06000000,
 979	0x10c, 0x00000001, 0x00010003,
 980	0xa02c, 0xffffffff, 0x0000009b,
 981	0x913c, 0x0000000f, 0x0100000a,
 982	0x8d00, 0xffff7f7f, 0x100e4848,
 983	0x8d04, 0x00ffffff, 0x00164745,
 984	0x8c00, 0xfffc0003, 0xe4000003,
 985	0x8c04, 0xf8ff00ff, 0x40600060,
 986	0x8c08, 0x00ff00ff, 0x001c001c,
 987	0x8cf0, 0x1fff1fff, 0x08e00410,
 988	0x8c20, 0x0fff0fff, 0x00800080,
 989	0x8c24, 0x0fff0fff, 0x00800080,
 990	0x8c18, 0xffffffff, 0x20202078,
 991	0x8c1c, 0x0000ffff, 0x00001010,
 992	0x28350, 0x00000f01, 0x00000000,
 993	0x9508, 0x3700001f, 0x00000002,
 994	0x960c, 0xffffffff, 0x54763210,
 995	0x88c4, 0x001f3ae3, 0x000000c2,
 996	0x88d4, 0x0000001f, 0x00000010,
 997	0x8974, 0xffffffff, 0x00000000
 998};
 999
1000static void evergreen_init_golden_registers(struct radeon_device *rdev)
1001{
1002	switch (rdev->family) {
1003	case CHIP_CYPRESS:
1004	case CHIP_HEMLOCK:
1005		radeon_program_register_sequence(rdev,
1006						 evergreen_golden_registers,
1007						 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1008		radeon_program_register_sequence(rdev,
1009						 evergreen_golden_registers2,
1010						 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1011		radeon_program_register_sequence(rdev,
1012						 cypress_mgcg_init,
1013						 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1014		break;
1015	case CHIP_JUNIPER:
1016		radeon_program_register_sequence(rdev,
1017						 evergreen_golden_registers,
1018						 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1019		radeon_program_register_sequence(rdev,
1020						 evergreen_golden_registers2,
1021						 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1022		radeon_program_register_sequence(rdev,
1023						 juniper_mgcg_init,
1024						 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1025		break;
1026	case CHIP_REDWOOD:
1027		radeon_program_register_sequence(rdev,
1028						 evergreen_golden_registers,
1029						 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1030		radeon_program_register_sequence(rdev,
1031						 evergreen_golden_registers2,
1032						 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1033		radeon_program_register_sequence(rdev,
1034						 redwood_mgcg_init,
1035						 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1036		break;
1037	case CHIP_CEDAR:
1038		radeon_program_register_sequence(rdev,
1039						 cedar_golden_registers,
1040						 (const u32)ARRAY_SIZE(cedar_golden_registers));
1041		radeon_program_register_sequence(rdev,
1042						 evergreen_golden_registers2,
1043						 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1044		radeon_program_register_sequence(rdev,
1045						 cedar_mgcg_init,
1046						 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1047		break;
1048	case CHIP_PALM:
1049		radeon_program_register_sequence(rdev,
1050						 wrestler_golden_registers,
1051						 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1052		break;
1053	case CHIP_SUMO:
1054		radeon_program_register_sequence(rdev,
1055						 supersumo_golden_registers,
1056						 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1057		break;
1058	case CHIP_SUMO2:
1059		radeon_program_register_sequence(rdev,
1060						 supersumo_golden_registers,
1061						 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1062		radeon_program_register_sequence(rdev,
1063						 sumo_golden_registers,
1064						 (const u32)ARRAY_SIZE(sumo_golden_registers));
1065		break;
1066	case CHIP_BARTS:
1067		radeon_program_register_sequence(rdev,
1068						 barts_golden_registers,
1069						 (const u32)ARRAY_SIZE(barts_golden_registers));
1070		break;
1071	case CHIP_TURKS:
1072		radeon_program_register_sequence(rdev,
1073						 turks_golden_registers,
1074						 (const u32)ARRAY_SIZE(turks_golden_registers));
1075		break;
1076	case CHIP_CAICOS:
1077		radeon_program_register_sequence(rdev,
1078						 caicos_golden_registers,
1079						 (const u32)ARRAY_SIZE(caicos_golden_registers));
1080		break;
1081	default:
1082		break;
1083	}
1084}
1085
1086/**
1087 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1088 *
1089 * @rdev: radeon_device pointer
1090 * @reg: register offset in bytes
1091 * @val: register value
1092 *
1093 * Returns 0 for success or -EINVAL for an invalid register
1094 *
1095 */
1096int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1097					u32 reg, u32 *val)
1098{
1099	switch (reg) {
1100	case GRBM_STATUS:
1101	case GRBM_STATUS_SE0:
1102	case GRBM_STATUS_SE1:
1103	case SRBM_STATUS:
1104	case SRBM_STATUS2:
1105	case DMA_STATUS_REG:
1106	case UVD_STATUS:
1107		*val = RREG32(reg);
1108		return 0;
1109	default:
1110		return -EINVAL;
1111	}
1112}
1113
1114void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1115			     unsigned *bankh, unsigned *mtaspect,
1116			     unsigned *tile_split)
1117{
1118	*bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1119	*bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1120	*mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1121	*tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1122	switch (*bankw) {
1123	default:
1124	case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1125	case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1126	case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1127	case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1128	}
1129	switch (*bankh) {
1130	default:
1131	case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1132	case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1133	case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1134	case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1135	}
1136	switch (*mtaspect) {
1137	default:
1138	case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1139	case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1140	case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1141	case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1142	}
1143}
1144
1145static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1146			      u32 cntl_reg, u32 status_reg)
1147{
1148	int r, i;
1149	struct atom_clock_dividers dividers;
1150
1151	r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1152					   clock, false, &dividers);
1153	if (r)
1154		return r;
1155
1156	WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1157
1158	for (i = 0; i < 100; i++) {
1159		if (RREG32(status_reg) & DCLK_STATUS)
1160			break;
1161		mdelay(10);
1162	}
1163	if (i == 100)
1164		return -ETIMEDOUT;
1165
1166	return 0;
1167}
1168
1169int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1170{
1171	int r = 0;
1172	u32 cg_scratch = RREG32(CG_SCRATCH1);
1173
1174	r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1175	if (r)
1176		goto done;
1177	cg_scratch &= 0xffff0000;
1178	cg_scratch |= vclk / 100; /* Mhz */
1179
1180	r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1181	if (r)
1182		goto done;
1183	cg_scratch &= 0x0000ffff;
1184	cg_scratch |= (dclk / 100) << 16; /* Mhz */
1185
1186done:
1187	WREG32(CG_SCRATCH1, cg_scratch);
1188
1189	return r;
1190}
1191
1192int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1193{
1194	/* start off with something large */
1195	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1196	int r;
1197
1198	/* bypass vclk and dclk with bclk */
1199	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1200		VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1201		~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1202
1203	/* put PLL in bypass mode */
1204	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1205
1206	if (!vclk || !dclk) {
1207		/* keep the Bypass mode, put PLL to sleep */
1208		WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1209		return 0;
1210	}
1211
1212	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1213					  16384, 0x03FFFFFF, 0, 128, 5,
1214					  &fb_div, &vclk_div, &dclk_div);
1215	if (r)
1216		return r;
1217
1218	/* set VCO_MODE to 1 */
1219	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1220
1221	/* toggle UPLL_SLEEP to 1 then back to 0 */
1222	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1223	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1224
1225	/* deassert UPLL_RESET */
1226	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1227
1228	mdelay(1);
1229
1230	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1231	if (r)
1232		return r;
1233
1234	/* assert UPLL_RESET again */
1235	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1236
1237	/* disable spread spectrum. */
1238	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1239
1240	/* set feedback divider */
1241	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1242
1243	/* set ref divider to 0 */
1244	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1245
1246	if (fb_div < 307200)
1247		WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1248	else
1249		WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1250
1251	/* set PDIV_A and PDIV_B */
1252	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1253		UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1254		~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1255
1256	/* give the PLL some time to settle */
1257	mdelay(15);
1258
1259	/* deassert PLL_RESET */
1260	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1261
1262	mdelay(15);
1263
1264	/* switch from bypass mode to normal mode */
1265	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1266
1267	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1268	if (r)
1269		return r;
1270
1271	/* switch VCLK and DCLK selection */
1272	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1273		VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1274		~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1275
1276	mdelay(100);
1277
1278	return 0;
1279}
1280
1281void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1282{
1283	int readrq;
1284	u16 v;
1285
1286	readrq = pcie_get_readrq(rdev->pdev);
1287	v = ffs(readrq) - 8;
1288	/* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1289	 * to avoid hangs or perfomance issues
1290	 */
1291	if ((v == 0) || (v == 6) || (v == 7))
1292		pcie_set_readrq(rdev->pdev, 512);
1293}
1294
1295void dce4_program_fmt(struct drm_encoder *encoder)
1296{
1297	struct drm_device *dev = encoder->dev;
1298	struct radeon_device *rdev = dev->dev_private;
1299	struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1300	struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1301	struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1302	int bpc = 0;
1303	u32 tmp = 0;
1304	enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1305
1306	if (connector) {
1307		struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1308		bpc = radeon_get_monitor_bpc(connector);
1309		dither = radeon_connector->dither;
1310	}
1311
1312	/* LVDS/eDP FMT is set up by atom */
1313	if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1314		return;
1315
1316	/* not needed for analog */
1317	if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1318	    (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1319		return;
1320
1321	if (bpc == 0)
1322		return;
1323
1324	switch (bpc) {
1325	case 6:
1326		if (dither == RADEON_FMT_DITHER_ENABLE)
1327			/* XXX sort out optimal dither settings */
1328			tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1329				FMT_SPATIAL_DITHER_EN);
1330		else
1331			tmp |= FMT_TRUNCATE_EN;
1332		break;
1333	case 8:
1334		if (dither == RADEON_FMT_DITHER_ENABLE)
1335			/* XXX sort out optimal dither settings */
1336			tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1337				FMT_RGB_RANDOM_ENABLE |
1338				FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1339		else
1340			tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1341		break;
1342	case 10:
1343	default:
1344		/* not needed */
1345		break;
1346	}
1347
1348	WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1349}
1350
1351static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1352{
1353	if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1354		return true;
1355	else
1356		return false;
1357}
1358
1359static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1360{
1361	u32 pos1, pos2;
1362
1363	pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1364	pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1365
1366	if (pos1 != pos2)
1367		return true;
1368	else
1369		return false;
1370}
1371
1372/**
1373 * dce4_wait_for_vblank - vblank wait asic callback.
1374 *
1375 * @rdev: radeon_device pointer
1376 * @crtc: crtc to wait for vblank on
1377 *
1378 * Wait for vblank on the requested crtc (evergreen+).
1379 */
1380void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1381{
1382	unsigned i = 0;
1383
1384	if (crtc >= rdev->num_crtc)
1385		return;
1386
1387	if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1388		return;
1389
1390	/* depending on when we hit vblank, we may be close to active; if so,
1391	 * wait for another frame.
1392	 */
1393	while (dce4_is_in_vblank(rdev, crtc)) {
1394		if (i++ % 100 == 0) {
1395			if (!dce4_is_counter_moving(rdev, crtc))
1396				break;
1397		}
1398	}
1399
1400	while (!dce4_is_in_vblank(rdev, crtc)) {
1401		if (i++ % 100 == 0) {
1402			if (!dce4_is_counter_moving(rdev, crtc))
1403				break;
1404		}
1405	}
1406}
1407
1408/**
1409 * evergreen_page_flip - pageflip callback.
1410 *
1411 * @rdev: radeon_device pointer
1412 * @crtc_id: crtc to cleanup pageflip on
1413 * @crtc_base: new address of the crtc (GPU MC address)
1414 *
1415 * Triggers the actual pageflip by updating the primary
1416 * surface base address (evergreen+).
1417 */
1418void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1419			 bool async)
1420{
1421	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1422
1423	/* update the scanout addresses */
1424	WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1425	       async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1426	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1427	       upper_32_bits(crtc_base));
1428	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1429	       (u32)crtc_base);
1430	/* post the write */
1431	RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1432}
1433
1434/**
1435 * evergreen_page_flip_pending - check if page flip is still pending
1436 *
1437 * @rdev: radeon_device pointer
1438 * @crtc_id: crtc to check
1439 *
1440 * Returns the current update pending status.
1441 */
1442bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1443{
1444	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1445
1446	/* Return current update_pending status: */
1447	return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1448		EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1449}
1450
1451/* get temperature in millidegrees */
1452int evergreen_get_temp(struct radeon_device *rdev)
1453{
1454	u32 temp, toffset;
1455	int actual_temp = 0;
1456
1457	if (rdev->family == CHIP_JUNIPER) {
1458		toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1459			TOFFSET_SHIFT;
1460		temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1461			TS0_ADC_DOUT_SHIFT;
1462
1463		if (toffset & 0x100)
1464			actual_temp = temp / 2 - (0x200 - toffset);
1465		else
1466			actual_temp = temp / 2 + toffset;
1467
1468		actual_temp = actual_temp * 1000;
1469
1470	} else {
1471		temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1472			ASIC_T_SHIFT;
1473
1474		if (temp & 0x400)
1475			actual_temp = -256;
1476		else if (temp & 0x200)
1477			actual_temp = 255;
1478		else if (temp & 0x100) {
1479			actual_temp = temp & 0x1ff;
1480			actual_temp |= ~0x1ff;
1481		} else
1482			actual_temp = temp & 0xff;
1483
1484		actual_temp = (actual_temp * 1000) / 2;
1485	}
1486
1487	return actual_temp;
1488}
1489
1490int sumo_get_temp(struct radeon_device *rdev)
1491{
1492	u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1493	int actual_temp = temp - 49;
1494
1495	return actual_temp * 1000;
1496}
1497
1498/**
1499 * sumo_pm_init_profile - Initialize power profiles callback.
1500 *
1501 * @rdev: radeon_device pointer
1502 *
1503 * Initialize the power states used in profile mode
1504 * (sumo, trinity, SI).
1505 * Used for profile mode only.
1506 */
1507void sumo_pm_init_profile(struct radeon_device *rdev)
1508{
1509	int idx;
1510
1511	/* default */
1512	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1513	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1514	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1515	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1516
1517	/* low,mid sh/mh */
1518	if (rdev->flags & RADEON_IS_MOBILITY)
1519		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1520	else
1521		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1522
1523	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1524	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1525	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1526	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1527
1528	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1529	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1530	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1531	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1532
1533	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1534	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1535	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1536	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1537
1538	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1539	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1540	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1541	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1542
1543	/* high sh/mh */
1544	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1545	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1546	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1547	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1548	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1549		rdev->pm.power_state[idx].num_clock_modes - 1;
1550
1551	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1552	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1553	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1554	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1555		rdev->pm.power_state[idx].num_clock_modes - 1;
1556}
1557
1558/**
1559 * btc_pm_init_profile - Initialize power profiles callback.
1560 *
1561 * @rdev: radeon_device pointer
1562 *
1563 * Initialize the power states used in profile mode
1564 * (BTC, cayman).
1565 * Used for profile mode only.
1566 */
1567void btc_pm_init_profile(struct radeon_device *rdev)
1568{
1569	int idx;
1570
1571	/* default */
1572	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1573	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1574	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1575	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1576	/* starting with BTC, there is one state that is used for both
1577	 * MH and SH.  Difference is that we always use the high clock index for
1578	 * mclk.
1579	 */
1580	if (rdev->flags & RADEON_IS_MOBILITY)
1581		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1582	else
1583		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1584	/* low sh */
1585	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1586	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1587	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1588	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1589	/* mid sh */
1590	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1591	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1592	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1593	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1594	/* high sh */
1595	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1596	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1597	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1598	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1599	/* low mh */
1600	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1601	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1602	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1603	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1604	/* mid mh */
1605	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1606	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1607	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1608	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1609	/* high mh */
1610	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1611	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1612	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1613	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1614}
1615
1616/**
1617 * evergreen_pm_misc - set additional pm hw parameters callback.
1618 *
1619 * @rdev: radeon_device pointer
1620 *
1621 * Set non-clock parameters associated with a power state
1622 * (voltage, etc.) (evergreen+).
1623 */
1624void evergreen_pm_misc(struct radeon_device *rdev)
1625{
1626	int req_ps_idx = rdev->pm.requested_power_state_index;
1627	int req_cm_idx = rdev->pm.requested_clock_mode_index;
1628	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1629	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1630
1631	if (voltage->type == VOLTAGE_SW) {
1632		/* 0xff0x are flags rather then an actual voltage */
1633		if ((voltage->voltage & 0xff00) == 0xff00)
1634			return;
1635		if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1636			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1637			rdev->pm.current_vddc = voltage->voltage;
1638			DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1639		}
1640
1641		/* starting with BTC, there is one state that is used for both
1642		 * MH and SH.  Difference is that we always use the high clock index for
1643		 * mclk and vddci.
1644		 */
1645		if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1646		    (rdev->family >= CHIP_BARTS) &&
1647		    rdev->pm.active_crtc_count &&
1648		    ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1649		     (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1650			voltage = &rdev->pm.power_state[req_ps_idx].
1651				clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1652
1653		/* 0xff0x are flags rather then an actual voltage */
1654		if ((voltage->vddci & 0xff00) == 0xff00)
1655			return;
1656		if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1657			radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1658			rdev->pm.current_vddci = voltage->vddci;
1659			DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1660		}
1661	}
1662}
1663
1664/**
1665 * evergreen_pm_prepare - pre-power state change callback.
1666 *
1667 * @rdev: radeon_device pointer
1668 *
1669 * Prepare for a power state change (evergreen+).
1670 */
1671void evergreen_pm_prepare(struct radeon_device *rdev)
1672{
1673	struct drm_device *ddev = rdev->ddev;
1674	struct drm_crtc *crtc;
1675	struct radeon_crtc *radeon_crtc;
1676	u32 tmp;
1677
1678	/* disable any active CRTCs */
1679	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1680		radeon_crtc = to_radeon_crtc(crtc);
1681		if (radeon_crtc->enabled) {
1682			tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1683			tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1684			WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1685		}
1686	}
1687}
1688
1689/**
1690 * evergreen_pm_finish - post-power state change callback.
1691 *
1692 * @rdev: radeon_device pointer
1693 *
1694 * Clean up after a power state change (evergreen+).
1695 */
1696void evergreen_pm_finish(struct radeon_device *rdev)
1697{
1698	struct drm_device *ddev = rdev->ddev;
1699	struct drm_crtc *crtc;
1700	struct radeon_crtc *radeon_crtc;
1701	u32 tmp;
1702
1703	/* enable any active CRTCs */
1704	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1705		radeon_crtc = to_radeon_crtc(crtc);
1706		if (radeon_crtc->enabled) {
1707			tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1708			tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1709			WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1710		}
1711	}
1712}
1713
1714/**
1715 * evergreen_hpd_sense - hpd sense callback.
1716 *
1717 * @rdev: radeon_device pointer
1718 * @hpd: hpd (hotplug detect) pin
1719 *
1720 * Checks if a digital monitor is connected (evergreen+).
1721 * Returns true if connected, false if not connected.
1722 */
1723bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1724{
1725	if (hpd == RADEON_HPD_NONE)
1726		return false;
1727
1728	return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1729}
1730
1731/**
1732 * evergreen_hpd_set_polarity - hpd set polarity callback.
1733 *
1734 * @rdev: radeon_device pointer
1735 * @hpd: hpd (hotplug detect) pin
1736 *
1737 * Set the polarity of the hpd pin (evergreen+).
1738 */
1739void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1740				enum radeon_hpd_id hpd)
1741{
1742	bool connected = evergreen_hpd_sense(rdev, hpd);
1743
1744	if (hpd == RADEON_HPD_NONE)
1745		return;
1746
1747	if (connected)
1748		WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1749	else
1750		WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1751}
1752
1753/**
1754 * evergreen_hpd_init - hpd setup callback.
1755 *
1756 * @rdev: radeon_device pointer
1757 *
1758 * Setup the hpd pins used by the card (evergreen+).
1759 * Enable the pin, set the polarity, and enable the hpd interrupts.
1760 */
1761void evergreen_hpd_init(struct radeon_device *rdev)
1762{
1763	struct drm_device *dev = rdev->ddev;
1764	struct drm_connector *connector;
1765	unsigned enabled = 0;
1766	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1767		DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1768
1769	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1770		enum radeon_hpd_id hpd =
1771			to_radeon_connector(connector)->hpd.hpd;
1772
1773		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1774		    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1775			/* don't try to enable hpd on eDP or LVDS avoid breaking the
1776			 * aux dp channel on imac and help (but not completely fix)
1777			 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1778			 * also avoid interrupt storms during dpms.
1779			 */
1780			continue;
1781		}
1782
1783		if (hpd == RADEON_HPD_NONE)
1784			continue;
1785
1786		WREG32(DC_HPDx_CONTROL(hpd), tmp);
1787		enabled |= 1 << hpd;
1788
1789		radeon_hpd_set_polarity(rdev, hpd);
1790	}
1791	radeon_irq_kms_enable_hpd(rdev, enabled);
1792}
1793
1794/**
1795 * evergreen_hpd_fini - hpd tear down callback.
1796 *
1797 * @rdev: radeon_device pointer
1798 *
1799 * Tear down the hpd pins used by the card (evergreen+).
1800 * Disable the hpd interrupts.
1801 */
1802void evergreen_hpd_fini(struct radeon_device *rdev)
1803{
1804	struct drm_device *dev = rdev->ddev;
1805	struct drm_connector *connector;
1806	unsigned disabled = 0;
1807
1808	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1809		enum radeon_hpd_id hpd =
1810			to_radeon_connector(connector)->hpd.hpd;
1811
1812		if (hpd == RADEON_HPD_NONE)
1813			continue;
1814
1815		WREG32(DC_HPDx_CONTROL(hpd), 0);
1816		disabled |= 1 << hpd;
1817	}
1818	radeon_irq_kms_disable_hpd(rdev, disabled);
1819}
1820
1821/* watermark setup */
1822
1823static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1824					struct radeon_crtc *radeon_crtc,
1825					struct drm_display_mode *mode,
1826					struct drm_display_mode *other_mode)
1827{
1828	u32 tmp, buffer_alloc, i;
1829	u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1830	/*
1831	 * Line Buffer Setup
1832	 * There are 3 line buffers, each one shared by 2 display controllers.
1833	 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1834	 * the display controllers.  The paritioning is done via one of four
1835	 * preset allocations specified in bits 2:0:
1836	 * first display controller
1837	 *  0 - first half of lb (3840 * 2)
1838	 *  1 - first 3/4 of lb (5760 * 2)
1839	 *  2 - whole lb (7680 * 2), other crtc must be disabled
1840	 *  3 - first 1/4 of lb (1920 * 2)
1841	 * second display controller
1842	 *  4 - second half of lb (3840 * 2)
1843	 *  5 - second 3/4 of lb (5760 * 2)
1844	 *  6 - whole lb (7680 * 2), other crtc must be disabled
1845	 *  7 - last 1/4 of lb (1920 * 2)
1846	 */
1847	/* this can get tricky if we have two large displays on a paired group
1848	 * of crtcs.  Ideally for multiple large displays we'd assign them to
1849	 * non-linked crtcs for maximum line buffer allocation.
1850	 */
1851	if (radeon_crtc->base.enabled && mode) {
1852		if (other_mode) {
1853			tmp = 0; /* 1/2 */
1854			buffer_alloc = 1;
1855		} else {
1856			tmp = 2; /* whole */
1857			buffer_alloc = 2;
1858		}
1859	} else {
1860		tmp = 0;
1861		buffer_alloc = 0;
1862	}
1863
1864	/* second controller of the pair uses second half of the lb */
1865	if (radeon_crtc->crtc_id % 2)
1866		tmp += 4;
1867	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1868
1869	if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1870		WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1871		       DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1872		for (i = 0; i < rdev->usec_timeout; i++) {
1873			if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1874			    DMIF_BUFFERS_ALLOCATED_COMPLETED)
1875				break;
1876			udelay(1);
1877		}
1878	}
1879
1880	if (radeon_crtc->base.enabled && mode) {
1881		switch (tmp) {
1882		case 0:
1883		case 4:
1884		default:
1885			if (ASIC_IS_DCE5(rdev))
1886				return 4096 * 2;
1887			else
1888				return 3840 * 2;
1889		case 1:
1890		case 5:
1891			if (ASIC_IS_DCE5(rdev))
1892				return 6144 * 2;
1893			else
1894				return 5760 * 2;
1895		case 2:
1896		case 6:
1897			if (ASIC_IS_DCE5(rdev))
1898				return 8192 * 2;
1899			else
1900				return 7680 * 2;
1901		case 3:
1902		case 7:
1903			if (ASIC_IS_DCE5(rdev))
1904				return 2048 * 2;
1905			else
1906				return 1920 * 2;
1907		}
1908	}
1909
1910	/* controller not enabled, so no lb used */
1911	return 0;
1912}
1913
1914u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1915{
1916	u32 tmp = RREG32(MC_SHARED_CHMAP);
1917
1918	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1919	case 0:
1920	default:
1921		return 1;
1922	case 1:
1923		return 2;
1924	case 2:
1925		return 4;
1926	case 3:
1927		return 8;
1928	}
1929}
1930
1931struct evergreen_wm_params {
1932	u32 dram_channels; /* number of dram channels */
1933	u32 yclk;          /* bandwidth per dram data pin in kHz */
1934	u32 sclk;          /* engine clock in kHz */
1935	u32 disp_clk;      /* display clock in kHz */
1936	u32 src_width;     /* viewport width */
1937	u32 active_time;   /* active display time in ns */
1938	u32 blank_time;    /* blank time in ns */
1939	bool interlaced;    /* mode is interlaced */
1940	fixed20_12 vsc;    /* vertical scale ratio */
1941	u32 num_heads;     /* number of active crtcs */
1942	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1943	u32 lb_size;       /* line buffer allocated to pipe */
1944	u32 vtaps;         /* vertical scaler taps */
1945};
1946
1947static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1948{
1949	/* Calculate DRAM Bandwidth and the part allocated to display. */
1950	fixed20_12 dram_efficiency; /* 0.7 */
1951	fixed20_12 yclk, dram_channels, bandwidth;
1952	fixed20_12 a;
1953
1954	a.full = dfixed_const(1000);
1955	yclk.full = dfixed_const(wm->yclk);
1956	yclk.full = dfixed_div(yclk, a);
1957	dram_channels.full = dfixed_const(wm->dram_channels * 4);
1958	a.full = dfixed_const(10);
1959	dram_efficiency.full = dfixed_const(7);
1960	dram_efficiency.full = dfixed_div(dram_efficiency, a);
1961	bandwidth.full = dfixed_mul(dram_channels, yclk);
1962	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1963
1964	return dfixed_trunc(bandwidth);
1965}
1966
1967static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1968{
1969	/* Calculate DRAM Bandwidth and the part allocated to display. */
1970	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1971	fixed20_12 yclk, dram_channels, bandwidth;
1972	fixed20_12 a;
1973
1974	a.full = dfixed_const(1000);
1975	yclk.full = dfixed_const(wm->yclk);
1976	yclk.full = dfixed_div(yclk, a);
1977	dram_channels.full = dfixed_const(wm->dram_channels * 4);
1978	a.full = dfixed_const(10);
1979	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1980	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1981	bandwidth.full = dfixed_mul(dram_channels, yclk);
1982	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1983
1984	return dfixed_trunc(bandwidth);
1985}
1986
1987static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1988{
1989	/* Calculate the display Data return Bandwidth */
1990	fixed20_12 return_efficiency; /* 0.8 */
1991	fixed20_12 sclk, bandwidth;
1992	fixed20_12 a;
1993
1994	a.full = dfixed_const(1000);
1995	sclk.full = dfixed_const(wm->sclk);
1996	sclk.full = dfixed_div(sclk, a);
1997	a.full = dfixed_const(10);
1998	return_efficiency.full = dfixed_const(8);
1999	return_efficiency.full = dfixed_div(return_efficiency, a);
2000	a.full = dfixed_const(32);
2001	bandwidth.full = dfixed_mul(a, sclk);
2002	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2003
2004	return dfixed_trunc(bandwidth);
2005}
2006
2007static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2008{
2009	/* Calculate the DMIF Request Bandwidth */
2010	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2011	fixed20_12 disp_clk, bandwidth;
2012	fixed20_12 a;
2013
2014	a.full = dfixed_const(1000);
2015	disp_clk.full = dfixed_const(wm->disp_clk);
2016	disp_clk.full = dfixed_div(disp_clk, a);
2017	a.full = dfixed_const(10);
2018	disp_clk_request_efficiency.full = dfixed_const(8);
2019	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2020	a.full = dfixed_const(32);
2021	bandwidth.full = dfixed_mul(a, disp_clk);
2022	bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2023
2024	return dfixed_trunc(bandwidth);
2025}
2026
2027static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2028{
2029	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2030	u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2031	u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2032	u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2033
2034	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2035}
2036
2037static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2038{
2039	/* Calculate the display mode Average Bandwidth
2040	 * DisplayMode should contain the source and destination dimensions,
2041	 * timing, etc.
2042	 */
2043	fixed20_12 bpp;
2044	fixed20_12 line_time;
2045	fixed20_12 src_width;
2046	fixed20_12 bandwidth;
2047	fixed20_12 a;
2048
2049	a.full = dfixed_const(1000);
2050	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2051	line_time.full = dfixed_div(line_time, a);
2052	bpp.full = dfixed_const(wm->bytes_per_pixel);
2053	src_width.full = dfixed_const(wm->src_width);
2054	bandwidth.full = dfixed_mul(src_width, bpp);
2055	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2056	bandwidth.full = dfixed_div(bandwidth, line_time);
2057
2058	return dfixed_trunc(bandwidth);
2059}
2060
2061static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2062{
2063	/* First calcualte the latency in ns */
2064	u32 mc_latency = 2000; /* 2000 ns. */
2065	u32 available_bandwidth = evergreen_available_bandwidth(wm);
2066	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2067	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2068	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2069	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2070		(wm->num_heads * cursor_line_pair_return_time);
2071	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2072	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2073	fixed20_12 a, b, c;
2074
2075	if (wm->num_heads == 0)
2076		return 0;
2077
2078	a.full = dfixed_const(2);
2079	b.full = dfixed_const(1);
2080	if ((wm->vsc.full > a.full) ||
2081	    ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2082	    (wm->vtaps >= 5) ||
2083	    ((wm->vsc.full >= a.full) && wm->interlaced))
2084		max_src_lines_per_dst_line = 4;
2085	else
2086		max_src_lines_per_dst_line = 2;
2087
2088	a.full = dfixed_const(available_bandwidth);
2089	b.full = dfixed_const(wm->num_heads);
2090	a.full = dfixed_div(a, b);
2091
2092	lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2093
2094	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2095	b.full = dfixed_const(1000);
2096	c.full = dfixed_const(lb_fill_bw);
2097	b.full = dfixed_div(c, b);
2098	a.full = dfixed_div(a, b);
2099	line_fill_time = dfixed_trunc(a);
2100
2101	if (line_fill_time < wm->active_time)
2102		return latency;
2103	else
2104		return latency + (line_fill_time - wm->active_time);
2105
2106}
2107
2108static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2109{
2110	if (evergreen_average_bandwidth(wm) <=
2111	    (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2112		return true;
2113	else
2114		return false;
2115};
2116
2117static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2118{
2119	if (evergreen_average_bandwidth(wm) <=
2120	    (evergreen_available_bandwidth(wm) / wm->num_heads))
2121		return true;
2122	else
2123		return false;
2124};
2125
2126static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2127{
2128	u32 lb_partitions = wm->lb_size / wm->src_width;
2129	u32 line_time = wm->active_time + wm->blank_time;
2130	u32 latency_tolerant_lines;
2131	u32 latency_hiding;
2132	fixed20_12 a;
2133
2134	a.full = dfixed_const(1);
2135	if (wm->vsc.full > a.full)
2136		latency_tolerant_lines = 1;
2137	else {
2138		if (lb_partitions <= (wm->vtaps + 1))
2139			latency_tolerant_lines = 1;
2140		else
2141			latency_tolerant_lines = 2;
2142	}
2143
2144	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2145
2146	if (evergreen_latency_watermark(wm) <= latency_hiding)
2147		return true;
2148	else
2149		return false;
2150}
2151
2152static void evergreen_program_watermarks(struct radeon_device *rdev,
2153					 struct radeon_crtc *radeon_crtc,
2154					 u32 lb_size, u32 num_heads)
2155{
2156	struct drm_display_mode *mode = &radeon_crtc->base.mode;
2157	struct evergreen_wm_params wm_low, wm_high;
2158	u32 dram_channels;
2159	u32 active_time;
2160	u32 line_time = 0;
2161	u32 latency_watermark_a = 0, latency_watermark_b = 0;
2162	u32 priority_a_mark = 0, priority_b_mark = 0;
2163	u32 priority_a_cnt = PRIORITY_OFF;
2164	u32 priority_b_cnt = PRIORITY_OFF;
2165	u32 pipe_offset = radeon_crtc->crtc_id * 16;
2166	u32 tmp, arb_control3;
2167	fixed20_12 a, b, c;
2168
2169	if (radeon_crtc->base.enabled && num_heads && mode) {
2170		active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2171					    (u32)mode->clock);
2172		line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2173					  (u32)mode->clock);
2174		line_time = min(line_time, (u32)65535);
2175		priority_a_cnt = 0;
2176		priority_b_cnt = 0;
2177		dram_channels = evergreen_get_number_of_dram_channels(rdev);
2178
2179		/* watermark for high clocks */
2180		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2181			wm_high.yclk =
2182				radeon_dpm_get_mclk(rdev, false) * 10;
2183			wm_high.sclk =
2184				radeon_dpm_get_sclk(rdev, false) * 10;
2185		} else {
2186			wm_high.yclk = rdev->pm.current_mclk * 10;
2187			wm_high.sclk = rdev->pm.current_sclk * 10;
2188		}
2189
2190		wm_high.disp_clk = mode->clock;
2191		wm_high.src_width = mode->crtc_hdisplay;
2192		wm_high.active_time = active_time;
2193		wm_high.blank_time = line_time - wm_high.active_time;
2194		wm_high.interlaced = false;
2195		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2196			wm_high.interlaced = true;
2197		wm_high.vsc = radeon_crtc->vsc;
2198		wm_high.vtaps = 1;
2199		if (radeon_crtc->rmx_type != RMX_OFF)
2200			wm_high.vtaps = 2;
2201		wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2202		wm_high.lb_size = lb_size;
2203		wm_high.dram_channels = dram_channels;
2204		wm_high.num_heads = num_heads;
2205
2206		/* watermark for low clocks */
2207		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2208			wm_low.yclk =
2209				radeon_dpm_get_mclk(rdev, true) * 10;
2210			wm_low.sclk =
2211				radeon_dpm_get_sclk(rdev, true) * 10;
2212		} else {
2213			wm_low.yclk = rdev->pm.current_mclk * 10;
2214			wm_low.sclk = rdev->pm.current_sclk * 10;
2215		}
2216
2217		wm_low.disp_clk = mode->clock;
2218		wm_low.src_width = mode->crtc_hdisplay;
2219		wm_low.active_time = active_time;
2220		wm_low.blank_time = line_time - wm_low.active_time;
2221		wm_low.interlaced = false;
2222		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2223			wm_low.interlaced = true;
2224		wm_low.vsc = radeon_crtc->vsc;
2225		wm_low.vtaps = 1;
2226		if (radeon_crtc->rmx_type != RMX_OFF)
2227			wm_low.vtaps = 2;
2228		wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2229		wm_low.lb_size = lb_size;
2230		wm_low.dram_channels = dram_channels;
2231		wm_low.num_heads = num_heads;
2232
2233		/* set for high clocks */
2234		latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2235		/* set for low clocks */
2236		latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2237
2238		/* possibly force display priority to high */
2239		/* should really do this at mode validation time... */
2240		if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2241		    !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2242		    !evergreen_check_latency_hiding(&wm_high) ||
2243		    (rdev->disp_priority == 2)) {
2244			DRM_DEBUG_KMS("force priority a to high\n");
2245			priority_a_cnt |= PRIORITY_ALWAYS_ON;
2246		}
2247		if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2248		    !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2249		    !evergreen_check_latency_hiding(&wm_low) ||
2250		    (rdev->disp_priority == 2)) {
2251			DRM_DEBUG_KMS("force priority b to high\n");
2252			priority_b_cnt |= PRIORITY_ALWAYS_ON;
2253		}
2254
2255		a.full = dfixed_const(1000);
2256		b.full = dfixed_const(mode->clock);
2257		b.full = dfixed_div(b, a);
2258		c.full = dfixed_const(latency_watermark_a);
2259		c.full = dfixed_mul(c, b);
2260		c.full = dfixed_mul(c, radeon_crtc->hsc);
2261		c.full = dfixed_div(c, a);
2262		a.full = dfixed_const(16);
2263		c.full = dfixed_div(c, a);
2264		priority_a_mark = dfixed_trunc(c);
2265		priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2266
2267		a.full = dfixed_const(1000);
2268		b.full = dfixed_const(mode->clock);
2269		b.full = dfixed_div(b, a);
2270		c.full = dfixed_const(latency_watermark_b);
2271		c.full = dfixed_mul(c, b);
2272		c.full = dfixed_mul(c, radeon_crtc->hsc);
2273		c.full = dfixed_div(c, a);
2274		a.full = dfixed_const(16);
2275		c.full = dfixed_div(c, a);
2276		priority_b_mark = dfixed_trunc(c);
2277		priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2278
2279		/* Save number of lines the linebuffer leads before the scanout */
2280		radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2281	}
2282
2283	/* select wm A */
2284	arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2285	tmp = arb_control3;
2286	tmp &= ~LATENCY_WATERMARK_MASK(3);
2287	tmp |= LATENCY_WATERMARK_MASK(1);
2288	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2289	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2290	       (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2291		LATENCY_HIGH_WATERMARK(line_time)));
2292	/* select wm B */
2293	tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2294	tmp &= ~LATENCY_WATERMARK_MASK(3);
2295	tmp |= LATENCY_WATERMARK_MASK(2);
2296	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2297	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2298	       (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2299		LATENCY_HIGH_WATERMARK(line_time)));
2300	/* restore original selection */
2301	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2302
2303	/* write the priority marks */
2304	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2305	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2306
2307	/* save values for DPM */
2308	radeon_crtc->line_time = line_time;
2309	radeon_crtc->wm_high = latency_watermark_a;
2310	radeon_crtc->wm_low = latency_watermark_b;
2311}
2312
2313/**
2314 * evergreen_bandwidth_update - update display watermarks callback.
2315 *
2316 * @rdev: radeon_device pointer
2317 *
2318 * Update the display watermarks based on the requested mode(s)
2319 * (evergreen+).
2320 */
2321void evergreen_bandwidth_update(struct radeon_device *rdev)
2322{
2323	struct drm_display_mode *mode0 = NULL;
2324	struct drm_display_mode *mode1 = NULL;
2325	u32 num_heads = 0, lb_size;
2326	int i;
2327
2328	if (!rdev->mode_info.mode_config_initialized)
2329		return;
2330
2331	radeon_update_display_priority(rdev);
2332
2333	for (i = 0; i < rdev->num_crtc; i++) {
2334		if (rdev->mode_info.crtcs[i]->base.enabled)
2335			num_heads++;
2336	}
2337	for (i = 0; i < rdev->num_crtc; i += 2) {
2338		mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2339		mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2340		lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2341		evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2342		lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2343		evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2344	}
2345}
2346
2347/**
2348 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2349 *
2350 * @rdev: radeon_device pointer
2351 *
2352 * Wait for the MC (memory controller) to be idle.
2353 * (evergreen+).
2354 * Returns 0 if the MC is idle, -1 if not.
2355 */
2356int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2357{
2358	unsigned i;
2359	u32 tmp;
2360
2361	for (i = 0; i < rdev->usec_timeout; i++) {
2362		/* read MC_STATUS */
2363		tmp = RREG32(SRBM_STATUS) & 0x1F00;
2364		if (!tmp)
2365			return 0;
2366		udelay(1);
2367	}
2368	return -1;
2369}
2370
2371/*
2372 * GART
2373 */
2374void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2375{
2376	unsigned i;
2377	u32 tmp;
2378
2379	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2380
2381	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2382	for (i = 0; i < rdev->usec_timeout; i++) {
2383		/* read MC_STATUS */
2384		tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2385		tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2386		if (tmp == 2) {
2387			pr_warn("[drm] r600 flush TLB failed\n");
2388			return;
2389		}
2390		if (tmp) {
2391			return;
2392		}
2393		udelay(1);
2394	}
2395}
2396
2397static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2398{
2399	u32 tmp;
2400	int r;
2401
2402	if (rdev->gart.robj == NULL) {
2403		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2404		return -EINVAL;
2405	}
2406	r = radeon_gart_table_vram_pin(rdev);
2407	if (r)
2408		return r;
2409	/* Setup L2 cache */
2410	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2411				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2412				EFFECTIVE_L2_QUEUE_SIZE(7));
2413	WREG32(VM_L2_CNTL2, 0);
2414	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2415	/* Setup TLB control */
2416	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2417		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2418		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2419		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2420	if (rdev->flags & RADEON_IS_IGP) {
2421		WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2422		WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2423		WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2424	} else {
2425		WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2426		WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2427		WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2428		if ((rdev->family == CHIP_JUNIPER) ||
2429		    (rdev->family == CHIP_CYPRESS) ||
2430		    (rdev->family == CHIP_HEMLOCK) ||
2431		    (rdev->family == CHIP_BARTS))
2432			WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2433	}
2434	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2435	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2436	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2437	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2438	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2439	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2440	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2441	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2442				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2443	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2444			(u32)(rdev->dummy_page.addr >> 12));
2445	WREG32(VM_CONTEXT1_CNTL, 0);
2446
2447	evergreen_pcie_gart_tlb_flush(rdev);
2448	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2449		 (unsigned)(rdev->mc.gtt_size >> 20),
2450		 (unsigned long long)rdev->gart.table_addr);
2451	rdev->gart.ready = true;
2452	return 0;
2453}
2454
2455static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2456{
2457	u32 tmp;
2458
2459	/* Disable all tables */
2460	WREG32(VM_CONTEXT0_CNTL, 0);
2461	WREG32(VM_CONTEXT1_CNTL, 0);
2462
2463	/* Setup L2 cache */
2464	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2465				EFFECTIVE_L2_QUEUE_SIZE(7));
2466	WREG32(VM_L2_CNTL2, 0);
2467	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2468	/* Setup TLB control */
2469	tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2470	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2471	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2472	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2473	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2474	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2475	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2476	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2477	radeon_gart_table_vram_unpin(rdev);
2478}
2479
2480static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2481{
2482	evergreen_pcie_gart_disable(rdev);
2483	radeon_gart_table_vram_free(rdev);
2484	radeon_gart_fini(rdev);
2485}
2486
2487
2488static void evergreen_agp_enable(struct radeon_device *rdev)
2489{
2490	u32 tmp;
2491
2492	/* Setup L2 cache */
2493	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2494				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2495				EFFECTIVE_L2_QUEUE_SIZE(7));
2496	WREG32(VM_L2_CNTL2, 0);
2497	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2498	/* Setup TLB control */
2499	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2500		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2501		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2502		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2503	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2504	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2505	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2506	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2507	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2508	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2509	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2510	WREG32(VM_CONTEXT0_CNTL, 0);
2511	WREG32(VM_CONTEXT1_CNTL, 0);
2512}
2513
2514static const unsigned ni_dig_offsets[] =
2515{
2516	NI_DIG0_REGISTER_OFFSET,
2517	NI_DIG1_REGISTER_OFFSET,
2518	NI_DIG2_REGISTER_OFFSET,
2519	NI_DIG3_REGISTER_OFFSET,
2520	NI_DIG4_REGISTER_OFFSET,
2521	NI_DIG5_REGISTER_OFFSET
2522};
2523
2524static const unsigned ni_tx_offsets[] =
2525{
2526	NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2527	NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2528	NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2529	NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2530	NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2531	NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2532};
2533
2534static const unsigned evergreen_dp_offsets[] =
2535{
2536	EVERGREEN_DP0_REGISTER_OFFSET,
2537	EVERGREEN_DP1_REGISTER_OFFSET,
2538	EVERGREEN_DP2_REGISTER_OFFSET,
2539	EVERGREEN_DP3_REGISTER_OFFSET,
2540	EVERGREEN_DP4_REGISTER_OFFSET,
2541	EVERGREEN_DP5_REGISTER_OFFSET
2542};
2543
2544static const unsigned evergreen_disp_int_status[] =
2545{
2546	DISP_INTERRUPT_STATUS,
2547	DISP_INTERRUPT_STATUS_CONTINUE,
2548	DISP_INTERRUPT_STATUS_CONTINUE2,
2549	DISP_INTERRUPT_STATUS_CONTINUE3,
2550	DISP_INTERRUPT_STATUS_CONTINUE4,
2551	DISP_INTERRUPT_STATUS_CONTINUE5
2552};
2553
2554/*
2555 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2556 * We go from crtc to connector and it is not relible  since it
2557 * should be an opposite direction .If crtc is enable then
2558 * find the dig_fe which selects this crtc and insure that it enable.
2559 * if such dig_fe is found then find dig_be which selects found dig_be and
2560 * insure that it enable and in DP_SST mode.
2561 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2562 * from dp symbols clocks .
2563 */
2564static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2565					       unsigned crtc_id, unsigned *ret_dig_fe)
2566{
2567	unsigned i;
2568	unsigned dig_fe;
2569	unsigned dig_be;
2570	unsigned dig_en_be;
2571	unsigned uniphy_pll;
2572	unsigned digs_fe_selected;
2573	unsigned dig_be_mode;
2574	unsigned dig_fe_mask;
2575	bool is_enabled = false;
2576	bool found_crtc = false;
2577
2578	/* loop through all running dig_fe to find selected crtc */
2579	for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2580		dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2581		if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2582		    crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2583			/* found running pipe */
2584			found_crtc = true;
2585			dig_fe_mask = 1 << i;
2586			dig_fe = i;
2587			break;
2588		}
2589	}
2590
2591	if (found_crtc) {
2592		/* loop through all running dig_be to find selected dig_fe */
2593		for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2594			dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2595			/* if dig_fe_selected by dig_be? */
2596			digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2597			dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2598			if (dig_fe_mask &  digs_fe_selected &&
2599			    /* if dig_be in sst mode? */
2600			    dig_be_mode == NI_DIG_BE_DPSST) {
2601				dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2602						   ni_dig_offsets[i]);
2603				uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2604						    ni_tx_offsets[i]);
2605				/* dig_be enable and tx is running */
2606				if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2607				    dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2608				    uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2609					is_enabled = true;
2610					*ret_dig_fe = dig_fe;
2611					break;
2612				}
2613			}
2614		}
2615	}
2616
2617	return is_enabled;
2618}
2619
2620/*
2621 * Blank dig when in dp sst mode
2622 * Dig ignores crtc timing
2623 */
2624static void evergreen_blank_dp_output(struct radeon_device *rdev,
2625				      unsigned dig_fe)
2626{
2627	unsigned stream_ctrl;
2628	unsigned fifo_ctrl;
2629	unsigned counter = 0;
2630
2631	if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2632		DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2633		return;
2634	}
2635
2636	stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2637			     evergreen_dp_offsets[dig_fe]);
2638	if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2639		DRM_ERROR("dig %d , should be enable\n", dig_fe);
2640		return;
2641	}
2642
2643	stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2644	WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2645	       evergreen_dp_offsets[dig_fe], stream_ctrl);
2646
2647	stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2648			     evergreen_dp_offsets[dig_fe]);
2649	while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2650		msleep(1);
2651		counter++;
2652		stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2653				     evergreen_dp_offsets[dig_fe]);
2654	}
2655	if (counter >= 32 )
2656		DRM_ERROR("counter exceeds %d\n", counter);
2657
2658	fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2659	fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2660	WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2661
2662}
2663
2664void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2665{
2666	u32 crtc_enabled, tmp, frame_count, blackout;
2667	int i, j;
2668	unsigned dig_fe;
2669
2670	if (!ASIC_IS_NODCE(rdev)) {
2671		save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2672		save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2673
2674		/* disable VGA render */
2675		WREG32(VGA_RENDER_CONTROL, 0);
2676	}
2677	/* blank the display controllers */
2678	for (i = 0; i < rdev->num_crtc; i++) {
2679		crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2680		if (crtc_enabled) {
2681			save->crtc_enabled[i] = true;
2682			if (ASIC_IS_DCE6(rdev)) {
2683				tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2684				if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2685					radeon_wait_for_vblank(rdev, i);
2686					WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2687					tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2688					WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2689					WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2690				}
2691			} else {
2692				tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2693				if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2694					radeon_wait_for_vblank(rdev, i);
2695					WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2696					tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2697					WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2698					WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2699				}
2700			}
2701			/* wait for the next frame */
2702			frame_count = radeon_get_vblank_counter(rdev, i);
2703			for (j = 0; j < rdev->usec_timeout; j++) {
2704				if (radeon_get_vblank_counter(rdev, i) != frame_count)
2705					break;
2706				udelay(1);
2707			}
2708			/*we should disable dig if it drives dp sst*/
2709			/*but we are in radeon_device_init and the topology is unknown*/
2710			/*and it is available after radeon_modeset_init*/
2711			/*the following method radeon_atom_encoder_dpms_dig*/
2712			/*does the job if we initialize it properly*/
2713			/*for now we do it this manually*/
2714			/**/
2715			if (ASIC_IS_DCE5(rdev) &&
2716			    evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2717				evergreen_blank_dp_output(rdev, dig_fe);
2718			/*we could remove 6 lines below*/
2719			/* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2720			WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2721			tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2722			tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2723			WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2724			WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2725			save->crtc_enabled[i] = false;
2726			/* ***** */
2727		} else {
2728			save->crtc_enabled[i] = false;
2729		}
2730	}
2731
2732	radeon_mc_wait_for_idle(rdev);
2733
2734	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2735	if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2736		/* Block CPU access */
2737		WREG32(BIF_FB_EN, 0);
2738		/* blackout the MC */
2739		blackout &= ~BLACKOUT_MODE_MASK;
2740		WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2741	}
2742	/* wait for the MC to settle */
2743	udelay(100);
2744
2745	/* lock double buffered regs */
2746	for (i = 0; i < rdev->num_crtc; i++) {
2747		if (save->crtc_enabled[i]) {
2748			tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2749			if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2750				tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2751				WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2752			}
2753			tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2754			if (!(tmp & 1)) {
2755				tmp |= 1;
2756				WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2757			}
2758		}
2759	}
2760}
2761
2762void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2763{
2764	u32 tmp, frame_count;
2765	int i, j;
2766
2767	/* update crtc base addresses */
2768	for (i = 0; i < rdev->num_crtc; i++) {
2769		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2770		       upper_32_bits(rdev->mc.vram_start));
2771		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2772		       upper_32_bits(rdev->mc.vram_start));
2773		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2774		       (u32)rdev->mc.vram_start);
2775		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2776		       (u32)rdev->mc.vram_start);
2777	}
2778
2779	if (!ASIC_IS_NODCE(rdev)) {
2780		WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2781		WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2782	}
2783
2784	/* unlock regs and wait for update */
2785	for (i = 0; i < rdev->num_crtc; i++) {
2786		if (save->crtc_enabled[i]) {
2787			tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2788			if ((tmp & 0x7) != 0) {
2789				tmp &= ~0x7;
2790				WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2791			}
2792			tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2793			if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2794				tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2795				WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2796			}
2797			tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2798			if (tmp & 1) {
2799				tmp &= ~1;
2800				WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2801			}
2802			for (j = 0; j < rdev->usec_timeout; j++) {
2803				tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2804				if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2805					break;
2806				udelay(1);
2807			}
2808		}
2809	}
2810
2811	/* unblackout the MC */
2812	tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2813	tmp &= ~BLACKOUT_MODE_MASK;
2814	WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2815	/* allow CPU access */
2816	WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2817
2818	for (i = 0; i < rdev->num_crtc; i++) {
2819		if (save->crtc_enabled[i]) {
2820			if (ASIC_IS_DCE6(rdev)) {
2821				tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2822				tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2823				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2824				WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2825				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2826			} else {
2827				tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2828				tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2829				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2830				WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2831				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2832			}
2833			/* wait for the next frame */
2834			frame_count = radeon_get_vblank_counter(rdev, i);
2835			for (j = 0; j < rdev->usec_timeout; j++) {
2836				if (radeon_get_vblank_counter(rdev, i) != frame_count)
2837					break;
2838				udelay(1);
2839			}
2840		}
2841	}
2842	if (!ASIC_IS_NODCE(rdev)) {
2843		/* Unlock vga access */
2844		WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2845		mdelay(1);
2846		WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2847	}
2848}
2849
2850void evergreen_mc_program(struct radeon_device *rdev)
2851{
2852	struct evergreen_mc_save save;
2853	u32 tmp;
2854	int i, j;
2855
2856	/* Initialize HDP */
2857	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2858		WREG32((0x2c14 + j), 0x00000000);
2859		WREG32((0x2c18 + j), 0x00000000);
2860		WREG32((0x2c1c + j), 0x00000000);
2861		WREG32((0x2c20 + j), 0x00000000);
2862		WREG32((0x2c24 + j), 0x00000000);
2863	}
2864	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2865
2866	evergreen_mc_stop(rdev, &save);
2867	if (evergreen_mc_wait_for_idle(rdev)) {
2868		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2869	}
2870	/* Lockout access through VGA aperture*/
2871	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2872	/* Update configuration */
2873	if (rdev->flags & RADEON_IS_AGP) {
2874		if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2875			/* VRAM before AGP */
2876			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2877				rdev->mc.vram_start >> 12);
2878			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2879				rdev->mc.gtt_end >> 12);
2880		} else {
2881			/* VRAM after AGP */
2882			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2883				rdev->mc.gtt_start >> 12);
2884			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2885				rdev->mc.vram_end >> 12);
2886		}
2887	} else {
2888		WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2889			rdev->mc.vram_start >> 12);
2890		WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2891			rdev->mc.vram_end >> 12);
2892	}
2893	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2894	/* llano/ontario only */
2895	if ((rdev->family == CHIP_PALM) ||
2896	    (rdev->family == CHIP_SUMO) ||
2897	    (rdev->family == CHIP_SUMO2)) {
2898		tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2899		tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2900		tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2901		WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2902	}
2903	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2904	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2905	WREG32(MC_VM_FB_LOCATION, tmp);
2906	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2907	WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2908	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2909	if (rdev->flags & RADEON_IS_AGP) {
2910		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2911		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2912		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2913	} else {
2914		WREG32(MC_VM_AGP_BASE, 0);
2915		WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2916		WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2917	}
2918	if (evergreen_mc_wait_for_idle(rdev)) {
2919		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2920	}
2921	evergreen_mc_resume(rdev, &save);
2922	/* we need to own VRAM, so turn off the VGA renderer here
2923	 * to stop it overwriting our objects */
2924	rv515_vga_render_disable(rdev);
2925}
2926
2927/*
2928 * CP.
2929 */
2930void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2931{
2932	struct radeon_ring *ring = &rdev->ring[ib->ring];
2933	u32 next_rptr;
2934
2935	/* set to DX10/11 mode */
2936	radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2937	radeon_ring_write(ring, 1);
2938
2939	if (ring->rptr_save_reg) {
2940		next_rptr = ring->wptr + 3 + 4;
2941		radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2942		radeon_ring_write(ring, ((ring->rptr_save_reg - 
2943					  PACKET3_SET_CONFIG_REG_START) >> 2));
2944		radeon_ring_write(ring, next_rptr);
2945	} else if (rdev->wb.enabled) {
2946		next_rptr = ring->wptr + 5 + 4;
2947		radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2948		radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2949		radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2950		radeon_ring_write(ring, next_rptr);
2951		radeon_ring_write(ring, 0);
2952	}
2953
2954	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2955	radeon_ring_write(ring,
2956#ifdef __BIG_ENDIAN
2957			  (2 << 0) |
2958#endif
2959			  (ib->gpu_addr & 0xFFFFFFFC));
2960	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2961	radeon_ring_write(ring, ib->length_dw);
2962}
2963
2964
2965static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2966{
2967	const __be32 *fw_data;
2968	int i;
2969
2970	if (!rdev->me_fw || !rdev->pfp_fw)
2971		return -EINVAL;
2972
2973	r700_cp_stop(rdev);
2974	WREG32(CP_RB_CNTL,
2975#ifdef __BIG_ENDIAN
2976	       BUF_SWAP_32BIT |
2977#endif
2978	       RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2979
2980	fw_data = (const __be32 *)rdev->pfp_fw->data;
2981	WREG32(CP_PFP_UCODE_ADDR, 0);
2982	for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2983		WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2984	WREG32(CP_PFP_UCODE_ADDR, 0);
2985
2986	fw_data = (const __be32 *)rdev->me_fw->data;
2987	WREG32(CP_ME_RAM_WADDR, 0);
2988	for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2989		WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2990
2991	WREG32(CP_PFP_UCODE_ADDR, 0);
2992	WREG32(CP_ME_RAM_WADDR, 0);
2993	WREG32(CP_ME_RAM_RADDR, 0);
2994	return 0;
2995}
2996
2997static int evergreen_cp_start(struct radeon_device *rdev)
2998{
2999	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3000	int r, i;
3001	uint32_t cp_me;
3002
3003	r = radeon_ring_lock(rdev, ring, 7);
3004	if (r) {
3005		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3006		return r;
3007	}
3008	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3009	radeon_ring_write(ring, 0x1);
3010	radeon_ring_write(ring, 0x0);
3011	radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3012	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3013	radeon_ring_write(ring, 0);
3014	radeon_ring_write(ring, 0);
3015	radeon_ring_unlock_commit(rdev, ring, false);
3016
3017	cp_me = 0xff;
3018	WREG32(CP_ME_CNTL, cp_me);
3019
3020	r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3021	if (r) {
3022		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3023		return r;
3024	}
3025
3026	/* setup clear context state */
3027	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3028	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3029
3030	for (i = 0; i < evergreen_default_size; i++)
3031		radeon_ring_write(ring, evergreen_default_state[i]);
3032
3033	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3034	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3035
3036	/* set clear context state */
3037	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3038	radeon_ring_write(ring, 0);
3039
3040	/* SQ_VTX_BASE_VTX_LOC */
3041	radeon_ring_write(ring, 0xc0026f00);
3042	radeon_ring_write(ring, 0x00000000);
3043	radeon_ring_write(ring, 0x00000000);
3044	radeon_ring_write(ring, 0x00000000);
3045
3046	/* Clear consts */
3047	radeon_ring_write(ring, 0xc0036f00);
3048	radeon_ring_write(ring, 0x00000bc4);
3049	radeon_ring_write(ring, 0xffffffff);
3050	radeon_ring_write(ring, 0xffffffff);
3051	radeon_ring_write(ring, 0xffffffff);
3052
3053	radeon_ring_write(ring, 0xc0026900);
3054	radeon_ring_write(ring, 0x00000316);
3055	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3056	radeon_ring_write(ring, 0x00000010); /*  */
3057
3058	radeon_ring_unlock_commit(rdev, ring, false);
3059
3060	return 0;
3061}
3062
3063static int evergreen_cp_resume(struct radeon_device *rdev)
3064{
3065	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3066	u32 tmp;
3067	u32 rb_bufsz;
3068	int r;
3069
3070	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3071	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3072				 SOFT_RESET_PA |
3073				 SOFT_RESET_SH |
3074				 SOFT_RESET_VGT |
3075				 SOFT_RESET_SPI |
3076				 SOFT_RESET_SX));
3077	RREG32(GRBM_SOFT_RESET);
3078	mdelay(15);
3079	WREG32(GRBM_SOFT_RESET, 0);
3080	RREG32(GRBM_SOFT_RESET);
3081
3082	/* Set ring buffer size */
3083	rb_bufsz = order_base_2(ring->ring_size / 8);
3084	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3085#ifdef __BIG_ENDIAN
3086	tmp |= BUF_SWAP_32BIT;
3087#endif
3088	WREG32(CP_RB_CNTL, tmp);
3089	WREG32(CP_SEM_WAIT_TIMER, 0x0);
3090	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3091
3092	/* Set the write pointer delay */
3093	WREG32(CP_RB_WPTR_DELAY, 0);
3094
3095	/* Initialize the ring buffer's read and write pointers */
3096	WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3097	WREG32(CP_RB_RPTR_WR, 0);
3098	ring->wptr = 0;
3099	WREG32(CP_RB_WPTR, ring->wptr);
3100
3101	/* set the wb address whether it's enabled or not */
3102	WREG32(CP_RB_RPTR_ADDR,
3103	       ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3104	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3105	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3106
3107	if (rdev->wb.enabled)
3108		WREG32(SCRATCH_UMSK, 0xff);
3109	else {
3110		tmp |= RB_NO_UPDATE;
3111		WREG32(SCRATCH_UMSK, 0);
3112	}
3113
3114	mdelay(1);
3115	WREG32(CP_RB_CNTL, tmp);
3116
3117	WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3118	WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3119
3120	evergreen_cp_start(rdev);
3121	ring->ready = true;
3122	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3123	if (r) {
3124		ring->ready = false;
3125		return r;
3126	}
3127	return 0;
3128}
3129
3130/*
3131 * Core functions
3132 */
3133static void evergreen_gpu_init(struct radeon_device *rdev)
3134{
3135	u32 gb_addr_config;
3136	u32 mc_shared_chmap, mc_arb_ramcfg;
3137	u32 sx_debug_1;
3138	u32 smx_dc_ctl0;
3139	u32 sq_config;
3140	u32 sq_lds_resource_mgmt;
3141	u32 sq_gpr_resource_mgmt_1;
3142	u32 sq_gpr_resource_mgmt_2;
3143	u32 sq_gpr_resource_mgmt_3;
3144	u32 sq_thread_resource_mgmt;
3145	u32 sq_thread_resource_mgmt_2;
3146	u32 sq_stack_resource_mgmt_1;
3147	u32 sq_stack_resource_mgmt_2;
3148	u32 sq_stack_resource_mgmt_3;
3149	u32 vgt_cache_invalidation;
3150	u32 hdp_host_path_cntl, tmp;
3151	u32 disabled_rb_mask;
3152	int i, j, ps_thread_count;
3153
3154	switch (rdev->family) {
3155	case CHIP_CYPRESS:
3156	case CHIP_HEMLOCK:
3157		rdev->config.evergreen.num_ses = 2;
3158		rdev->config.evergreen.max_pipes = 4;
3159		rdev->config.evergreen.max_tile_pipes = 8;
3160		rdev->config.evergreen.max_simds = 10;
3161		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3162		rdev->config.evergreen.max_gprs = 256;
3163		rdev->config.evergreen.max_threads = 248;
3164		rdev->config.evergreen.max_gs_threads = 32;
3165		rdev->config.evergreen.max_stack_entries = 512;
3166		rdev->config.evergreen.sx_num_of_sets = 4;
3167		rdev->config.evergreen.sx_max_export_size = 256;
3168		rdev->config.evergreen.sx_max_export_pos_size = 64;
3169		rdev->config.evergreen.sx_max_export_smx_size = 192;
3170		rdev->config.evergreen.max_hw_contexts = 8;
3171		rdev->config.evergreen.sq_num_cf_insts = 2;
3172
3173		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3174		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3175		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3176		gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3177		break;
3178	case CHIP_JUNIPER:
3179		rdev->config.evergreen.num_ses = 1;
3180		rdev->config.evergreen.max_pipes = 4;
3181		rdev->config.evergreen.max_tile_pipes = 4;
3182		rdev->config.evergreen.max_simds = 10;
3183		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3184		rdev->config.evergreen.max_gprs = 256;
3185		rdev->config.evergreen.max_threads = 248;
3186		rdev->config.evergreen.max_gs_threads = 32;
3187		rdev->config.evergreen.max_stack_entries = 512;
3188		rdev->config.evergreen.sx_num_of_sets = 4;
3189		rdev->config.evergreen.sx_max_export_size = 256;
3190		rdev->config.evergreen.sx_max_export_pos_size = 64;
3191		rdev->config.evergreen.sx_max_export_smx_size = 192;
3192		rdev->config.evergreen.max_hw_contexts = 8;
3193		rdev->config.evergreen.sq_num_cf_insts = 2;
3194
3195		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3196		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3197		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3198		gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3199		break;
3200	case CHIP_REDWOOD:
3201		rdev->config.evergreen.num_ses = 1;
3202		rdev->config.evergreen.max_pipes = 4;
3203		rdev->config.evergreen.max_tile_pipes = 4;
3204		rdev->config.evergreen.max_simds = 5;
3205		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3206		rdev->config.evergreen.max_gprs = 256;
3207		rdev->config.evergreen.max_threads = 248;
3208		rdev->config.evergreen.max_gs_threads = 32;
3209		rdev->config.evergreen.max_stack_entries = 256;
3210		rdev->config.evergreen.sx_num_of_sets = 4;
3211		rdev->config.evergreen.sx_max_export_size = 256;
3212		rdev->config.evergreen.sx_max_export_pos_size = 64;
3213		rdev->config.evergreen.sx_max_export_smx_size = 192;
3214		rdev->config.evergreen.max_hw_contexts = 8;
3215		rdev->config.evergreen.sq_num_cf_insts = 2;
3216
3217		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3218		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3219		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3220		gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3221		break;
3222	case CHIP_CEDAR:
3223	default:
3224		rdev->config.evergreen.num_ses = 1;
3225		rdev->config.evergreen.max_pipes = 2;
3226		rdev->config.evergreen.max_tile_pipes = 2;
3227		rdev->config.evergreen.max_simds = 2;
3228		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3229		rdev->config.evergreen.max_gprs = 256;
3230		rdev->config.evergreen.max_threads = 192;
3231		rdev->config.evergreen.max_gs_threads = 16;
3232		rdev->config.evergreen.max_stack_entries = 256;
3233		rdev->config.evergreen.sx_num_of_sets = 4;
3234		rdev->config.evergreen.sx_max_export_size = 128;
3235		rdev->config.evergreen.sx_max_export_pos_size = 32;
3236		rdev->config.evergreen.sx_max_export_smx_size = 96;
3237		rdev->config.evergreen.max_hw_contexts = 4;
3238		rdev->config.evergreen.sq_num_cf_insts = 1;
3239
3240		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3241		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3242		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3243		gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3244		break;
3245	case CHIP_PALM:
3246		rdev->config.evergreen.num_ses = 1;
3247		rdev->config.evergreen.max_pipes = 2;
3248		rdev->config.evergreen.max_tile_pipes = 2;
3249		rdev->config.evergreen.max_simds = 2;
3250		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3251		rdev->config.evergreen.max_gprs = 256;
3252		rdev->config.evergreen.max_threads = 192;
3253		rdev->config.evergreen.max_gs_threads = 16;
3254		rdev->config.evergreen.max_stack_entries = 256;
3255		rdev->config.evergreen.sx_num_of_sets = 4;
3256		rdev->config.evergreen.sx_max_export_size = 128;
3257		rdev->config.evergreen.sx_max_export_pos_size = 32;
3258		rdev->config.evergreen.sx_max_export_smx_size = 96;
3259		rdev->config.evergreen.max_hw_contexts = 4;
3260		rdev->config.evergreen.sq_num_cf_insts = 1;
3261
3262		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3263		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3264		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3265		gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3266		break;
3267	case CHIP_SUMO:
3268		rdev->config.evergreen.num_ses = 1;
3269		rdev->config.evergreen.max_pipes = 4;
3270		rdev->config.evergreen.max_tile_pipes = 4;
3271		if (rdev->pdev->device == 0x9648)
3272			rdev->config.evergreen.max_simds = 3;
3273		else if ((rdev->pdev->device == 0x9647) ||
3274			 (rdev->pdev->device == 0x964a))
3275			rdev->config.evergreen.max_simds = 4;
3276		else
3277			rdev->config.evergreen.max_simds = 5;
3278		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3279		rdev->config.evergreen.max_gprs = 256;
3280		rdev->config.evergreen.max_threads = 248;
3281		rdev->config.evergreen.max_gs_threads = 32;
3282		rdev->config.evergreen.max_stack_entries = 256;
3283		rdev->config.evergreen.sx_num_of_sets = 4;
3284		rdev->config.evergreen.sx_max_export_size = 256;
3285		rdev->config.evergreen.sx_max_export_pos_size = 64;
3286		rdev->config.evergreen.sx_max_export_smx_size = 192;
3287		rdev->config.evergreen.max_hw_contexts = 8;
3288		rdev->config.evergreen.sq_num_cf_insts = 2;
3289
3290		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3291		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3292		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3293		gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3294		break;
3295	case CHIP_SUMO2:
3296		rdev->config.evergreen.num_ses = 1;
3297		rdev->config.evergreen.max_pipes = 4;
3298		rdev->config.evergreen.max_tile_pipes = 4;
3299		rdev->config.evergreen.max_simds = 2;
3300		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3301		rdev->config.evergreen.max_gprs = 256;
3302		rdev->config.evergreen.max_threads = 248;
3303		rdev->config.evergreen.max_gs_threads = 32;
3304		rdev->config.evergreen.max_stack_entries = 512;
3305		rdev->config.evergreen.sx_num_of_sets = 4;
3306		rdev->config.evergreen.sx_max_export_size = 256;
3307		rdev->config.evergreen.sx_max_export_pos_size = 64;
3308		rdev->config.evergreen.sx_max_export_smx_size = 192;
3309		rdev->config.evergreen.max_hw_contexts = 4;
3310		rdev->config.evergreen.sq_num_cf_insts = 2;
3311
3312		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3313		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3314		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3315		gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3316		break;
3317	case CHIP_BARTS:
3318		rdev->config.evergreen.num_ses = 2;
3319		rdev->config.evergreen.max_pipes = 4;
3320		rdev->config.evergreen.max_tile_pipes = 8;
3321		rdev->config.evergreen.max_simds = 7;
3322		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3323		rdev->config.evergreen.max_gprs = 256;
3324		rdev->config.evergreen.max_threads = 248;
3325		rdev->config.evergreen.max_gs_threads = 32;
3326		rdev->config.evergreen.max_stack_entries = 512;
3327		rdev->config.evergreen.sx_num_of_sets = 4;
3328		rdev->config.evergreen.sx_max_export_size = 256;
3329		rdev->config.evergreen.sx_max_export_pos_size = 64;
3330		rdev->config.evergreen.sx_max_export_smx_size = 192;
3331		rdev->config.evergreen.max_hw_contexts = 8;
3332		rdev->config.evergreen.sq_num_cf_insts = 2;
3333
3334		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3335		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3336		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3337		gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3338		break;
3339	case CHIP_TURKS:
3340		rdev->config.evergreen.num_ses = 1;
3341		rdev->config.evergreen.max_pipes = 4;
3342		rdev->config.evergreen.max_tile_pipes = 4;
3343		rdev->config.evergreen.max_simds = 6;
3344		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3345		rdev->config.evergreen.max_gprs = 256;
3346		rdev->config.evergreen.max_threads = 248;
3347		rdev->config.evergreen.max_gs_threads = 32;
3348		rdev->config.evergreen.max_stack_entries = 256;
3349		rdev->config.evergreen.sx_num_of_sets = 4;
3350		rdev->config.evergreen.sx_max_export_size = 256;
3351		rdev->config.evergreen.sx_max_export_pos_size = 64;
3352		rdev->config.evergreen.sx_max_export_smx_size = 192;
3353		rdev->config.evergreen.max_hw_contexts = 8;
3354		rdev->config.evergreen.sq_num_cf_insts = 2;
3355
3356		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3357		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3358		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3359		gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3360		break;
3361	case CHIP_CAICOS:
3362		rdev->config.evergreen.num_ses = 1;
3363		rdev->config.evergreen.max_pipes = 2;
3364		rdev->config.evergreen.max_tile_pipes = 2;
3365		rdev->config.evergreen.max_simds = 2;
3366		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3367		rdev->config.evergreen.max_gprs = 256;
3368		rdev->config.evergreen.max_threads = 192;
3369		rdev->config.evergreen.max_gs_threads = 16;
3370		rdev->config.evergreen.max_stack_entries = 256;
3371		rdev->config.evergreen.sx_num_of_sets = 4;
3372		rdev->config.evergreen.sx_max_export_size = 128;
3373		rdev->config.evergreen.sx_max_export_pos_size = 32;
3374		rdev->config.evergreen.sx_max_export_smx_size = 96;
3375		rdev->config.evergreen.max_hw_contexts = 4;
3376		rdev->config.evergreen.sq_num_cf_insts = 1;
3377
3378		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3379		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3380		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3381		gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3382		break;
3383	}
3384
3385	/* Initialize HDP */
3386	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3387		WREG32((0x2c14 + j), 0x00000000);
3388		WREG32((0x2c18 + j), 0x00000000);
3389		WREG32((0x2c1c + j), 0x00000000);
3390		WREG32((0x2c20 + j), 0x00000000);
3391		WREG32((0x2c24 + j), 0x00000000);
3392	}
3393
3394	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3395	WREG32(SRBM_INT_CNTL, 0x1);
3396	WREG32(SRBM_INT_ACK, 0x1);
3397
3398	evergreen_fix_pci_max_read_req_size(rdev);
3399
3400	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3401	if ((rdev->family == CHIP_PALM) ||
3402	    (rdev->family == CHIP_SUMO) ||
3403	    (rdev->family == CHIP_SUMO2))
3404		mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3405	else
3406		mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3407
3408	/* setup tiling info dword.  gb_addr_config is not adequate since it does
3409	 * not have bank info, so create a custom tiling dword.
3410	 * bits 3:0   num_pipes
3411	 * bits 7:4   num_banks
3412	 * bits 11:8  group_size
3413	 * bits 15:12 row_size
3414	 */
3415	rdev->config.evergreen.tile_config = 0;
3416	switch (rdev->config.evergreen.max_tile_pipes) {
3417	case 1:
3418	default:
3419		rdev->config.evergreen.tile_config |= (0 << 0);
3420		break;
3421	case 2:
3422		rdev->config.evergreen.tile_config |= (1 << 0);
3423		break;
3424	case 4:
3425		rdev->config.evergreen.tile_config |= (2 << 0);
3426		break;
3427	case 8:
3428		rdev->config.evergreen.tile_config |= (3 << 0);
3429		break;
3430	}
3431	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3432	if (rdev->flags & RADEON_IS_IGP)
3433		rdev->config.evergreen.tile_config |= 1 << 4;
3434	else {
3435		switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3436		case 0: /* four banks */
3437			rdev->config.evergreen.tile_config |= 0 << 4;
3438			break;
3439		case 1: /* eight banks */
3440			rdev->config.evergreen.tile_config |= 1 << 4;
3441			break;
3442		case 2: /* sixteen banks */
3443		default:
3444			rdev->config.evergreen.tile_config |= 2 << 4;
3445			break;
3446		}
3447	}
3448	rdev->config.evergreen.tile_config |= 0 << 8;
3449	rdev->config.evergreen.tile_config |=
3450		((gb_addr_config & 0x30000000) >> 28) << 12;
3451
3452	if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3453		u32 efuse_straps_4;
3454		u32 efuse_straps_3;
3455
3456		efuse_straps_4 = RREG32_RCU(0x204);
3457		efuse_straps_3 = RREG32_RCU(0x203);
3458		tmp = (((efuse_straps_4 & 0xf) << 4) |
3459		      ((efuse_straps_3 & 0xf0000000) >> 28));
3460	} else {
3461		tmp = 0;
3462		for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3463			u32 rb_disable_bitmap;
3464
3465			WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3466			WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3467			rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3468			tmp <<= 4;
3469			tmp |= rb_disable_bitmap;
3470		}
3471	}
3472	/* enabled rb are just the one not disabled :) */
3473	disabled_rb_mask = tmp;
3474	tmp = 0;
3475	for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3476		tmp |= (1 << i);
3477	/* if all the backends are disabled, fix it up here */
3478	if ((disabled_rb_mask & tmp) == tmp) {
3479		for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3480			disabled_rb_mask &= ~(1 << i);
3481	}
3482
3483	for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3484		u32 simd_disable_bitmap;
3485
3486		WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3487		WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3488		simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3489		simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3490		tmp <<= 16;
3491		tmp |= simd_disable_bitmap;
3492	}
3493	rdev->config.evergreen.active_simds = hweight32(~tmp);
3494
3495	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3496	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3497
3498	WREG32(GB_ADDR_CONFIG, gb_addr_config);
3499	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3500	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3501	WREG32(DMA_TILING_CONFIG, gb_addr_config);
3502	WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3503	WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3504	WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3505
3506	if ((rdev->config.evergreen.max_backends == 1) &&
3507	    (rdev->flags & RADEON_IS_IGP)) {
3508		if ((disabled_rb_mask & 3) == 1) {
3509			/* RB0 disabled, RB1 enabled */
3510			tmp = 0x11111111;
3511		} else {
3512			/* RB1 disabled, RB0 enabled */
3513			tmp = 0x00000000;
3514		}
3515	} else {
3516		tmp = gb_addr_config & NUM_PIPES_MASK;
3517		tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3518						EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3519	}
3520	rdev->config.evergreen.backend_map = tmp;
3521	WREG32(GB_BACKEND_MAP, tmp);
3522
3523	WREG32(CGTS_SYS_TCC_DISABLE, 0);
3524	WREG32(CGTS_TCC_DISABLE, 0);
3525	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3526	WREG32(CGTS_USER_TCC_DISABLE, 0);
3527
3528	/* set HW defaults for 3D engine */
3529	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3530				     ROQ_IB2_START(0x2b)));
3531
3532	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3533
3534	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3535			     SYNC_GRADIENT |
3536			     SYNC_WALKER |
3537			     SYNC_ALIGNER));
3538
3539	sx_debug_1 = RREG32(SX_DEBUG_1);
3540	sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3541	WREG32(SX_DEBUG_1, sx_debug_1);
3542
3543
3544	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3545	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3546	smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3547	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3548
3549	if (rdev->family <= CHIP_SUMO2)
3550		WREG32(SMX_SAR_CTL0, 0x00010000);
3551
3552	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3553					POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3554					SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3555
3556	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3557				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3558				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3559
3560	WREG32(VGT_NUM_INSTANCES, 1);
3561	WREG32(SPI_CONFIG_CNTL, 0);
3562	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3563	WREG32(CP_PERFMON_CNTL, 0);
3564
3565	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3566				  FETCH_FIFO_HIWATER(0x4) |
3567				  DONE_FIFO_HIWATER(0xe0) |
3568				  ALU_UPDATE_FIFO_HIWATER(0x8)));
3569
3570	sq_config = RREG32(SQ_CONFIG);
3571	sq_config &= ~(PS_PRIO(3) |
3572		       VS_PRIO(3) |
3573		       GS_PRIO(3) |
3574		       ES_PRIO(3));
3575	sq_config |= (VC_ENABLE |
3576		      EXPORT_SRC_C |
3577		      PS_PRIO(0) |
3578		      VS_PRIO(1) |
3579		      GS_PRIO(2) |
3580		      ES_PRIO(3));
3581
3582	switch (rdev->family) {
3583	case CHIP_CEDAR:
3584	case CHIP_PALM:
3585	case CHIP_SUMO:
3586	case CHIP_SUMO2:
3587	case CHIP_CAICOS:
3588		/* no vertex cache */
3589		sq_config &= ~VC_ENABLE;
3590		break;
3591	default:
3592		break;
3593	}
3594
3595	sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3596
3597	sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3598	sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3599	sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3600	sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3601	sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3602	sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3603	sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3604
3605	switch (rdev->family) {
3606	case CHIP_CEDAR:
3607	case CHIP_PALM:
3608	case CHIP_SUMO:
3609	case CHIP_SUMO2:
3610		ps_thread_count = 96;
3611		break;
3612	default:
3613		ps_thread_count = 128;
3614		break;
3615	}
3616
3617	sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3618	sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3619	sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3620	sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3621	sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3622	sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3623
3624	sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3625	sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3626	sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3627	sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3628	sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3629	sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3630
3631	WREG32(SQ_CONFIG, sq_config);
3632	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3633	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3634	WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3635	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3636	WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3637	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3638	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3639	WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3640	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3641	WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3642
3643	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3644					  FORCE_EOV_MAX_REZ_CNT(255)));
3645
3646	switch (rdev->family) {
3647	case CHIP_CEDAR:
3648	case CHIP_PALM:
3649	case CHIP_SUMO:
3650	case CHIP_SUMO2:
3651	case CHIP_CAICOS:
3652		vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3653		break;
3654	default:
3655		vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3656		break;
3657	}
3658	vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3659	WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3660
3661	WREG32(VGT_GS_VERTEX_REUSE, 16);
3662	WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3663	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3664
3665	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3666	WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3667
3668	WREG32(CB_PERF_CTR0_SEL_0, 0);
3669	WREG32(CB_PERF_CTR0_SEL_1, 0);
3670	WREG32(CB_PERF_CTR1_SEL_0, 0);
3671	WREG32(CB_PERF_CTR1_SEL_1, 0);
3672	WREG32(CB_PERF_CTR2_SEL_0, 0);
3673	WREG32(CB_PERF_CTR2_SEL_1, 0);
3674	WREG32(CB_PERF_CTR3_SEL_0, 0);
3675	WREG32(CB_PERF_CTR3_SEL_1, 0);
3676
3677	/* clear render buffer base addresses */
3678	WREG32(CB_COLOR0_BASE, 0);
3679	WREG32(CB_COLOR1_BASE, 0);
3680	WREG32(CB_COLOR2_BASE, 0);
3681	WREG32(CB_COLOR3_BASE, 0);
3682	WREG32(CB_COLOR4_BASE, 0);
3683	WREG32(CB_COLOR5_BASE, 0);
3684	WREG32(CB_COLOR6_BASE, 0);
3685	WREG32(CB_COLOR7_BASE, 0);
3686	WREG32(CB_COLOR8_BASE, 0);
3687	WREG32(CB_COLOR9_BASE, 0);
3688	WREG32(CB_COLOR10_BASE, 0);
3689	WREG32(CB_COLOR11_BASE, 0);
3690
3691	/* set the shader const cache sizes to 0 */
3692	for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3693		WREG32(i, 0);
3694	for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3695		WREG32(i, 0);
3696
3697	tmp = RREG32(HDP_MISC_CNTL);
3698	tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3699	WREG32(HDP_MISC_CNTL, tmp);
3700
3701	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3702	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3703
3704	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3705
3706	udelay(50);
3707
3708}
3709
3710int evergreen_mc_init(struct radeon_device *rdev)
3711{
3712	u32 tmp;
3713	int chansize, numchan;
3714
3715	/* Get VRAM informations */
3716	rdev->mc.vram_is_ddr = true;
3717	if ((rdev->family == CHIP_PALM) ||
3718	    (rdev->family == CHIP_SUMO) ||
3719	    (rdev->family == CHIP_SUMO2))
3720		tmp = RREG32(FUS_MC_ARB_RAMCFG);
3721	else
3722		tmp = RREG32(MC_ARB_RAMCFG);
3723	if (tmp & CHANSIZE_OVERRIDE) {
3724		chansize = 16;
3725	} else if (tmp & CHANSIZE_MASK) {
3726		chansize = 64;
3727	} else {
3728		chansize = 32;
3729	}
3730	tmp = RREG32(MC_SHARED_CHMAP);
3731	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3732	case 0:
3733	default:
3734		numchan = 1;
3735		break;
3736	case 1:
3737		numchan = 2;
3738		break;
3739	case 2:
3740		numchan = 4;
3741		break;
3742	case 3:
3743		numchan = 8;
3744		break;
3745	}
3746	rdev->mc.vram_width = numchan * chansize;
3747	/* Could aper size report 0 ? */
3748	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3749	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3750	/* Setup GPU memory space */
3751	if ((rdev->family == CHIP_PALM) ||
3752	    (rdev->family == CHIP_SUMO) ||
3753	    (rdev->family == CHIP_SUMO2)) {
3754		/* size in bytes on fusion */
3755		rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3756		rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3757	} else {
3758		/* size in MB on evergreen/cayman/tn */
3759		rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3760		rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3761	}
3762	rdev->mc.visible_vram_size = rdev->mc.aper_size;
3763	r700_vram_gtt_location(rdev, &rdev->mc);
3764	radeon_update_bandwidth_info(rdev);
3765
3766	return 0;
3767}
3768
3769void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3770{
3771	dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3772		RREG32(GRBM_STATUS));
3773	dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3774		RREG32(GRBM_STATUS_SE0));
3775	dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3776		RREG32(GRBM_STATUS_SE1));
3777	dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3778		RREG32(SRBM_STATUS));
3779	dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3780		RREG32(SRBM_STATUS2));
3781	dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3782		RREG32(CP_STALLED_STAT1));
3783	dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3784		RREG32(CP_STALLED_STAT2));
3785	dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3786		RREG32(CP_BUSY_STAT));
3787	dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3788		RREG32(CP_STAT));
3789	dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3790		RREG32(DMA_STATUS_REG));
3791	if (rdev->family >= CHIP_CAYMAN) {
3792		dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3793			 RREG32(DMA_STATUS_REG + 0x800));
3794	}
3795}
3796
3797bool evergreen_is_display_hung(struct radeon_device *rdev)
3798{
3799	u32 crtc_hung = 0;
3800	u32 crtc_status[6];
3801	u32 i, j, tmp;
3802
3803	for (i = 0; i < rdev->num_crtc; i++) {
3804		if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3805			crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3806			crtc_hung |= (1 << i);
3807		}
3808	}
3809
3810	for (j = 0; j < 10; j++) {
3811		for (i = 0; i < rdev->num_crtc; i++) {
3812			if (crtc_hung & (1 << i)) {
3813				tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3814				if (tmp != crtc_status[i])
3815					crtc_hung &= ~(1 << i);
3816			}
3817		}
3818		if (crtc_hung == 0)
3819			return false;
3820		udelay(100);
3821	}
3822
3823	return true;
3824}
3825
3826u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3827{
3828	u32 reset_mask = 0;
3829	u32 tmp;
3830
3831	/* GRBM_STATUS */
3832	tmp = RREG32(GRBM_STATUS);
3833	if (tmp & (PA_BUSY | SC_BUSY |
3834		   SH_BUSY | SX_BUSY |
3835		   TA_BUSY | VGT_BUSY |
3836		   DB_BUSY | CB_BUSY |
3837		   SPI_BUSY | VGT_BUSY_NO_DMA))
3838		reset_mask |= RADEON_RESET_GFX;
3839
3840	if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3841		   CP_BUSY | CP_COHERENCY_BUSY))
3842		reset_mask |= RADEON_RESET_CP;
3843
3844	if (tmp & GRBM_EE_BUSY)
3845		reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3846
3847	/* DMA_STATUS_REG */
3848	tmp = RREG32(DMA_STATUS_REG);
3849	if (!(tmp & DMA_IDLE))
3850		reset_mask |= RADEON_RESET_DMA;
3851
3852	/* SRBM_STATUS2 */
3853	tmp = RREG32(SRBM_STATUS2);
3854	if (tmp & DMA_BUSY)
3855		reset_mask |= RADEON_RESET_DMA;
3856
3857	/* SRBM_STATUS */
3858	tmp = RREG32(SRBM_STATUS);
3859	if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3860		reset_mask |= RADEON_RESET_RLC;
3861
3862	if (tmp & IH_BUSY)
3863		reset_mask |= RADEON_RESET_IH;
3864
3865	if (tmp & SEM_BUSY)
3866		reset_mask |= RADEON_RESET_SEM;
3867
3868	if (tmp & GRBM_RQ_PENDING)
3869		reset_mask |= RADEON_RESET_GRBM;
3870
3871	if (tmp & VMC_BUSY)
3872		reset_mask |= RADEON_RESET_VMC;
3873
3874	if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3875		   MCC_BUSY | MCD_BUSY))
3876		reset_mask |= RADEON_RESET_MC;
3877
3878	if (evergreen_is_display_hung(rdev))
3879		reset_mask |= RADEON_RESET_DISPLAY;
3880
3881	/* VM_L2_STATUS */
3882	tmp = RREG32(VM_L2_STATUS);
3883	if (tmp & L2_BUSY)
3884		reset_mask |= RADEON_RESET_VMC;
3885
3886	/* Skip MC reset as it's mostly likely not hung, just busy */
3887	if (reset_mask & RADEON_RESET_MC) {
3888		DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3889		reset_mask &= ~RADEON_RESET_MC;
3890	}
3891
3892	return reset_mask;
3893}
3894
3895static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3896{
3897	struct evergreen_mc_save save;
3898	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3899	u32 tmp;
3900
3901	if (reset_mask == 0)
3902		return;
3903
3904	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3905
3906	evergreen_print_gpu_status_regs(rdev);
3907
3908	/* Disable CP parsing/prefetching */
3909	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3910
3911	if (reset_mask & RADEON_RESET_DMA) {
3912		/* Disable DMA */
3913		tmp = RREG32(DMA_RB_CNTL);
3914		tmp &= ~DMA_RB_ENABLE;
3915		WREG32(DMA_RB_CNTL, tmp);
3916	}
3917
3918	udelay(50);
3919
3920	evergreen_mc_stop(rdev, &save);
3921	if (evergreen_mc_wait_for_idle(rdev)) {
3922		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3923	}
3924
3925	if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3926		grbm_soft_reset |= SOFT_RESET_DB |
3927			SOFT_RESET_CB |
3928			SOFT_RESET_PA |
3929			SOFT_RESET_SC |
3930			SOFT_RESET_SPI |
3931			SOFT_RESET_SX |
3932			SOFT_RESET_SH |
3933			SOFT_RESET_TC |
3934			SOFT_RESET_TA |
3935			SOFT_RESET_VC |
3936			SOFT_RESET_VGT;
3937	}
3938
3939	if (reset_mask & RADEON_RESET_CP) {
3940		grbm_soft_reset |= SOFT_RESET_CP |
3941			SOFT_RESET_VGT;
3942
3943		srbm_soft_reset |= SOFT_RESET_GRBM;
3944	}
3945
3946	if (reset_mask & RADEON_RESET_DMA)
3947		srbm_soft_reset |= SOFT_RESET_DMA;
3948
3949	if (reset_mask & RADEON_RESET_DISPLAY)
3950		srbm_soft_reset |= SOFT_RESET_DC;
3951
3952	if (reset_mask & RADEON_RESET_RLC)
3953		srbm_soft_reset |= SOFT_RESET_RLC;
3954
3955	if (reset_mask & RADEON_RESET_SEM)
3956		srbm_soft_reset |= SOFT_RESET_SEM;
3957
3958	if (reset_mask & RADEON_RESET_IH)
3959		srbm_soft_reset |= SOFT_RESET_IH;
3960
3961	if (reset_mask & RADEON_RESET_GRBM)
3962		srbm_soft_reset |= SOFT_RESET_GRBM;
3963
3964	if (reset_mask & RADEON_RESET_VMC)
3965		srbm_soft_reset |= SOFT_RESET_VMC;
3966
3967	if (!(rdev->flags & RADEON_IS_IGP)) {
3968		if (reset_mask & RADEON_RESET_MC)
3969			srbm_soft_reset |= SOFT_RESET_MC;
3970	}
3971
3972	if (grbm_soft_reset) {
3973		tmp = RREG32(GRBM_SOFT_RESET);
3974		tmp |= grbm_soft_reset;
3975		dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3976		WREG32(GRBM_SOFT_RESET, tmp);
3977		tmp = RREG32(GRBM_SOFT_RESET);
3978
3979		udelay(50);
3980
3981		tmp &= ~grbm_soft_reset;
3982		WREG32(GRBM_SOFT_RESET, tmp);
3983		tmp = RREG32(GRBM_SOFT_RESET);
3984	}
3985
3986	if (srbm_soft_reset) {
3987		tmp = RREG32(SRBM_SOFT_RESET);
3988		tmp |= srbm_soft_reset;
3989		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3990		WREG32(SRBM_SOFT_RESET, tmp);
3991		tmp = RREG32(SRBM_SOFT_RESET);
3992
3993		udelay(50);
3994
3995		tmp &= ~srbm_soft_reset;
3996		WREG32(SRBM_SOFT_RESET, tmp);
3997		tmp = RREG32(SRBM_SOFT_RESET);
3998	}
3999
4000	/* Wait a little for things to settle down */
4001	udelay(50);
4002
4003	evergreen_mc_resume(rdev, &save);
4004	udelay(50);
4005
4006	evergreen_print_gpu_status_regs(rdev);
4007}
4008
4009void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4010{
4011	struct evergreen_mc_save save;
4012	u32 tmp, i;
4013
4014	dev_info(rdev->dev, "GPU pci config reset\n");
4015
4016	/* disable dpm? */
4017
4018	/* Disable CP parsing/prefetching */
4019	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4020	udelay(50);
4021	/* Disable DMA */
4022	tmp = RREG32(DMA_RB_CNTL);
4023	tmp &= ~DMA_RB_ENABLE;
4024	WREG32(DMA_RB_CNTL, tmp);
4025	/* XXX other engines? */
4026
4027	/* halt the rlc */
4028	r600_rlc_stop(rdev);
4029
4030	udelay(50);
4031
4032	/* set mclk/sclk to bypass */
4033	rv770_set_clk_bypass_mode(rdev);
4034	/* disable BM */
4035	pci_clear_master(rdev->pdev);
4036	/* disable mem access */
4037	evergreen_mc_stop(rdev, &save);
4038	if (evergreen_mc_wait_for_idle(rdev)) {
4039		dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4040	}
4041	/* reset */
4042	radeon_pci_config_reset(rdev);
4043	/* wait for asic to come out of reset */
4044	for (i = 0; i < rdev->usec_timeout; i++) {
4045		if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4046			break;
4047		udelay(1);
4048	}
4049}
4050
4051int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4052{
4053	u32 reset_mask;
4054
4055	if (hard) {
4056		evergreen_gpu_pci_config_reset(rdev);
4057		return 0;
4058	}
4059
4060	reset_mask = evergreen_gpu_check_soft_reset(rdev);
4061
4062	if (reset_mask)
4063		r600_set_bios_scratch_engine_hung(rdev, true);
4064
4065	/* try soft reset */
4066	evergreen_gpu_soft_reset(rdev, reset_mask);
4067
4068	reset_mask = evergreen_gpu_check_soft_reset(rdev);
4069
4070	/* try pci config reset */
4071	if (reset_mask && radeon_hard_reset)
4072		evergreen_gpu_pci_config_reset(rdev);
4073
4074	reset_mask = evergreen_gpu_check_soft_reset(rdev);
4075
4076	if (!reset_mask)
4077		r600_set_bios_scratch_engine_hung(rdev, false);
4078
4079	return 0;
4080}
4081
4082/**
4083 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4084 *
4085 * @rdev: radeon_device pointer
4086 * @ring: radeon_ring structure holding ring information
4087 *
4088 * Check if the GFX engine is locked up.
4089 * Returns true if the engine appears to be locked up, false if not.
4090 */
4091bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4092{
4093	u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4094
4095	if (!(reset_mask & (RADEON_RESET_GFX |
4096			    RADEON_RESET_COMPUTE |
4097			    RADEON_RESET_CP))) {
4098		radeon_ring_lockup_update(rdev, ring);
4099		return false;
4100	}
4101	return radeon_ring_test_lockup(rdev, ring);
4102}
4103
4104/*
4105 * RLC
4106 */
4107#define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
4108#define RLC_CLEAR_STATE_END_MARKER          0x00000001
4109
4110void sumo_rlc_fini(struct radeon_device *rdev)
4111{
4112	int r;
4113
4114	/* save restore block */
4115	if (rdev->rlc.save_restore_obj) {
4116		r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4117		if (unlikely(r != 0))
4118			dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4119		radeon_bo_unpin(rdev->rlc.save_restore_obj);
4120		radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4121
4122		radeon_bo_unref(&rdev->rlc.save_restore_obj);
4123		rdev->rlc.save_restore_obj = NULL;
4124	}
4125
4126	/* clear state block */
4127	if (rdev->rlc.clear_state_obj) {
4128		r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4129		if (unlikely(r != 0))
4130			dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4131		radeon_bo_unpin(rdev->rlc.clear_state_obj);
4132		radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4133
4134		radeon_bo_unref(&rdev->rlc.clear_state_obj);
4135		rdev->rlc.clear_state_obj = NULL;
4136	}
4137
4138	/* clear state block */
4139	if (rdev->rlc.cp_table_obj) {
4140		r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4141		if (unlikely(r != 0))
4142			dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4143		radeon_bo_unpin(rdev->rlc.cp_table_obj);
4144		radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4145
4146		radeon_bo_unref(&rdev->rlc.cp_table_obj);
4147		rdev->rlc.cp_table_obj = NULL;
4148	}
4149}
4150
4151#define CP_ME_TABLE_SIZE    96
4152
4153int sumo_rlc_init(struct radeon_device *rdev)
4154{
4155	const u32 *src_ptr;
4156	volatile u32 *dst_ptr;
4157	u32 dws, data, i, j, k, reg_num;
4158	u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4159	u64 reg_list_mc_addr;
4160	const struct cs_section_def *cs_data;
4161	int r;
4162
4163	src_ptr = rdev->rlc.reg_list;
4164	dws = rdev->rlc.reg_list_size;
4165	if (rdev->family >= CHIP_BONAIRE) {
4166		dws += (5 * 16) + 48 + 48 + 64;
4167	}
4168	cs_data = rdev->rlc.cs_data;
4169
4170	if (src_ptr) {
4171		/* save restore block */
4172		if (rdev->rlc.save_restore_obj == NULL) {
4173			r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4174					     RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4175					     NULL, &rdev->rlc.save_restore_obj);
4176			if (r) {
4177				dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4178				return r;
4179			}
4180		}
4181
4182		r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4183		if (unlikely(r != 0)) {
4184			sumo_rlc_fini(rdev);
4185			return r;
4186		}
4187		r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4188				  &rdev->rlc.save_restore_gpu_addr);
4189		if (r) {
4190			radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4191			dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4192			sumo_rlc_fini(rdev);
4193			return r;
4194		}
4195
4196		r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4197		if (r) {
4198			dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4199			sumo_rlc_fini(rdev);
4200			return r;
4201		}
4202		/* write the sr buffer */
4203		dst_ptr = rdev->rlc.sr_ptr;
4204		if (rdev->family >= CHIP_TAHITI) {
4205			/* SI */
4206			for (i = 0; i < rdev->rlc.reg_list_size; i++)
4207				dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4208		} else {
4209			/* ON/LN/TN */
4210			/* format:
4211			 * dw0: (reg2 << 16) | reg1
4212			 * dw1: reg1 save space
4213			 * dw2: reg2 save space
4214			 */
4215			for (i = 0; i < dws; i++) {
4216				data = src_ptr[i] >> 2;
4217				i++;
4218				if (i < dws)
4219					data |= (src_ptr[i] >> 2) << 16;
4220				j = (((i - 1) * 3) / 2);
4221				dst_ptr[j] = cpu_to_le32(data);
4222			}
4223			j = ((i * 3) / 2);
4224			dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4225		}
4226		radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4227		radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4228	}
4229
4230	if (cs_data) {
4231		/* clear state block */
4232		if (rdev->family >= CHIP_BONAIRE) {
4233			rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4234		} else if (rdev->family >= CHIP_TAHITI) {
4235			rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4236			dws = rdev->rlc.clear_state_size + (256 / 4);
4237		} else {
4238			reg_list_num = 0;
4239			dws = 0;
4240			for (i = 0; cs_data[i].section != NULL; i++) {
4241				for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4242					reg_list_num++;
4243					dws += cs_data[i].section[j].reg_count;
4244				}
4245			}
4246			reg_list_blk_index = (3 * reg_list_num + 2);
4247			dws += reg_list_blk_index;
4248			rdev->rlc.clear_state_size = dws;
4249		}
4250
4251		if (rdev->rlc.clear_state_obj == NULL) {
4252			r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4253					     RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4254					     NULL, &rdev->rlc.clear_state_obj);
4255			if (r) {
4256				dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4257				sumo_rlc_fini(rdev);
4258				return r;
4259			}
4260		}
4261		r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4262		if (unlikely(r != 0)) {
4263			sumo_rlc_fini(rdev);
4264			return r;
4265		}
4266		r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4267				  &rdev->rlc.clear_state_gpu_addr);
4268		if (r) {
4269			radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4270			dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4271			sumo_rlc_fini(rdev);
4272			return r;
4273		}
4274
4275		r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4276		if (r) {
4277			dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4278			sumo_rlc_fini(rdev);
4279			return r;
4280		}
4281		/* set up the cs buffer */
4282		dst_ptr = rdev->rlc.cs_ptr;
4283		if (rdev->family >= CHIP_BONAIRE) {
4284			cik_get_csb_buffer(rdev, dst_ptr);
4285		} else if (rdev->family >= CHIP_TAHITI) {
4286			reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4287			dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4288			dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4289			dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4290			si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4291		} else {
4292			reg_list_hdr_blk_index = 0;
4293			reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4294			data = upper_32_bits(reg_list_mc_addr);
4295			dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4296			reg_list_hdr_blk_index++;
4297			for (i = 0; cs_data[i].section != NULL; i++) {
4298				for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4299					reg_num = cs_data[i].section[j].reg_count;
4300					data = reg_list_mc_addr & 0xffffffff;
4301					dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4302					reg_list_hdr_blk_index++;
4303
4304					data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4305					dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4306					reg_list_hdr_blk_index++;
4307
4308					data = 0x08000000 | (reg_num * 4);
4309					dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4310					reg_list_hdr_blk_index++;
4311
4312					for (k = 0; k < reg_num; k++) {
4313						data = cs_data[i].section[j].extent[k];
4314						dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4315					}
4316					reg_list_mc_addr += reg_num * 4;
4317					reg_list_blk_index += reg_num;
4318				}
4319			}
4320			dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4321		}
4322		radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4323		radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4324	}
4325
4326	if (rdev->rlc.cp_table_size) {
4327		if (rdev->rlc.cp_table_obj == NULL) {
4328			r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4329					     PAGE_SIZE, true,
4330					     RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4331					     NULL, &rdev->rlc.cp_table_obj);
4332			if (r) {
4333				dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4334				sumo_rlc_fini(rdev);
4335				return r;
4336			}
4337		}
4338
4339		r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4340		if (unlikely(r != 0)) {
4341			dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4342			sumo_rlc_fini(rdev);
4343			return r;
4344		}
4345		r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4346				  &rdev->rlc.cp_table_gpu_addr);
4347		if (r) {
4348			radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4349			dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4350			sumo_rlc_fini(rdev);
4351			return r;
4352		}
4353		r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4354		if (r) {
4355			dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4356			sumo_rlc_fini(rdev);
4357			return r;
4358		}
4359
4360		cik_init_cp_pg_table(rdev);
4361
4362		radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4363		radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4364
4365	}
4366
4367	return 0;
4368}
4369
4370static void evergreen_rlc_start(struct radeon_device *rdev)
4371{
4372	u32 mask = RLC_ENABLE;
4373
4374	if (rdev->flags & RADEON_IS_IGP) {
4375		mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4376	}
4377
4378	WREG32(RLC_CNTL, mask);
4379}
4380
4381int evergreen_rlc_resume(struct radeon_device *rdev)
4382{
4383	u32 i;
4384	const __be32 *fw_data;
4385
4386	if (!rdev->rlc_fw)
4387		return -EINVAL;
4388
4389	r600_rlc_stop(rdev);
4390
4391	WREG32(RLC_HB_CNTL, 0);
4392
4393	if (rdev->flags & RADEON_IS_IGP) {
4394		if (rdev->family == CHIP_ARUBA) {
4395			u32 always_on_bitmap =
4396				3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4397			/* find out the number of active simds */
4398			u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4399			tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4400			tmp = hweight32(~tmp);
4401			if (tmp == rdev->config.cayman.max_simds_per_se) {
4402				WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4403				WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4404				WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4405				WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4406				WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4407			}
4408		} else {
4409			WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4410			WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4411		}
4412		WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4413		WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4414	} else {
4415		WREG32(RLC_HB_BASE, 0);
4416		WREG32(RLC_HB_RPTR, 0);
4417		WREG32(RLC_HB_WPTR, 0);
4418		WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4419		WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4420	}
4421	WREG32(RLC_MC_CNTL, 0);
4422	WREG32(RLC_UCODE_CNTL, 0);
4423
4424	fw_data = (const __be32 *)rdev->rlc_fw->data;
4425	if (rdev->family >= CHIP_ARUBA) {
4426		for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4427			WREG32(RLC_UCODE_ADDR, i);
4428			WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4429		}
4430	} else if (rdev->family >= CHIP_CAYMAN) {
4431		for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4432			WREG32(RLC_UCODE_ADDR, i);
4433			WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4434		}
4435	} else {
4436		for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4437			WREG32(RLC_UCODE_ADDR, i);
4438			WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4439		}
4440	}
4441	WREG32(RLC_UCODE_ADDR, 0);
4442
4443	evergreen_rlc_start(rdev);
4444
4445	return 0;
4446}
4447
4448/* Interrupts */
4449
4450u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4451{
4452	if (crtc >= rdev->num_crtc)
4453		return 0;
4454	else
4455		return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4456}
4457
4458void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4459{
4460	int i;
4461	u32 tmp;
4462
4463	if (rdev->family >= CHIP_CAYMAN) {
4464		cayman_cp_int_cntl_setup(rdev, 0,
4465					 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4466		cayman_cp_int_cntl_setup(rdev, 1, 0);
4467		cayman_cp_int_cntl_setup(rdev, 2, 0);
4468		tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4469		WREG32(CAYMAN_DMA1_CNTL, tmp);
4470	} else
4471		WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4472	tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4473	WREG32(DMA_CNTL, tmp);
4474	WREG32(GRBM_INT_CNTL, 0);
4475	WREG32(SRBM_INT_CNTL, 0);
4476	for (i = 0; i < rdev->num_crtc; i++)
4477		WREG32(INT_MASK + crtc_offsets[i], 0);
4478	for (i = 0; i < rdev->num_crtc; i++)
4479		WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4480
4481	/* only one DAC on DCE5 */
4482	if (!ASIC_IS_DCE5(rdev))
4483		WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4484	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4485
4486	for (i = 0; i < 6; i++)
4487		WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4488}
4489
4490/* Note that the order we write back regs here is important */
4491int evergreen_irq_set(struct radeon_device *rdev)
4492{
4493	int i;
4494	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4495	u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4496	u32 grbm_int_cntl = 0;
4497	u32 dma_cntl, dma_cntl1 = 0;
4498	u32 thermal_int = 0;
4499
4500	if (!rdev->irq.installed) {
4501		WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4502		return -EINVAL;
4503	}
4504	/* don't enable anything if the ih is disabled */
4505	if (!rdev->ih.enabled) {
4506		r600_disable_interrupts(rdev);
4507		/* force the active interrupt state to all disabled */
4508		evergreen_disable_interrupt_state(rdev);
4509		return 0;
4510	}
4511
4512	if (rdev->family == CHIP_ARUBA)
4513		thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4514			~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4515	else
4516		thermal_int = RREG32(CG_THERMAL_INT) &
4517			~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4518
4519	dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4520
4521	if (rdev->family >= CHIP_CAYMAN) {
4522		/* enable CP interrupts on all rings */
4523		if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4524			DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4525			cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4526		}
4527		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4528			DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4529			cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4530		}
4531		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4532			DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4533			cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4534		}
4535	} else {
4536		if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4537			DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4538			cp_int_cntl |= RB_INT_ENABLE;
4539			cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4540		}
4541	}
4542
4543	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4544		DRM_DEBUG("r600_irq_set: sw int dma\n");
4545		dma_cntl |= TRAP_ENABLE;
4546	}
4547
4548	if (rdev->family >= CHIP_CAYMAN) {
4549		dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4550		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4551			DRM_DEBUG("r600_irq_set: sw int dma1\n");
4552			dma_cntl1 |= TRAP_ENABLE;
4553		}
4554	}
4555
4556	if (rdev->irq.dpm_thermal) {
4557		DRM_DEBUG("dpm thermal\n");
4558		thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4559	}
4560
4561	if (rdev->family >= CHIP_CAYMAN) {
4562		cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4563		cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4564		cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4565	} else
4566		WREG32(CP_INT_CNTL, cp_int_cntl);
4567
4568	WREG32(DMA_CNTL, dma_cntl);
4569
4570	if (rdev->family >= CHIP_CAYMAN)
4571		WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4572
4573	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4574
4575	for (i = 0; i < rdev->num_crtc; i++) {
4576		radeon_irq_kms_set_irq_n_enabled(
4577		    rdev, INT_MASK + crtc_offsets[i],
4578		    VBLANK_INT_MASK,
4579		    rdev->irq.crtc_vblank_int[i] ||
4580		    atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4581	}
4582
4583	for (i = 0; i < rdev->num_crtc; i++)
4584		WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4585
4586	for (i = 0; i < 6; i++) {
4587		radeon_irq_kms_set_irq_n_enabled(
4588		    rdev, DC_HPDx_INT_CONTROL(i),
4589		    DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4590		    rdev->irq.hpd[i], "HPD", i);
4591	}
4592
4593	if (rdev->family == CHIP_ARUBA)
4594		WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4595	else
4596		WREG32(CG_THERMAL_INT, thermal_int);
4597
4598	for (i = 0; i < 6; i++) {
4599		radeon_irq_kms_set_irq_n_enabled(
4600		    rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4601		    AFMT_AZ_FORMAT_WTRIG_MASK,
4602		    rdev->irq.afmt[i], "HDMI", i);
4603	}
4604
4605	/* posting read */
4606	RREG32(SRBM_STATUS);
4607
4608	return 0;
4609}
4610
4611/* Note that the order we write back regs here is important */
4612static void evergreen_irq_ack(struct radeon_device *rdev)
4613{
4614	int i, j;
4615	u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4616	u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4617	u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4618
4619	for (i = 0; i < 6; i++) {
4620		disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4621		afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4622		if (i < rdev->num_crtc)
4623			grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4624	}
4625
4626	/* We write back each interrupt register in pairs of two */
4627	for (i = 0; i < rdev->num_crtc; i += 2) {
4628		for (j = i; j < (i + 2); j++) {
4629			if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4630				WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4631				       GRPH_PFLIP_INT_CLEAR);
4632		}
4633
4634		for (j = i; j < (i + 2); j++) {
4635			if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4636				WREG32(VBLANK_STATUS + crtc_offsets[j],
4637				       VBLANK_ACK);
4638			if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4639				WREG32(VLINE_STATUS + crtc_offsets[j],
4640				       VLINE_ACK);
4641		}
4642	}
4643
4644	for (i = 0; i < 6; i++) {
4645		if (disp_int[i] & DC_HPD1_INTERRUPT)
4646			WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4647	}
4648
4649	for (i = 0; i < 6; i++) {
4650		if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4651			WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4652	}
4653
4654	for (i = 0; i < 6; i++) {
4655		if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4656			WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4657				  AFMT_AZ_FORMAT_WTRIG_ACK);
4658	}
4659}
4660
4661static void evergreen_irq_disable(struct radeon_device *rdev)
4662{
4663	r600_disable_interrupts(rdev);
4664	/* Wait and acknowledge irq */
4665	mdelay(1);
4666	evergreen_irq_ack(rdev);
4667	evergreen_disable_interrupt_state(rdev);
4668}
4669
4670void evergreen_irq_suspend(struct radeon_device *rdev)
4671{
4672	evergreen_irq_disable(rdev);
4673	r600_rlc_stop(rdev);
4674}
4675
4676static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4677{
4678	u32 wptr, tmp;
4679
4680	if (rdev->wb.enabled)
4681		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4682	else
4683		wptr = RREG32(IH_RB_WPTR);
4684
4685	if (wptr & RB_OVERFLOW) {
4686		wptr &= ~RB_OVERFLOW;
4687		/* When a ring buffer overflow happen start parsing interrupt
4688		 * from the last not overwritten vector (wptr + 16). Hopefully
4689		 * this should allow us to catchup.
4690		 */
4691		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4692			 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4693		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4694		tmp = RREG32(IH_RB_CNTL);
4695		tmp |= IH_WPTR_OVERFLOW_CLEAR;
4696		WREG32(IH_RB_CNTL, tmp);
4697	}
4698	return (wptr & rdev->ih.ptr_mask);
4699}
4700
4701int evergreen_irq_process(struct radeon_device *rdev)
4702{
4703	u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4704	u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4705	u32 crtc_idx, hpd_idx, afmt_idx;
4706	u32 mask;
4707	u32 wptr;
4708	u32 rptr;
4709	u32 src_id, src_data;
4710	u32 ring_index;
4711	bool queue_hotplug = false;
4712	bool queue_hdmi = false;
4713	bool queue_dp = false;
4714	bool queue_thermal = false;
4715	u32 status, addr;
4716	const char *event_name;
4717
4718	if (!rdev->ih.enabled || rdev->shutdown)
4719		return IRQ_NONE;
4720
4721	wptr = evergreen_get_ih_wptr(rdev);
4722
4723restart_ih:
4724	/* is somebody else already processing irqs? */
4725	if (atomic_xchg(&rdev->ih.lock, 1))
4726		return IRQ_NONE;
4727
4728	rptr = rdev->ih.rptr;
4729	DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4730
4731	/* Order reading of wptr vs. reading of IH ring data */
4732	rmb();
4733
4734	/* display interrupts */
4735	evergreen_irq_ack(rdev);
4736
4737	while (rptr != wptr) {
4738		/* wptr/rptr are in bytes! */
4739		ring_index = rptr / 4;
4740		src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4741		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4742
4743		switch (src_id) {
4744		case 1: /* D1 vblank/vline */
4745		case 2: /* D2 vblank/vline */
4746		case 3: /* D3 vblank/vline */
4747		case 4: /* D4 vblank/vline */
4748		case 5: /* D5 vblank/vline */
4749		case 6: /* D6 vblank/vline */
4750			crtc_idx = src_id - 1;
4751
4752			if (src_data == 0) { /* vblank */
4753				mask = LB_D1_VBLANK_INTERRUPT;
4754				event_name = "vblank";
4755
4756				if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4757					drm_handle_vblank(rdev->ddev, crtc_idx);
4758					rdev->pm.vblank_sync = true;
4759					wake_up(&rdev->irq.vblank_queue);
4760				}
4761				if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4762					radeon_crtc_handle_vblank(rdev,
4763								  crtc_idx);
4764				}
4765
4766			} else if (src_data == 1) { /* vline */
4767				mask = LB_D1_VLINE_INTERRUPT;
4768				event_name = "vline";
4769			} else {
4770				DRM_DEBUG("Unhandled interrupt: %d %d\n",
4771					  src_id, src_data);
4772				break;
4773			}
4774
4775			if (!(disp_int[crtc_idx] & mask)) {
4776				DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4777					  crtc_idx + 1, event_name);
4778			}
4779
4780			disp_int[crtc_idx] &= ~mask;
4781			DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4782
4783			break;
4784		case 8: /* D1 page flip */
4785		case 10: /* D2 page flip */
4786		case 12: /* D3 page flip */
4787		case 14: /* D4 page flip */
4788		case 16: /* D5 page flip */
4789		case 18: /* D6 page flip */
4790			DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4791			if (radeon_use_pflipirq > 0)
4792				radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4793			break;
4794		case 42: /* HPD hotplug */
4795			if (src_data <= 5) {
4796				hpd_idx = src_data;
4797				mask = DC_HPD1_INTERRUPT;
4798				queue_hotplug = true;
4799				event_name = "HPD";
4800
4801			} else if (src_data <= 11) {
4802				hpd_idx = src_data - 6;
4803				mask = DC_HPD1_RX_INTERRUPT;
4804				queue_dp = true;
4805				event_name = "HPD_RX";
4806
4807			} else {
4808				DRM_DEBUG("Unhandled interrupt: %d %d\n",
4809					  src_id, src_data);
4810				break;
4811			}
4812
4813			if (!(disp_int[hpd_idx] & mask))
4814				DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4815
4816			disp_int[hpd_idx] &= ~mask;
4817			DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4818
4819			break;
4820		case 44: /* hdmi */
4821			afmt_idx = src_data;
4822			if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4823				DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4824
4825			if (afmt_idx > 5) {
4826				DRM_ERROR("Unhandled interrupt: %d %d\n",
4827					  src_id, src_data);
4828				break;
4829			}
4830			afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4831			queue_hdmi = true;
4832			DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4833			break;
4834		case 96:
4835			DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4836			WREG32(SRBM_INT_ACK, 0x1);
4837			break;
4838		case 124: /* UVD */
4839			DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4840			radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4841			break;
4842		case 146:
4843		case 147:
4844			addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4845			status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4846			/* reset addr and status */
4847			WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4848			if (addr == 0x0 && status == 0x0)
4849				break;
4850			dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4851			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4852				addr);
4853			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4854				status);
4855			cayman_vm_decode_fault(rdev, status, addr);
4856			break;
4857		case 176: /* CP_INT in ring buffer */
4858		case 177: /* CP_INT in IB1 */
4859		case 178: /* CP_INT in IB2 */
4860			DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4861			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4862			break;
4863		case 181: /* CP EOP event */
4864			DRM_DEBUG("IH: CP EOP\n");
4865			if (rdev->family >= CHIP_CAYMAN) {
4866				switch (src_data) {
4867				case 0:
4868					radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4869					break;
4870				case 1:
4871					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4872					break;
4873				case 2:
4874					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4875					break;
4876				}
4877			} else
4878				radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4879			break;
4880		case 224: /* DMA trap event */
4881			DRM_DEBUG("IH: DMA trap\n");
4882			radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4883			break;
4884		case 230: /* thermal low to high */
4885			DRM_DEBUG("IH: thermal low to high\n");
4886			rdev->pm.dpm.thermal.high_to_low = false;
4887			queue_thermal = true;
4888			break;
4889		case 231: /* thermal high to low */
4890			DRM_DEBUG("IH: thermal high to low\n");
4891			rdev->pm.dpm.thermal.high_to_low = true;
4892			queue_thermal = true;
4893			break;
4894		case 233: /* GUI IDLE */
4895			DRM_DEBUG("IH: GUI idle\n");
4896			break;
4897		case 244: /* DMA trap event */
4898			if (rdev->family >= CHIP_CAYMAN) {
4899				DRM_DEBUG("IH: DMA1 trap\n");
4900				radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4901			}
4902			break;
4903		default:
4904			DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4905			break;
4906		}
4907
4908		/* wptr/rptr are in bytes! */
4909		rptr += 16;
4910		rptr &= rdev->ih.ptr_mask;
4911		WREG32(IH_RB_RPTR, rptr);
4912	}
4913	if (queue_dp)
4914		schedule_work(&rdev->dp_work);
4915	if (queue_hotplug)
4916		schedule_delayed_work(&rdev->hotplug_work, 0);
4917	if (queue_hdmi)
4918		schedule_work(&rdev->audio_work);
4919	if (queue_thermal && rdev->pm.dpm_enabled)
4920		schedule_work(&rdev->pm.dpm.thermal.work);
4921	rdev->ih.rptr = rptr;
4922	atomic_set(&rdev->ih.lock, 0);
4923
4924	/* make sure wptr hasn't changed while processing */
4925	wptr = evergreen_get_ih_wptr(rdev);
4926	if (wptr != rptr)
4927		goto restart_ih;
4928
4929	return IRQ_HANDLED;
4930}
4931
4932static void evergreen_uvd_init(struct radeon_device *rdev)
4933{
4934	int r;
4935
4936	if (!rdev->has_uvd)
4937		return;
4938
4939	r = radeon_uvd_init(rdev);
4940	if (r) {
4941		dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4942		/*
4943		 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4944		 * to early fails uvd_v2_2_resume() and thus nothing happens
4945		 * there. So it is pointless to try to go through that code
4946		 * hence why we disable uvd here.
4947		 */
4948		rdev->has_uvd = 0;
4949		return;
4950	}
4951	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4952	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4953}
4954
4955static void evergreen_uvd_start(struct radeon_device *rdev)
4956{
4957	int r;
4958
4959	if (!rdev->has_uvd)
4960		return;
4961
4962	r = uvd_v2_2_resume(rdev);
4963	if (r) {
4964		dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4965		goto error;
4966	}
4967	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4968	if (r) {
4969		dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4970		goto error;
4971	}
4972	return;
4973
4974error:
4975	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4976}
4977
4978static void evergreen_uvd_resume(struct radeon_device *rdev)
4979{
4980	struct radeon_ring *ring;
4981	int r;
4982
4983	if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4984		return;
4985
4986	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4987	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4988	if (r) {
4989		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4990		return;
4991	}
4992	r = uvd_v1_0_init(rdev);
4993	if (r) {
4994		dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4995		return;
4996	}
4997}
4998
4999static int evergreen_startup(struct radeon_device *rdev)
5000{
5001	struct radeon_ring *ring;
5002	int r;
5003
5004	/* enable pcie gen2 link */
5005	evergreen_pcie_gen2_enable(rdev);
5006	/* enable aspm */
5007	evergreen_program_aspm(rdev);
5008
5009	/* scratch needs to be initialized before MC */
5010	r = r600_vram_scratch_init(rdev);
5011	if (r)
5012		return r;
5013
5014	evergreen_mc_program(rdev);
5015
5016	if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5017		r = ni_mc_load_microcode(rdev);
5018		if (r) {
5019			DRM_ERROR("Failed to load MC firmware!\n");
5020			return r;
5021		}
5022	}
5023
5024	if (rdev->flags & RADEON_IS_AGP) {
5025		evergreen_agp_enable(rdev);
5026	} else {
5027		r = evergreen_pcie_gart_enable(rdev);
5028		if (r)
5029			return r;
5030	}
5031	evergreen_gpu_init(rdev);
5032
5033	/* allocate rlc buffers */
5034	if (rdev->flags & RADEON_IS_IGP) {
5035		rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5036		rdev->rlc.reg_list_size =
5037			(u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5038		rdev->rlc.cs_data = evergreen_cs_data;
5039		r = sumo_rlc_init(rdev);
5040		if (r) {
5041			DRM_ERROR("Failed to init rlc BOs!\n");
5042			return r;
5043		}
5044	}
5045
5046	/* allocate wb buffer */
5047	r = radeon_wb_init(rdev);
5048	if (r)
5049		return r;
5050
5051	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5052	if (r) {
5053		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5054		return r;
5055	}
5056
5057	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5058	if (r) {
5059		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5060		return r;
5061	}
5062
5063	evergreen_uvd_start(rdev);
5064
5065	/* Enable IRQ */
5066	if (!rdev->irq.installed) {
5067		r = radeon_irq_kms_init(rdev);
5068		if (r)
5069			return r;
5070	}
5071
5072	r = r600_irq_init(rdev);
5073	if (r) {
5074		DRM_ERROR("radeon: IH init failed (%d).\n", r);
5075		radeon_irq_kms_fini(rdev);
5076		return r;
5077	}
5078	evergreen_irq_set(rdev);
5079
5080	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5081	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5082			     RADEON_CP_PACKET2);
5083	if (r)
5084		return r;
5085
5086	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5087	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5088			     DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5089	if (r)
5090		return r;
5091
5092	r = evergreen_cp_load_microcode(rdev);
5093	if (r)
5094		return r;
5095	r = evergreen_cp_resume(rdev);
5096	if (r)
5097		return r;
5098	r = r600_dma_resume(rdev);
5099	if (r)
5100		return r;
5101
5102	evergreen_uvd_resume(rdev);
5103
5104	r = radeon_ib_pool_init(rdev);
5105	if (r) {
5106		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5107		return r;
5108	}
5109
5110	r = radeon_audio_init(rdev);
5111	if (r) {
5112		DRM_ERROR("radeon: audio init failed\n");
5113		return r;
5114	}
5115
5116	return 0;
5117}
5118
5119int evergreen_resume(struct radeon_device *rdev)
5120{
5121	int r;
5122
5123	/* reset the asic, the gfx blocks are often in a bad state
5124	 * after the driver is unloaded or after a resume
5125	 */
5126	if (radeon_asic_reset(rdev))
5127		dev_warn(rdev->dev, "GPU reset failed !\n");
5128	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5129	 * posting will perform necessary task to bring back GPU into good
5130	 * shape.
5131	 */
5132	/* post card */
5133	atom_asic_init(rdev->mode_info.atom_context);
5134
5135	/* init golden registers */
5136	evergreen_init_golden_registers(rdev);
5137
5138	if (rdev->pm.pm_method == PM_METHOD_DPM)
5139		radeon_pm_resume(rdev);
5140
5141	rdev->accel_working = true;
5142	r = evergreen_startup(rdev);
5143	if (r) {
5144		DRM_ERROR("evergreen startup failed on resume\n");
5145		rdev->accel_working = false;
5146		return r;
5147	}
5148
5149	return r;
5150
5151}
5152
5153int evergreen_suspend(struct radeon_device *rdev)
5154{
5155	radeon_pm_suspend(rdev);
5156	radeon_audio_fini(rdev);
5157	if (rdev->has_uvd) {
5158		uvd_v1_0_fini(rdev);
5159		radeon_uvd_suspend(rdev);
5160	}
5161	r700_cp_stop(rdev);
5162	r600_dma_stop(rdev);
5163	evergreen_irq_suspend(rdev);
5164	radeon_wb_disable(rdev);
5165	evergreen_pcie_gart_disable(rdev);
5166
5167	return 0;
5168}
5169
5170/* Plan is to move initialization in that function and use
5171 * helper function so that radeon_device_init pretty much
5172 * do nothing more than calling asic specific function. This
5173 * should also allow to remove a bunch of callback function
5174 * like vram_info.
5175 */
5176int evergreen_init(struct radeon_device *rdev)
5177{
5178	int r;
5179
5180	/* Read BIOS */
5181	if (!radeon_get_bios(rdev)) {
5182		if (ASIC_IS_AVIVO(rdev))
5183			return -EINVAL;
5184	}
5185	/* Must be an ATOMBIOS */
5186	if (!rdev->is_atom_bios) {
5187		dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5188		return -EINVAL;
5189	}
5190	r = radeon_atombios_init(rdev);
5191	if (r)
5192		return r;
5193	/* reset the asic, the gfx blocks are often in a bad state
5194	 * after the driver is unloaded or after a resume
5195	 */
5196	if (radeon_asic_reset(rdev))
5197		dev_warn(rdev->dev, "GPU reset failed !\n");
5198	/* Post card if necessary */
5199	if (!radeon_card_posted(rdev)) {
5200		if (!rdev->bios) {
5201			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5202			return -EINVAL;
5203		}
5204		DRM_INFO("GPU not posted. posting now...\n");
5205		atom_asic_init(rdev->mode_info.atom_context);
5206	}
5207	/* init golden registers */
5208	evergreen_init_golden_registers(rdev);
5209	/* Initialize scratch registers */
5210	r600_scratch_init(rdev);
5211	/* Initialize surface registers */
5212	radeon_surface_init(rdev);
5213	/* Initialize clocks */
5214	radeon_get_clock_info(rdev->ddev);
5215	/* Fence driver */
5216	r = radeon_fence_driver_init(rdev);
5217	if (r)
5218		return r;
5219	/* initialize AGP */
5220	if (rdev->flags & RADEON_IS_AGP) {
5221		r = radeon_agp_init(rdev);
5222		if (r)
5223			radeon_agp_disable(rdev);
5224	}
5225	/* initialize memory controller */
5226	r = evergreen_mc_init(rdev);
5227	if (r)
5228		return r;
5229	/* Memory manager */
5230	r = radeon_bo_init(rdev);
5231	if (r)
5232		return r;
5233
5234	if (ASIC_IS_DCE5(rdev)) {
5235		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5236			r = ni_init_microcode(rdev);
5237			if (r) {
5238				DRM_ERROR("Failed to load firmware!\n");
5239				return r;
5240			}
5241		}
5242	} else {
5243		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5244			r = r600_init_microcode(rdev);
5245			if (r) {
5246				DRM_ERROR("Failed to load firmware!\n");
5247				return r;
5248			}
5249		}
5250	}
5251
5252	/* Initialize power management */
5253	radeon_pm_init(rdev);
5254
5255	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5256	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5257
5258	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5259	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5260
5261	evergreen_uvd_init(rdev);
5262
5263	rdev->ih.ring_obj = NULL;
5264	r600_ih_ring_init(rdev, 64 * 1024);
5265
5266	r = r600_pcie_gart_init(rdev);
5267	if (r)
5268		return r;
5269
5270	rdev->accel_working = true;
5271	r = evergreen_startup(rdev);
5272	if (r) {
5273		dev_err(rdev->dev, "disabling GPU acceleration\n");
5274		r700_cp_fini(rdev);
5275		r600_dma_fini(rdev);
5276		r600_irq_fini(rdev);
5277		if (rdev->flags & RADEON_IS_IGP)
5278			sumo_rlc_fini(rdev);
5279		radeon_wb_fini(rdev);
5280		radeon_ib_pool_fini(rdev);
5281		radeon_irq_kms_fini(rdev);
5282		evergreen_pcie_gart_fini(rdev);
5283		rdev->accel_working = false;
5284	}
5285
5286	/* Don't start up if the MC ucode is missing on BTC parts.
5287	 * The default clocks and voltages before the MC ucode
5288	 * is loaded are not suffient for advanced operations.
5289	 */
5290	if (ASIC_IS_DCE5(rdev)) {
5291		if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5292			DRM_ERROR("radeon: MC ucode required for NI+.\n");
5293			return -EINVAL;
5294		}
5295	}
5296
5297	return 0;
5298}
5299
5300void evergreen_fini(struct radeon_device *rdev)
5301{
5302	radeon_pm_fini(rdev);
5303	radeon_audio_fini(rdev);
5304	r700_cp_fini(rdev);
5305	r600_dma_fini(rdev);
5306	r600_irq_fini(rdev);
5307	if (rdev->flags & RADEON_IS_IGP)
5308		sumo_rlc_fini(rdev);
5309	radeon_wb_fini(rdev);
5310	radeon_ib_pool_fini(rdev);
5311	radeon_irq_kms_fini(rdev);
5312	uvd_v1_0_fini(rdev);
5313	radeon_uvd_fini(rdev);
5314	evergreen_pcie_gart_fini(rdev);
5315	r600_vram_scratch_fini(rdev);
5316	radeon_gem_fini(rdev);
5317	radeon_fence_driver_fini(rdev);
5318	radeon_agp_fini(rdev);
5319	radeon_bo_fini(rdev);
5320	radeon_atombios_fini(rdev);
5321	kfree(rdev->bios);
5322	rdev->bios = NULL;
5323}
5324
5325void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5326{
5327	u32 link_width_cntl, speed_cntl;
5328
5329	if (radeon_pcie_gen2 == 0)
5330		return;
5331
5332	if (rdev->flags & RADEON_IS_IGP)
5333		return;
5334
5335	if (!(rdev->flags & RADEON_IS_PCIE))
5336		return;
5337
5338	/* x2 cards have a special sequence */
5339	if (ASIC_IS_X2(rdev))
5340		return;
5341
5342	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5343		(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5344		return;
5345
5346	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5347	if (speed_cntl & LC_CURRENT_DATA_RATE) {
5348		DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5349		return;
5350	}
5351
5352	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5353
5354	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5355	    (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5356
5357		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5358		link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5359		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5360
5361		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5362		speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5363		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5364
5365		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5366		speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5367		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5368
5369		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5370		speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5371		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5372
5373		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5374		speed_cntl |= LC_GEN2_EN_STRAP;
5375		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5376
5377	} else {
5378		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5379		/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5380		if (1)
5381			link_width_cntl |= LC_UPCONFIGURE_DIS;
5382		else
5383			link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5384		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5385	}
5386}
5387
5388void evergreen_program_aspm(struct radeon_device *rdev)
5389{
5390	u32 data, orig;
5391	u32 pcie_lc_cntl, pcie_lc_cntl_old;
5392	bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5393	/* fusion_platform = true
5394	 * if the system is a fusion system
5395	 * (APU or DGPU in a fusion system).
5396	 * todo: check if the system is a fusion platform.
5397	 */
5398	bool fusion_platform = false;
5399
5400	if (radeon_aspm == 0)
5401		return;
5402
5403	if (!(rdev->flags & RADEON_IS_PCIE))
5404		return;
5405
5406	switch (rdev->family) {
5407	case CHIP_CYPRESS:
5408	case CHIP_HEMLOCK:
5409	case CHIP_JUNIPER:
5410	case CHIP_REDWOOD:
5411	case CHIP_CEDAR:
5412	case CHIP_SUMO:
5413	case CHIP_SUMO2:
5414	case CHIP_PALM:
5415	case CHIP_ARUBA:
5416		disable_l0s = true;
5417		break;
5418	default:
5419		disable_l0s = false;
5420		break;
5421	}
5422
5423	if (rdev->flags & RADEON_IS_IGP)
5424		fusion_platform = true; /* XXX also dGPUs in a fusion system */
5425
5426	data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5427	if (fusion_platform)
5428		data &= ~MULTI_PIF;
5429	else
5430		data |= MULTI_PIF;
5431	if (data != orig)
5432		WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5433
5434	data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5435	if (fusion_platform)
5436		data &= ~MULTI_PIF;
5437	else
5438		data |= MULTI_PIF;
5439	if (data != orig)
5440		WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5441
5442	pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5443	pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5444	if (!disable_l0s) {
5445		if (rdev->family >= CHIP_BARTS)
5446			pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5447		else
5448			pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5449	}
5450
5451	if (!disable_l1) {
5452		if (rdev->family >= CHIP_BARTS)
5453			pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5454		else
5455			pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5456
5457		if (!disable_plloff_in_l1) {
5458			data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5459			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5460			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5461			if (data != orig)
5462				WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5463
5464			data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5465			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5466			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5467			if (data != orig)
5468				WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5469
5470			data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5471			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5472			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5473			if (data != orig)
5474				WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5475
5476			data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5477			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5478			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5479			if (data != orig)
5480				WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5481
5482			if (rdev->family >= CHIP_BARTS) {
5483				data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5484				data &= ~PLL_RAMP_UP_TIME_0_MASK;
5485				data |= PLL_RAMP_UP_TIME_0(4);
5486				if (data != orig)
5487					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5488
5489				data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5490				data &= ~PLL_RAMP_UP_TIME_1_MASK;
5491				data |= PLL_RAMP_UP_TIME_1(4);
5492				if (data != orig)
5493					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5494
5495				data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5496				data &= ~PLL_RAMP_UP_TIME_0_MASK;
5497				data |= PLL_RAMP_UP_TIME_0(4);
5498				if (data != orig)
5499					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5500
5501				data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5502				data &= ~PLL_RAMP_UP_TIME_1_MASK;
5503				data |= PLL_RAMP_UP_TIME_1(4);
5504				if (data != orig)
5505					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5506			}
5507
5508			data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5509			data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5510			data |= LC_DYN_LANES_PWR_STATE(3);
5511			if (data != orig)
5512				WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5513
5514			if (rdev->family >= CHIP_BARTS) {
5515				data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5516				data &= ~LS2_EXIT_TIME_MASK;
5517				data |= LS2_EXIT_TIME(1);
5518				if (data != orig)
5519					WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5520
5521				data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5522				data &= ~LS2_EXIT_TIME_MASK;
5523				data |= LS2_EXIT_TIME(1);
5524				if (data != orig)
5525					WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5526			}
5527		}
5528	}
5529
5530	/* evergreen parts only */
5531	if (rdev->family < CHIP_BARTS)
5532		pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5533
5534	if (pcie_lc_cntl != pcie_lc_cntl_old)
5535		WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5536}