1/*
   2 * Copyright 2010 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24
  25#include <linux/firmware.h>
  26#include <linux/pci.h>
  27#include <linux/slab.h>
  28
  29#include <drm/drm_edid.h>
  30#include <drm/drm_vblank.h>
  31#include <drm/radeon_drm.h>
  32#include <drm/drm_fourcc.h>
  33#include <drm/drm_framebuffer.h>
  34
  35#include "atom.h"
  36#include "avivod.h"
  37#include "cik.h"
  38#include "ni.h"
  39#include "rv770.h"
  40#include "evergreen.h"
  41#include "evergreen_blit_shaders.h"
  42#include "evergreen_reg.h"
  43#include "evergreend.h"
  44#include "radeon.h"
  45#include "radeon_asic.h"
  46#include "radeon_audio.h"
  47#include "radeon_ucode.h"
  48#include "si.h"
  49
  50#define DC_HPDx_CONTROL(x)        (DC_HPD1_CONTROL     + (x * 0xc))
  51#define DC_HPDx_INT_CONTROL(x)    (DC_HPD1_INT_CONTROL + (x * 0xc))
  52#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS  + (x * 0xc))
  53
  54/*
  55 * Indirect registers accessor
  56 */
  57u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
  58{
  59	unsigned long flags;
  60	u32 r;
  61
  62	spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  63	WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  64	r = RREG32(EVERGREEN_CG_IND_DATA);
  65	spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  66	return r;
  67}
  68
  69void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  70{
  71	unsigned long flags;
  72
  73	spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  74	WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  75	WREG32(EVERGREEN_CG_IND_DATA, (v));
  76	spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  77}
  78
  79u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
  80{
  81	unsigned long flags;
  82	u32 r;
  83
  84	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  85	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  86	r = RREG32(EVERGREEN_PIF_PHY0_DATA);
  87	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  88	return r;
  89}
  90
  91void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  92{
  93	unsigned long flags;
  94
  95	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  96	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  97	WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
  98	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  99}
 100
 101u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
 102{
 103	unsigned long flags;
 104	u32 r;
 105
 106	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 107	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 108	r = RREG32(EVERGREEN_PIF_PHY1_DATA);
 109	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 110	return r;
 111}
 112
 113void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 114{
 115	unsigned long flags;
 116
 117	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 118	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 119	WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
 120	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 121}
 122
 123static const u32 crtc_offsets[6] =
 124{
 125	EVERGREEN_CRTC0_REGISTER_OFFSET,
 126	EVERGREEN_CRTC1_REGISTER_OFFSET,
 127	EVERGREEN_CRTC2_REGISTER_OFFSET,
 128	EVERGREEN_CRTC3_REGISTER_OFFSET,
 129	EVERGREEN_CRTC4_REGISTER_OFFSET,
 130	EVERGREEN_CRTC5_REGISTER_OFFSET
 131};
 132
 133#include "clearstate_evergreen.h"
 134
 135static const u32 sumo_rlc_save_restore_register_list[] =
 136{
 137	0x98fc,
 138	0x9830,
 139	0x9834,
 140	0x9838,
 141	0x9870,
 142	0x9874,
 143	0x8a14,
 144	0x8b24,
 145	0x8bcc,
 146	0x8b10,
 147	0x8d00,
 148	0x8d04,
 149	0x8c00,
 150	0x8c04,
 151	0x8c08,
 152	0x8c0c,
 153	0x8d8c,
 154	0x8c20,
 155	0x8c24,
 156	0x8c28,
 157	0x8c18,
 158	0x8c1c,
 159	0x8cf0,
 160	0x8e2c,
 161	0x8e38,
 162	0x8c30,
 163	0x9508,
 164	0x9688,
 165	0x9608,
 166	0x960c,
 167	0x9610,
 168	0x9614,
 169	0x88c4,
 170	0x88d4,
 171	0xa008,
 172	0x900c,
 173	0x9100,
 174	0x913c,
 175	0x98f8,
 176	0x98f4,
 177	0x9b7c,
 178	0x3f8c,
 179	0x8950,
 180	0x8954,
 181	0x8a18,
 182	0x8b28,
 183	0x9144,
 184	0x9148,
 185	0x914c,
 186	0x3f90,
 187	0x3f94,
 188	0x915c,
 189	0x9160,
 190	0x9178,
 191	0x917c,
 192	0x9180,
 193	0x918c,
 194	0x9190,
 195	0x9194,
 196	0x9198,
 197	0x919c,
 198	0x91a8,
 199	0x91ac,
 200	0x91b0,
 201	0x91b4,
 202	0x91b8,
 203	0x91c4,
 204	0x91c8,
 205	0x91cc,
 206	0x91d0,
 207	0x91d4,
 208	0x91e0,
 209	0x91e4,
 210	0x91ec,
 211	0x91f0,
 212	0x91f4,
 213	0x9200,
 214	0x9204,
 215	0x929c,
 216	0x9150,
 217	0x802c,
 218};
 219
 220static void evergreen_gpu_init(struct radeon_device *rdev);
 221void evergreen_fini(struct radeon_device *rdev);
 222void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 223void evergreen_program_aspm(struct radeon_device *rdev);
 224
 225static const u32 evergreen_golden_registers[] =
 226{
 227	0x3f90, 0xffff0000, 0xff000000,
 228	0x9148, 0xffff0000, 0xff000000,
 229	0x3f94, 0xffff0000, 0xff000000,
 230	0x914c, 0xffff0000, 0xff000000,
 231	0x9b7c, 0xffffffff, 0x00000000,
 232	0x8a14, 0xffffffff, 0x00000007,
 233	0x8b10, 0xffffffff, 0x00000000,
 234	0x960c, 0xffffffff, 0x54763210,
 235	0x88c4, 0xffffffff, 0x000000c2,
 236	0x88d4, 0xffffffff, 0x00000010,
 237	0x8974, 0xffffffff, 0x00000000,
 238	0xc78, 0x00000080, 0x00000080,
 239	0x5eb4, 0xffffffff, 0x00000002,
 240	0x5e78, 0xffffffff, 0x001000f0,
 241	0x6104, 0x01000300, 0x00000000,
 242	0x5bc0, 0x00300000, 0x00000000,
 243	0x7030, 0xffffffff, 0x00000011,
 244	0x7c30, 0xffffffff, 0x00000011,
 245	0x10830, 0xffffffff, 0x00000011,
 246	0x11430, 0xffffffff, 0x00000011,
 247	0x12030, 0xffffffff, 0x00000011,
 248	0x12c30, 0xffffffff, 0x00000011,
 249	0xd02c, 0xffffffff, 0x08421000,
 250	0x240c, 0xffffffff, 0x00000380,
 251	0x8b24, 0xffffffff, 0x00ff0fff,
 252	0x28a4c, 0x06000000, 0x06000000,
 253	0x10c, 0x00000001, 0x00000001,
 254	0x8d00, 0xffffffff, 0x100e4848,
 255	0x8d04, 0xffffffff, 0x00164745,
 256	0x8c00, 0xffffffff, 0xe4000003,
 257	0x8c04, 0xffffffff, 0x40600060,
 258	0x8c08, 0xffffffff, 0x001c001c,
 259	0x8cf0, 0xffffffff, 0x08e00620,
 260	0x8c20, 0xffffffff, 0x00800080,
 261	0x8c24, 0xffffffff, 0x00800080,
 262	0x8c18, 0xffffffff, 0x20202078,
 263	0x8c1c, 0xffffffff, 0x00001010,
 264	0x28350, 0xffffffff, 0x00000000,
 265	0xa008, 0xffffffff, 0x00010000,
 266	0x5c4, 0xffffffff, 0x00000001,
 267	0x9508, 0xffffffff, 0x00000002,
 268	0x913c, 0x0000000f, 0x0000000a
 269};
 270
 271static const u32 evergreen_golden_registers2[] =
 272{
 273	0x2f4c, 0xffffffff, 0x00000000,
 274	0x54f4, 0xffffffff, 0x00000000,
 275	0x54f0, 0xffffffff, 0x00000000,
 276	0x5498, 0xffffffff, 0x00000000,
 277	0x549c, 0xffffffff, 0x00000000,
 278	0x5494, 0xffffffff, 0x00000000,
 279	0x53cc, 0xffffffff, 0x00000000,
 280	0x53c8, 0xffffffff, 0x00000000,
 281	0x53c4, 0xffffffff, 0x00000000,
 282	0x53c0, 0xffffffff, 0x00000000,
 283	0x53bc, 0xffffffff, 0x00000000,
 284	0x53b8, 0xffffffff, 0x00000000,
 285	0x53b4, 0xffffffff, 0x00000000,
 286	0x53b0, 0xffffffff, 0x00000000
 287};
 288
 289static const u32 cypress_mgcg_init[] =
 290{
 291	0x802c, 0xffffffff, 0xc0000000,
 292	0x5448, 0xffffffff, 0x00000100,
 293	0x55e4, 0xffffffff, 0x00000100,
 294	0x160c, 0xffffffff, 0x00000100,
 295	0x5644, 0xffffffff, 0x00000100,
 296	0xc164, 0xffffffff, 0x00000100,
 297	0x8a18, 0xffffffff, 0x00000100,
 298	0x897c, 0xffffffff, 0x06000100,
 299	0x8b28, 0xffffffff, 0x00000100,
 300	0x9144, 0xffffffff, 0x00000100,
 301	0x9a60, 0xffffffff, 0x00000100,
 302	0x9868, 0xffffffff, 0x00000100,
 303	0x8d58, 0xffffffff, 0x00000100,
 304	0x9510, 0xffffffff, 0x00000100,
 305	0x949c, 0xffffffff, 0x00000100,
 306	0x9654, 0xffffffff, 0x00000100,
 307	0x9030, 0xffffffff, 0x00000100,
 308	0x9034, 0xffffffff, 0x00000100,
 309	0x9038, 0xffffffff, 0x00000100,
 310	0x903c, 0xffffffff, 0x00000100,
 311	0x9040, 0xffffffff, 0x00000100,
 312	0xa200, 0xffffffff, 0x00000100,
 313	0xa204, 0xffffffff, 0x00000100,
 314	0xa208, 0xffffffff, 0x00000100,
 315	0xa20c, 0xffffffff, 0x00000100,
 316	0x971c, 0xffffffff, 0x00000100,
 317	0x977c, 0xffffffff, 0x00000100,
 318	0x3f80, 0xffffffff, 0x00000100,
 319	0xa210, 0xffffffff, 0x00000100,
 320	0xa214, 0xffffffff, 0x00000100,
 321	0x4d8, 0xffffffff, 0x00000100,
 322	0x9784, 0xffffffff, 0x00000100,
 323	0x9698, 0xffffffff, 0x00000100,
 324	0x4d4, 0xffffffff, 0x00000200,
 325	0x30cc, 0xffffffff, 0x00000100,
 326	0xd0c0, 0xffffffff, 0xff000100,
 327	0x802c, 0xffffffff, 0x40000000,
 328	0x915c, 0xffffffff, 0x00010000,
 329	0x9160, 0xffffffff, 0x00030002,
 330	0x9178, 0xffffffff, 0x00070000,
 331	0x917c, 0xffffffff, 0x00030002,
 332	0x9180, 0xffffffff, 0x00050004,
 333	0x918c, 0xffffffff, 0x00010006,
 334	0x9190, 0xffffffff, 0x00090008,
 335	0x9194, 0xffffffff, 0x00070000,
 336	0x9198, 0xffffffff, 0x00030002,
 337	0x919c, 0xffffffff, 0x00050004,
 338	0x91a8, 0xffffffff, 0x00010006,
 339	0x91ac, 0xffffffff, 0x00090008,
 340	0x91b0, 0xffffffff, 0x00070000,
 341	0x91b4, 0xffffffff, 0x00030002,
 342	0x91b8, 0xffffffff, 0x00050004,
 343	0x91c4, 0xffffffff, 0x00010006,
 344	0x91c8, 0xffffffff, 0x00090008,
 345	0x91cc, 0xffffffff, 0x00070000,
 346	0x91d0, 0xffffffff, 0x00030002,
 347	0x91d4, 0xffffffff, 0x00050004,
 348	0x91e0, 0xffffffff, 0x00010006,
 349	0x91e4, 0xffffffff, 0x00090008,
 350	0x91e8, 0xffffffff, 0x00000000,
 351	0x91ec, 0xffffffff, 0x00070000,
 352	0x91f0, 0xffffffff, 0x00030002,
 353	0x91f4, 0xffffffff, 0x00050004,
 354	0x9200, 0xffffffff, 0x00010006,
 355	0x9204, 0xffffffff, 0x00090008,
 356	0x9208, 0xffffffff, 0x00070000,
 357	0x920c, 0xffffffff, 0x00030002,
 358	0x9210, 0xffffffff, 0x00050004,
 359	0x921c, 0xffffffff, 0x00010006,
 360	0x9220, 0xffffffff, 0x00090008,
 361	0x9224, 0xffffffff, 0x00070000,
 362	0x9228, 0xffffffff, 0x00030002,
 363	0x922c, 0xffffffff, 0x00050004,
 364	0x9238, 0xffffffff, 0x00010006,
 365	0x923c, 0xffffffff, 0x00090008,
 366	0x9240, 0xffffffff, 0x00070000,
 367	0x9244, 0xffffffff, 0x00030002,
 368	0x9248, 0xffffffff, 0x00050004,
 369	0x9254, 0xffffffff, 0x00010006,
 370	0x9258, 0xffffffff, 0x00090008,
 371	0x925c, 0xffffffff, 0x00070000,
 372	0x9260, 0xffffffff, 0x00030002,
 373	0x9264, 0xffffffff, 0x00050004,
 374	0x9270, 0xffffffff, 0x00010006,
 375	0x9274, 0xffffffff, 0x00090008,
 376	0x9278, 0xffffffff, 0x00070000,
 377	0x927c, 0xffffffff, 0x00030002,
 378	0x9280, 0xffffffff, 0x00050004,
 379	0x928c, 0xffffffff, 0x00010006,
 380	0x9290, 0xffffffff, 0x00090008,
 381	0x9294, 0xffffffff, 0x00000000,
 382	0x929c, 0xffffffff, 0x00000001,
 383	0x802c, 0xffffffff, 0x40010000,
 384	0x915c, 0xffffffff, 0x00010000,
 385	0x9160, 0xffffffff, 0x00030002,
 386	0x9178, 0xffffffff, 0x00070000,
 387	0x917c, 0xffffffff, 0x00030002,
 388	0x9180, 0xffffffff, 0x00050004,
 389	0x918c, 0xffffffff, 0x00010006,
 390	0x9190, 0xffffffff, 0x00090008,
 391	0x9194, 0xffffffff, 0x00070000,
 392	0x9198, 0xffffffff, 0x00030002,
 393	0x919c, 0xffffffff, 0x00050004,
 394	0x91a8, 0xffffffff, 0x00010006,
 395	0x91ac, 0xffffffff, 0x00090008,
 396	0x91b0, 0xffffffff, 0x00070000,
 397	0x91b4, 0xffffffff, 0x00030002,
 398	0x91b8, 0xffffffff, 0x00050004,
 399	0x91c4, 0xffffffff, 0x00010006,
 400	0x91c8, 0xffffffff, 0x00090008,
 401	0x91cc, 0xffffffff, 0x00070000,
 402	0x91d0, 0xffffffff, 0x00030002,
 403	0x91d4, 0xffffffff, 0x00050004,
 404	0x91e0, 0xffffffff, 0x00010006,
 405	0x91e4, 0xffffffff, 0x00090008,
 406	0x91e8, 0xffffffff, 0x00000000,
 407	0x91ec, 0xffffffff, 0x00070000,
 408	0x91f0, 0xffffffff, 0x00030002,
 409	0x91f4, 0xffffffff, 0x00050004,
 410	0x9200, 0xffffffff, 0x00010006,
 411	0x9204, 0xffffffff, 0x00090008,
 412	0x9208, 0xffffffff, 0x00070000,
 413	0x920c, 0xffffffff, 0x00030002,
 414	0x9210, 0xffffffff, 0x00050004,
 415	0x921c, 0xffffffff, 0x00010006,
 416	0x9220, 0xffffffff, 0x00090008,
 417	0x9224, 0xffffffff, 0x00070000,
 418	0x9228, 0xffffffff, 0x00030002,
 419	0x922c, 0xffffffff, 0x00050004,
 420	0x9238, 0xffffffff, 0x00010006,
 421	0x923c, 0xffffffff, 0x00090008,
 422	0x9240, 0xffffffff, 0x00070000,
 423	0x9244, 0xffffffff, 0x00030002,
 424	0x9248, 0xffffffff, 0x00050004,
 425	0x9254, 0xffffffff, 0x00010006,
 426	0x9258, 0xffffffff, 0x00090008,
 427	0x925c, 0xffffffff, 0x00070000,
 428	0x9260, 0xffffffff, 0x00030002,
 429	0x9264, 0xffffffff, 0x00050004,
 430	0x9270, 0xffffffff, 0x00010006,
 431	0x9274, 0xffffffff, 0x00090008,
 432	0x9278, 0xffffffff, 0x00070000,
 433	0x927c, 0xffffffff, 0x00030002,
 434	0x9280, 0xffffffff, 0x00050004,
 435	0x928c, 0xffffffff, 0x00010006,
 436	0x9290, 0xffffffff, 0x00090008,
 437	0x9294, 0xffffffff, 0x00000000,
 438	0x929c, 0xffffffff, 0x00000001,
 439	0x802c, 0xffffffff, 0xc0000000
 440};
 441
 442static const u32 redwood_mgcg_init[] =
 443{
 444	0x802c, 0xffffffff, 0xc0000000,
 445	0x5448, 0xffffffff, 0x00000100,
 446	0x55e4, 0xffffffff, 0x00000100,
 447	0x160c, 0xffffffff, 0x00000100,
 448	0x5644, 0xffffffff, 0x00000100,
 449	0xc164, 0xffffffff, 0x00000100,
 450	0x8a18, 0xffffffff, 0x00000100,
 451	0x897c, 0xffffffff, 0x06000100,
 452	0x8b28, 0xffffffff, 0x00000100,
 453	0x9144, 0xffffffff, 0x00000100,
 454	0x9a60, 0xffffffff, 0x00000100,
 455	0x9868, 0xffffffff, 0x00000100,
 456	0x8d58, 0xffffffff, 0x00000100,
 457	0x9510, 0xffffffff, 0x00000100,
 458	0x949c, 0xffffffff, 0x00000100,
 459	0x9654, 0xffffffff, 0x00000100,
 460	0x9030, 0xffffffff, 0x00000100,
 461	0x9034, 0xffffffff, 0x00000100,
 462	0x9038, 0xffffffff, 0x00000100,
 463	0x903c, 0xffffffff, 0x00000100,
 464	0x9040, 0xffffffff, 0x00000100,
 465	0xa200, 0xffffffff, 0x00000100,
 466	0xa204, 0xffffffff, 0x00000100,
 467	0xa208, 0xffffffff, 0x00000100,
 468	0xa20c, 0xffffffff, 0x00000100,
 469	0x971c, 0xffffffff, 0x00000100,
 470	0x977c, 0xffffffff, 0x00000100,
 471	0x3f80, 0xffffffff, 0x00000100,
 472	0xa210, 0xffffffff, 0x00000100,
 473	0xa214, 0xffffffff, 0x00000100,
 474	0x4d8, 0xffffffff, 0x00000100,
 475	0x9784, 0xffffffff, 0x00000100,
 476	0x9698, 0xffffffff, 0x00000100,
 477	0x4d4, 0xffffffff, 0x00000200,
 478	0x30cc, 0xffffffff, 0x00000100,
 479	0xd0c0, 0xffffffff, 0xff000100,
 480	0x802c, 0xffffffff, 0x40000000,
 481	0x915c, 0xffffffff, 0x00010000,
 482	0x9160, 0xffffffff, 0x00030002,
 483	0x9178, 0xffffffff, 0x00070000,
 484	0x917c, 0xffffffff, 0x00030002,
 485	0x9180, 0xffffffff, 0x00050004,
 486	0x918c, 0xffffffff, 0x00010006,
 487	0x9190, 0xffffffff, 0x00090008,
 488	0x9194, 0xffffffff, 0x00070000,
 489	0x9198, 0xffffffff, 0x00030002,
 490	0x919c, 0xffffffff, 0x00050004,
 491	0x91a8, 0xffffffff, 0x00010006,
 492	0x91ac, 0xffffffff, 0x00090008,
 493	0x91b0, 0xffffffff, 0x00070000,
 494	0x91b4, 0xffffffff, 0x00030002,
 495	0x91b8, 0xffffffff, 0x00050004,
 496	0x91c4, 0xffffffff, 0x00010006,
 497	0x91c8, 0xffffffff, 0x00090008,
 498	0x91cc, 0xffffffff, 0x00070000,
 499	0x91d0, 0xffffffff, 0x00030002,
 500	0x91d4, 0xffffffff, 0x00050004,
 501	0x91e0, 0xffffffff, 0x00010006,
 502	0x91e4, 0xffffffff, 0x00090008,
 503	0x91e8, 0xffffffff, 0x00000000,
 504	0x91ec, 0xffffffff, 0x00070000,
 505	0x91f0, 0xffffffff, 0x00030002,
 506	0x91f4, 0xffffffff, 0x00050004,
 507	0x9200, 0xffffffff, 0x00010006,
 508	0x9204, 0xffffffff, 0x00090008,
 509	0x9294, 0xffffffff, 0x00000000,
 510	0x929c, 0xffffffff, 0x00000001,
 511	0x802c, 0xffffffff, 0xc0000000
 512};
 513
 514static const u32 cedar_golden_registers[] =
 515{
 516	0x3f90, 0xffff0000, 0xff000000,
 517	0x9148, 0xffff0000, 0xff000000,
 518	0x3f94, 0xffff0000, 0xff000000,
 519	0x914c, 0xffff0000, 0xff000000,
 520	0x9b7c, 0xffffffff, 0x00000000,
 521	0x8a14, 0xffffffff, 0x00000007,
 522	0x8b10, 0xffffffff, 0x00000000,
 523	0x960c, 0xffffffff, 0x54763210,
 524	0x88c4, 0xffffffff, 0x000000c2,
 525	0x88d4, 0xffffffff, 0x00000000,
 526	0x8974, 0xffffffff, 0x00000000,
 527	0xc78, 0x00000080, 0x00000080,
 528	0x5eb4, 0xffffffff, 0x00000002,
 529	0x5e78, 0xffffffff, 0x001000f0,
 530	0x6104, 0x01000300, 0x00000000,
 531	0x5bc0, 0x00300000, 0x00000000,
 532	0x7030, 0xffffffff, 0x00000011,
 533	0x7c30, 0xffffffff, 0x00000011,
 534	0x10830, 0xffffffff, 0x00000011,
 535	0x11430, 0xffffffff, 0x00000011,
 536	0xd02c, 0xffffffff, 0x08421000,
 537	0x240c, 0xffffffff, 0x00000380,
 538	0x8b24, 0xffffffff, 0x00ff0fff,
 539	0x28a4c, 0x06000000, 0x06000000,
 540	0x10c, 0x00000001, 0x00000001,
 541	0x8d00, 0xffffffff, 0x100e4848,
 542	0x8d04, 0xffffffff, 0x00164745,
 543	0x8c00, 0xffffffff, 0xe4000003,
 544	0x8c04, 0xffffffff, 0x40600060,
 545	0x8c08, 0xffffffff, 0x001c001c,
 546	0x8cf0, 0xffffffff, 0x08e00410,
 547	0x8c20, 0xffffffff, 0x00800080,
 548	0x8c24, 0xffffffff, 0x00800080,
 549	0x8c18, 0xffffffff, 0x20202078,
 550	0x8c1c, 0xffffffff, 0x00001010,
 551	0x28350, 0xffffffff, 0x00000000,
 552	0xa008, 0xffffffff, 0x00010000,
 553	0x5c4, 0xffffffff, 0x00000001,
 554	0x9508, 0xffffffff, 0x00000002
 555};
 556
 557static const u32 cedar_mgcg_init[] =
 558{
 559	0x802c, 0xffffffff, 0xc0000000,
 560	0x5448, 0xffffffff, 0x00000100,
 561	0x55e4, 0xffffffff, 0x00000100,
 562	0x160c, 0xffffffff, 0x00000100,
 563	0x5644, 0xffffffff, 0x00000100,
 564	0xc164, 0xffffffff, 0x00000100,
 565	0x8a18, 0xffffffff, 0x00000100,
 566	0x897c, 0xffffffff, 0x06000100,
 567	0x8b28, 0xffffffff, 0x00000100,
 568	0x9144, 0xffffffff, 0x00000100,
 569	0x9a60, 0xffffffff, 0x00000100,
 570	0x9868, 0xffffffff, 0x00000100,
 571	0x8d58, 0xffffffff, 0x00000100,
 572	0x9510, 0xffffffff, 0x00000100,
 573	0x949c, 0xffffffff, 0x00000100,
 574	0x9654, 0xffffffff, 0x00000100,
 575	0x9030, 0xffffffff, 0x00000100,
 576	0x9034, 0xffffffff, 0x00000100,
 577	0x9038, 0xffffffff, 0x00000100,
 578	0x903c, 0xffffffff, 0x00000100,
 579	0x9040, 0xffffffff, 0x00000100,
 580	0xa200, 0xffffffff, 0x00000100,
 581	0xa204, 0xffffffff, 0x00000100,
 582	0xa208, 0xffffffff, 0x00000100,
 583	0xa20c, 0xffffffff, 0x00000100,
 584	0x971c, 0xffffffff, 0x00000100,
 585	0x977c, 0xffffffff, 0x00000100,
 586	0x3f80, 0xffffffff, 0x00000100,
 587	0xa210, 0xffffffff, 0x00000100,
 588	0xa214, 0xffffffff, 0x00000100,
 589	0x4d8, 0xffffffff, 0x00000100,
 590	0x9784, 0xffffffff, 0x00000100,
 591	0x9698, 0xffffffff, 0x00000100,
 592	0x4d4, 0xffffffff, 0x00000200,
 593	0x30cc, 0xffffffff, 0x00000100,
 594	0xd0c0, 0xffffffff, 0xff000100,
 595	0x802c, 0xffffffff, 0x40000000,
 596	0x915c, 0xffffffff, 0x00010000,
 597	0x9178, 0xffffffff, 0x00050000,
 598	0x917c, 0xffffffff, 0x00030002,
 599	0x918c, 0xffffffff, 0x00010004,
 600	0x9190, 0xffffffff, 0x00070006,
 601	0x9194, 0xffffffff, 0x00050000,
 602	0x9198, 0xffffffff, 0x00030002,
 603	0x91a8, 0xffffffff, 0x00010004,
 604	0x91ac, 0xffffffff, 0x00070006,
 605	0x91e8, 0xffffffff, 0x00000000,
 606	0x9294, 0xffffffff, 0x00000000,
 607	0x929c, 0xffffffff, 0x00000001,
 608	0x802c, 0xffffffff, 0xc0000000
 609};
 610
 611static const u32 juniper_mgcg_init[] =
 612{
 613	0x802c, 0xffffffff, 0xc0000000,
 614	0x5448, 0xffffffff, 0x00000100,
 615	0x55e4, 0xffffffff, 0x00000100,
 616	0x160c, 0xffffffff, 0x00000100,
 617	0x5644, 0xffffffff, 0x00000100,
 618	0xc164, 0xffffffff, 0x00000100,
 619	0x8a18, 0xffffffff, 0x00000100,
 620	0x897c, 0xffffffff, 0x06000100,
 621	0x8b28, 0xffffffff, 0x00000100,
 622	0x9144, 0xffffffff, 0x00000100,
 623	0x9a60, 0xffffffff, 0x00000100,
 624	0x9868, 0xffffffff, 0x00000100,
 625	0x8d58, 0xffffffff, 0x00000100,
 626	0x9510, 0xffffffff, 0x00000100,
 627	0x949c, 0xffffffff, 0x00000100,
 628	0x9654, 0xffffffff, 0x00000100,
 629	0x9030, 0xffffffff, 0x00000100,
 630	0x9034, 0xffffffff, 0x00000100,
 631	0x9038, 0xffffffff, 0x00000100,
 632	0x903c, 0xffffffff, 0x00000100,
 633	0x9040, 0xffffffff, 0x00000100,
 634	0xa200, 0xffffffff, 0x00000100,
 635	0xa204, 0xffffffff, 0x00000100,
 636	0xa208, 0xffffffff, 0x00000100,
 637	0xa20c, 0xffffffff, 0x00000100,
 638	0x971c, 0xffffffff, 0x00000100,
 639	0xd0c0, 0xffffffff, 0xff000100,
 640	0x802c, 0xffffffff, 0x40000000,
 641	0x915c, 0xffffffff, 0x00010000,
 642	0x9160, 0xffffffff, 0x00030002,
 643	0x9178, 0xffffffff, 0x00070000,
 644	0x917c, 0xffffffff, 0x00030002,
 645	0x9180, 0xffffffff, 0x00050004,
 646	0x918c, 0xffffffff, 0x00010006,
 647	0x9190, 0xffffffff, 0x00090008,
 648	0x9194, 0xffffffff, 0x00070000,
 649	0x9198, 0xffffffff, 0x00030002,
 650	0x919c, 0xffffffff, 0x00050004,
 651	0x91a8, 0xffffffff, 0x00010006,
 652	0x91ac, 0xffffffff, 0x00090008,
 653	0x91b0, 0xffffffff, 0x00070000,
 654	0x91b4, 0xffffffff, 0x00030002,
 655	0x91b8, 0xffffffff, 0x00050004,
 656	0x91c4, 0xffffffff, 0x00010006,
 657	0x91c8, 0xffffffff, 0x00090008,
 658	0x91cc, 0xffffffff, 0x00070000,
 659	0x91d0, 0xffffffff, 0x00030002,
 660	0x91d4, 0xffffffff, 0x00050004,
 661	0x91e0, 0xffffffff, 0x00010006,
 662	0x91e4, 0xffffffff, 0x00090008,
 663	0x91e8, 0xffffffff, 0x00000000,
 664	0x91ec, 0xffffffff, 0x00070000,
 665	0x91f0, 0xffffffff, 0x00030002,
 666	0x91f4, 0xffffffff, 0x00050004,
 667	0x9200, 0xffffffff, 0x00010006,
 668	0x9204, 0xffffffff, 0x00090008,
 669	0x9208, 0xffffffff, 0x00070000,
 670	0x920c, 0xffffffff, 0x00030002,
 671	0x9210, 0xffffffff, 0x00050004,
 672	0x921c, 0xffffffff, 0x00010006,
 673	0x9220, 0xffffffff, 0x00090008,
 674	0x9224, 0xffffffff, 0x00070000,
 675	0x9228, 0xffffffff, 0x00030002,
 676	0x922c, 0xffffffff, 0x00050004,
 677	0x9238, 0xffffffff, 0x00010006,
 678	0x923c, 0xffffffff, 0x00090008,
 679	0x9240, 0xffffffff, 0x00070000,
 680	0x9244, 0xffffffff, 0x00030002,
 681	0x9248, 0xffffffff, 0x00050004,
 682	0x9254, 0xffffffff, 0x00010006,
 683	0x9258, 0xffffffff, 0x00090008,
 684	0x925c, 0xffffffff, 0x00070000,
 685	0x9260, 0xffffffff, 0x00030002,
 686	0x9264, 0xffffffff, 0x00050004,
 687	0x9270, 0xffffffff, 0x00010006,
 688	0x9274, 0xffffffff, 0x00090008,
 689	0x9278, 0xffffffff, 0x00070000,
 690	0x927c, 0xffffffff, 0x00030002,
 691	0x9280, 0xffffffff, 0x00050004,
 692	0x928c, 0xffffffff, 0x00010006,
 693	0x9290, 0xffffffff, 0x00090008,
 694	0x9294, 0xffffffff, 0x00000000,
 695	0x929c, 0xffffffff, 0x00000001,
 696	0x802c, 0xffffffff, 0xc0000000,
 697	0x977c, 0xffffffff, 0x00000100,
 698	0x3f80, 0xffffffff, 0x00000100,
 699	0xa210, 0xffffffff, 0x00000100,
 700	0xa214, 0xffffffff, 0x00000100,
 701	0x4d8, 0xffffffff, 0x00000100,
 702	0x9784, 0xffffffff, 0x00000100,
 703	0x9698, 0xffffffff, 0x00000100,
 704	0x4d4, 0xffffffff, 0x00000200,
 705	0x30cc, 0xffffffff, 0x00000100,
 706	0x802c, 0xffffffff, 0xc0000000
 707};
 708
 709static const u32 supersumo_golden_registers[] =
 710{
 711	0x5eb4, 0xffffffff, 0x00000002,
 712	0x5c4, 0xffffffff, 0x00000001,
 713	0x7030, 0xffffffff, 0x00000011,
 714	0x7c30, 0xffffffff, 0x00000011,
 715	0x6104, 0x01000300, 0x00000000,
 716	0x5bc0, 0x00300000, 0x00000000,
 717	0x8c04, 0xffffffff, 0x40600060,
 718	0x8c08, 0xffffffff, 0x001c001c,
 719	0x8c20, 0xffffffff, 0x00800080,
 720	0x8c24, 0xffffffff, 0x00800080,
 721	0x8c18, 0xffffffff, 0x20202078,
 722	0x8c1c, 0xffffffff, 0x00001010,
 723	0x918c, 0xffffffff, 0x00010006,
 724	0x91a8, 0xffffffff, 0x00010006,
 725	0x91c4, 0xffffffff, 0x00010006,
 726	0x91e0, 0xffffffff, 0x00010006,
 727	0x9200, 0xffffffff, 0x00010006,
 728	0x9150, 0xffffffff, 0x6e944040,
 729	0x917c, 0xffffffff, 0x00030002,
 730	0x9180, 0xffffffff, 0x00050004,
 731	0x9198, 0xffffffff, 0x00030002,
 732	0x919c, 0xffffffff, 0x00050004,
 733	0x91b4, 0xffffffff, 0x00030002,
 734	0x91b8, 0xffffffff, 0x00050004,
 735	0x91d0, 0xffffffff, 0x00030002,
 736	0x91d4, 0xffffffff, 0x00050004,
 737	0x91f0, 0xffffffff, 0x00030002,
 738	0x91f4, 0xffffffff, 0x00050004,
 739	0x915c, 0xffffffff, 0x00010000,
 740	0x9160, 0xffffffff, 0x00030002,
 741	0x3f90, 0xffff0000, 0xff000000,
 742	0x9178, 0xffffffff, 0x00070000,
 743	0x9194, 0xffffffff, 0x00070000,
 744	0x91b0, 0xffffffff, 0x00070000,
 745	0x91cc, 0xffffffff, 0x00070000,
 746	0x91ec, 0xffffffff, 0x00070000,
 747	0x9148, 0xffff0000, 0xff000000,
 748	0x9190, 0xffffffff, 0x00090008,
 749	0x91ac, 0xffffffff, 0x00090008,
 750	0x91c8, 0xffffffff, 0x00090008,
 751	0x91e4, 0xffffffff, 0x00090008,
 752	0x9204, 0xffffffff, 0x00090008,
 753	0x3f94, 0xffff0000, 0xff000000,
 754	0x914c, 0xffff0000, 0xff000000,
 755	0x929c, 0xffffffff, 0x00000001,
 756	0x8a18, 0xffffffff, 0x00000100,
 757	0x8b28, 0xffffffff, 0x00000100,
 758	0x9144, 0xffffffff, 0x00000100,
 759	0x5644, 0xffffffff, 0x00000100,
 760	0x9b7c, 0xffffffff, 0x00000000,
 761	0x8030, 0xffffffff, 0x0000100a,
 762	0x8a14, 0xffffffff, 0x00000007,
 763	0x8b24, 0xffffffff, 0x00ff0fff,
 764	0x8b10, 0xffffffff, 0x00000000,
 765	0x28a4c, 0x06000000, 0x06000000,
 766	0x4d8, 0xffffffff, 0x00000100,
 767	0x913c, 0xffff000f, 0x0100000a,
 768	0x960c, 0xffffffff, 0x54763210,
 769	0x88c4, 0xffffffff, 0x000000c2,
 770	0x88d4, 0xffffffff, 0x00000010,
 771	0x8974, 0xffffffff, 0x00000000,
 772	0xc78, 0x00000080, 0x00000080,
 773	0x5e78, 0xffffffff, 0x001000f0,
 774	0xd02c, 0xffffffff, 0x08421000,
 775	0xa008, 0xffffffff, 0x00010000,
 776	0x8d00, 0xffffffff, 0x100e4848,
 777	0x8d04, 0xffffffff, 0x00164745,
 778	0x8c00, 0xffffffff, 0xe4000003,
 779	0x8cf0, 0x1fffffff, 0x08e00620,
 780	0x28350, 0xffffffff, 0x00000000,
 781	0x9508, 0xffffffff, 0x00000002
 782};
 783
 784static const u32 sumo_golden_registers[] =
 785{
 786	0x900c, 0x00ffffff, 0x0017071f,
 787	0x8c18, 0xffffffff, 0x10101060,
 788	0x8c1c, 0xffffffff, 0x00001010,
 789	0x8c30, 0x0000000f, 0x00000005,
 790	0x9688, 0x0000000f, 0x00000007
 791};
 792
 793static const u32 wrestler_golden_registers[] =
 794{
 795	0x5eb4, 0xffffffff, 0x00000002,
 796	0x5c4, 0xffffffff, 0x00000001,
 797	0x7030, 0xffffffff, 0x00000011,
 798	0x7c30, 0xffffffff, 0x00000011,
 799	0x6104, 0x01000300, 0x00000000,
 800	0x5bc0, 0x00300000, 0x00000000,
 801	0x918c, 0xffffffff, 0x00010006,
 802	0x91a8, 0xffffffff, 0x00010006,
 803	0x9150, 0xffffffff, 0x6e944040,
 804	0x917c, 0xffffffff, 0x00030002,
 805	0x9198, 0xffffffff, 0x00030002,
 806	0x915c, 0xffffffff, 0x00010000,
 807	0x3f90, 0xffff0000, 0xff000000,
 808	0x9178, 0xffffffff, 0x00070000,
 809	0x9194, 0xffffffff, 0x00070000,
 810	0x9148, 0xffff0000, 0xff000000,
 811	0x9190, 0xffffffff, 0x00090008,
 812	0x91ac, 0xffffffff, 0x00090008,
 813	0x3f94, 0xffff0000, 0xff000000,
 814	0x914c, 0xffff0000, 0xff000000,
 815	0x929c, 0xffffffff, 0x00000001,
 816	0x8a18, 0xffffffff, 0x00000100,
 817	0x8b28, 0xffffffff, 0x00000100,
 818	0x9144, 0xffffffff, 0x00000100,
 819	0x9b7c, 0xffffffff, 0x00000000,
 820	0x8030, 0xffffffff, 0x0000100a,
 821	0x8a14, 0xffffffff, 0x00000001,
 822	0x8b24, 0xffffffff, 0x00ff0fff,
 823	0x8b10, 0xffffffff, 0x00000000,
 824	0x28a4c, 0x06000000, 0x06000000,
 825	0x4d8, 0xffffffff, 0x00000100,
 826	0x913c, 0xffff000f, 0x0100000a,
 827	0x960c, 0xffffffff, 0x54763210,
 828	0x88c4, 0xffffffff, 0x000000c2,
 829	0x88d4, 0xffffffff, 0x00000010,
 830	0x8974, 0xffffffff, 0x00000000,
 831	0xc78, 0x00000080, 0x00000080,
 832	0x5e78, 0xffffffff, 0x001000f0,
 833	0xd02c, 0xffffffff, 0x08421000,
 834	0xa008, 0xffffffff, 0x00010000,
 835	0x8d00, 0xffffffff, 0x100e4848,
 836	0x8d04, 0xffffffff, 0x00164745,
 837	0x8c00, 0xffffffff, 0xe4000003,
 838	0x8cf0, 0x1fffffff, 0x08e00410,
 839	0x28350, 0xffffffff, 0x00000000,
 840	0x9508, 0xffffffff, 0x00000002,
 841	0x900c, 0xffffffff, 0x0017071f,
 842	0x8c18, 0xffffffff, 0x10101060,
 843	0x8c1c, 0xffffffff, 0x00001010
 844};
 845
 846static const u32 barts_golden_registers[] =
 847{
 848	0x5eb4, 0xffffffff, 0x00000002,
 849	0x5e78, 0x8f311ff1, 0x001000f0,
 850	0x3f90, 0xffff0000, 0xff000000,
 851	0x9148, 0xffff0000, 0xff000000,
 852	0x3f94, 0xffff0000, 0xff000000,
 853	0x914c, 0xffff0000, 0xff000000,
 854	0xc78, 0x00000080, 0x00000080,
 855	0xbd4, 0x70073777, 0x00010001,
 856	0xd02c, 0xbfffff1f, 0x08421000,
 857	0xd0b8, 0x03773777, 0x02011003,
 858	0x5bc0, 0x00200000, 0x50100000,
 859	0x98f8, 0x33773777, 0x02011003,
 860	0x98fc, 0xffffffff, 0x76543210,
 861	0x7030, 0x31000311, 0x00000011,
 862	0x2f48, 0x00000007, 0x02011003,
 863	0x6b28, 0x00000010, 0x00000012,
 864	0x7728, 0x00000010, 0x00000012,
 865	0x10328, 0x00000010, 0x00000012,
 866	0x10f28, 0x00000010, 0x00000012,
 867	0x11b28, 0x00000010, 0x00000012,
 868	0x12728, 0x00000010, 0x00000012,
 869	0x240c, 0x000007ff, 0x00000380,
 870	0x8a14, 0xf000001f, 0x00000007,
 871	0x8b24, 0x3fff3fff, 0x00ff0fff,
 872	0x8b10, 0x0000ff0f, 0x00000000,
 873	0x28a4c, 0x07ffffff, 0x06000000,
 874	0x10c, 0x00000001, 0x00010003,
 875	0xa02c, 0xffffffff, 0x0000009b,
 876	0x913c, 0x0000000f, 0x0100000a,
 877	0x8d00, 0xffff7f7f, 0x100e4848,
 878	0x8d04, 0x00ffffff, 0x00164745,
 879	0x8c00, 0xfffc0003, 0xe4000003,
 880	0x8c04, 0xf8ff00ff, 0x40600060,
 881	0x8c08, 0x00ff00ff, 0x001c001c,
 882	0x8cf0, 0x1fff1fff, 0x08e00620,
 883	0x8c20, 0x0fff0fff, 0x00800080,
 884	0x8c24, 0x0fff0fff, 0x00800080,
 885	0x8c18, 0xffffffff, 0x20202078,
 886	0x8c1c, 0x0000ffff, 0x00001010,
 887	0x28350, 0x00000f01, 0x00000000,
 888	0x9508, 0x3700001f, 0x00000002,
 889	0x960c, 0xffffffff, 0x54763210,
 890	0x88c4, 0x001f3ae3, 0x000000c2,
 891	0x88d4, 0x0000001f, 0x00000010,
 892	0x8974, 0xffffffff, 0x00000000
 893};
 894
 895static const u32 turks_golden_registers[] =
 896{
 897	0x5eb4, 0xffffffff, 0x00000002,
 898	0x5e78, 0x8f311ff1, 0x001000f0,
 899	0x8c8, 0x00003000, 0x00001070,
 900	0x8cc, 0x000fffff, 0x00040035,
 901	0x3f90, 0xffff0000, 0xfff00000,
 902	0x9148, 0xffff0000, 0xfff00000,
 903	0x3f94, 0xffff0000, 0xfff00000,
 904	0x914c, 0xffff0000, 0xfff00000,
 905	0xc78, 0x00000080, 0x00000080,
 906	0xbd4, 0x00073007, 0x00010002,
 907	0xd02c, 0xbfffff1f, 0x08421000,
 908	0xd0b8, 0x03773777, 0x02010002,
 909	0x5bc0, 0x00200000, 0x50100000,
 910	0x98f8, 0x33773777, 0x00010002,
 911	0x98fc, 0xffffffff, 0x33221100,
 912	0x7030, 0x31000311, 0x00000011,
 913	0x2f48, 0x33773777, 0x00010002,
 914	0x6b28, 0x00000010, 0x00000012,
 915	0x7728, 0x00000010, 0x00000012,
 916	0x10328, 0x00000010, 0x00000012,
 917	0x10f28, 0x00000010, 0x00000012,
 918	0x11b28, 0x00000010, 0x00000012,
 919	0x12728, 0x00000010, 0x00000012,
 920	0x240c, 0x000007ff, 0x00000380,
 921	0x8a14, 0xf000001f, 0x00000007,
 922	0x8b24, 0x3fff3fff, 0x00ff0fff,
 923	0x8b10, 0x0000ff0f, 0x00000000,
 924	0x28a4c, 0x07ffffff, 0x06000000,
 925	0x10c, 0x00000001, 0x00010003,
 926	0xa02c, 0xffffffff, 0x0000009b,
 927	0x913c, 0x0000000f, 0x0100000a,
 928	0x8d00, 0xffff7f7f, 0x100e4848,
 929	0x8d04, 0x00ffffff, 0x00164745,
 930	0x8c00, 0xfffc0003, 0xe4000003,
 931	0x8c04, 0xf8ff00ff, 0x40600060,
 932	0x8c08, 0x00ff00ff, 0x001c001c,
 933	0x8cf0, 0x1fff1fff, 0x08e00410,
 934	0x8c20, 0x0fff0fff, 0x00800080,
 935	0x8c24, 0x0fff0fff, 0x00800080,
 936	0x8c18, 0xffffffff, 0x20202078,
 937	0x8c1c, 0x0000ffff, 0x00001010,
 938	0x28350, 0x00000f01, 0x00000000,
 939	0x9508, 0x3700001f, 0x00000002,
 940	0x960c, 0xffffffff, 0x54763210,
 941	0x88c4, 0x001f3ae3, 0x000000c2,
 942	0x88d4, 0x0000001f, 0x00000010,
 943	0x8974, 0xffffffff, 0x00000000
 944};
 945
 946static const u32 caicos_golden_registers[] =
 947{
 948	0x5eb4, 0xffffffff, 0x00000002,
 949	0x5e78, 0x8f311ff1, 0x001000f0,
 950	0x8c8, 0x00003420, 0x00001450,
 951	0x8cc, 0x000fffff, 0x00040035,
 952	0x3f90, 0xffff0000, 0xfffc0000,
 953	0x9148, 0xffff0000, 0xfffc0000,
 954	0x3f94, 0xffff0000, 0xfffc0000,
 955	0x914c, 0xffff0000, 0xfffc0000,
 956	0xc78, 0x00000080, 0x00000080,
 957	0xbd4, 0x00073007, 0x00010001,
 958	0xd02c, 0xbfffff1f, 0x08421000,
 959	0xd0b8, 0x03773777, 0x02010001,
 960	0x5bc0, 0x00200000, 0x50100000,
 961	0x98f8, 0x33773777, 0x02010001,
 962	0x98fc, 0xffffffff, 0x33221100,
 963	0x7030, 0x31000311, 0x00000011,
 964	0x2f48, 0x33773777, 0x02010001,
 965	0x6b28, 0x00000010, 0x00000012,
 966	0x7728, 0x00000010, 0x00000012,
 967	0x10328, 0x00000010, 0x00000012,
 968	0x10f28, 0x00000010, 0x00000012,
 969	0x11b28, 0x00000010, 0x00000012,
 970	0x12728, 0x00000010, 0x00000012,
 971	0x240c, 0x000007ff, 0x00000380,
 972	0x8a14, 0xf000001f, 0x00000001,
 973	0x8b24, 0x3fff3fff, 0x00ff0fff,
 974	0x8b10, 0x0000ff0f, 0x00000000,
 975	0x28a4c, 0x07ffffff, 0x06000000,
 976	0x10c, 0x00000001, 0x00010003,
 977	0xa02c, 0xffffffff, 0x0000009b,
 978	0x913c, 0x0000000f, 0x0100000a,
 979	0x8d00, 0xffff7f7f, 0x100e4848,
 980	0x8d04, 0x00ffffff, 0x00164745,
 981	0x8c00, 0xfffc0003, 0xe4000003,
 982	0x8c04, 0xf8ff00ff, 0x40600060,
 983	0x8c08, 0x00ff00ff, 0x001c001c,
 984	0x8cf0, 0x1fff1fff, 0x08e00410,
 985	0x8c20, 0x0fff0fff, 0x00800080,
 986	0x8c24, 0x0fff0fff, 0x00800080,
 987	0x8c18, 0xffffffff, 0x20202078,
 988	0x8c1c, 0x0000ffff, 0x00001010,
 989	0x28350, 0x00000f01, 0x00000000,
 990	0x9508, 0x3700001f, 0x00000002,
 991	0x960c, 0xffffffff, 0x54763210,
 992	0x88c4, 0x001f3ae3, 0x000000c2,
 993	0x88d4, 0x0000001f, 0x00000010,
 994	0x8974, 0xffffffff, 0x00000000
 995};
 996
 997static void evergreen_init_golden_registers(struct radeon_device *rdev)
 998{
 999	switch (rdev->family) {
1000	case CHIP_CYPRESS:
1001	case CHIP_HEMLOCK:
1002		radeon_program_register_sequence(rdev,
1003						 evergreen_golden_registers,
1004						 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1005		radeon_program_register_sequence(rdev,
1006						 evergreen_golden_registers2,
1007						 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1008		radeon_program_register_sequence(rdev,
1009						 cypress_mgcg_init,
1010						 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1011		break;
1012	case CHIP_JUNIPER:
1013		radeon_program_register_sequence(rdev,
1014						 evergreen_golden_registers,
1015						 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1016		radeon_program_register_sequence(rdev,
1017						 evergreen_golden_registers2,
1018						 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1019		radeon_program_register_sequence(rdev,
1020						 juniper_mgcg_init,
1021						 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1022		break;
1023	case CHIP_REDWOOD:
1024		radeon_program_register_sequence(rdev,
1025						 evergreen_golden_registers,
1026						 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1027		radeon_program_register_sequence(rdev,
1028						 evergreen_golden_registers2,
1029						 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1030		radeon_program_register_sequence(rdev,
1031						 redwood_mgcg_init,
1032						 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1033		break;
1034	case CHIP_CEDAR:
1035		radeon_program_register_sequence(rdev,
1036						 cedar_golden_registers,
1037						 (const u32)ARRAY_SIZE(cedar_golden_registers));
1038		radeon_program_register_sequence(rdev,
1039						 evergreen_golden_registers2,
1040						 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1041		radeon_program_register_sequence(rdev,
1042						 cedar_mgcg_init,
1043						 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1044		break;
1045	case CHIP_PALM:
1046		radeon_program_register_sequence(rdev,
1047						 wrestler_golden_registers,
1048						 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1049		break;
1050	case CHIP_SUMO:
1051		radeon_program_register_sequence(rdev,
1052						 supersumo_golden_registers,
1053						 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1054		break;
1055	case CHIP_SUMO2:
1056		radeon_program_register_sequence(rdev,
1057						 supersumo_golden_registers,
1058						 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1059		radeon_program_register_sequence(rdev,
1060						 sumo_golden_registers,
1061						 (const u32)ARRAY_SIZE(sumo_golden_registers));
1062		break;
1063	case CHIP_BARTS:
1064		radeon_program_register_sequence(rdev,
1065						 barts_golden_registers,
1066						 (const u32)ARRAY_SIZE(barts_golden_registers));
1067		break;
1068	case CHIP_TURKS:
1069		radeon_program_register_sequence(rdev,
1070						 turks_golden_registers,
1071						 (const u32)ARRAY_SIZE(turks_golden_registers));
1072		break;
1073	case CHIP_CAICOS:
1074		radeon_program_register_sequence(rdev,
1075						 caicos_golden_registers,
1076						 (const u32)ARRAY_SIZE(caicos_golden_registers));
1077		break;
1078	default:
1079		break;
1080	}
1081}
1082
1083/**
1084 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1085 *
1086 * @rdev: radeon_device pointer
1087 * @reg: register offset in bytes
1088 * @val: register value
1089 *
1090 * Returns 0 for success or -EINVAL for an invalid register
1091 *
1092 */
1093int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1094					u32 reg, u32 *val)
1095{
1096	switch (reg) {
1097	case GRBM_STATUS:
1098	case GRBM_STATUS_SE0:
1099	case GRBM_STATUS_SE1:
1100	case SRBM_STATUS:
1101	case SRBM_STATUS2:
1102	case DMA_STATUS_REG:
1103	case UVD_STATUS:
1104		*val = RREG32(reg);
1105		return 0;
1106	default:
1107		return -EINVAL;
1108	}
1109}
1110
1111void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1112			     unsigned *bankh, unsigned *mtaspect,
1113			     unsigned *tile_split)
1114{
1115	*bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1116	*bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1117	*mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1118	*tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1119	switch (*bankw) {
1120	default:
1121	case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1122	case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1123	case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1124	case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1125	}
1126	switch (*bankh) {
1127	default:
1128	case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1129	case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1130	case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1131	case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1132	}
1133	switch (*mtaspect) {
1134	default:
1135	case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1136	case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1137	case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1138	case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1139	}
1140}
1141
1142static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1143			      u32 cntl_reg, u32 status_reg)
1144{
1145	int r, i;
1146	struct atom_clock_dividers dividers;
1147
1148	r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1149					   clock, false, &dividers);
1150	if (r)
1151		return r;
1152
1153	WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1154
1155	for (i = 0; i < 100; i++) {
1156		if (RREG32(status_reg) & DCLK_STATUS)
1157			break;
1158		mdelay(10);
1159	}
1160	if (i == 100)
1161		return -ETIMEDOUT;
1162
1163	return 0;
1164}
1165
1166int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1167{
1168	int r = 0;
1169	u32 cg_scratch = RREG32(CG_SCRATCH1);
1170
1171	r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1172	if (r)
1173		goto done;
1174	cg_scratch &= 0xffff0000;
1175	cg_scratch |= vclk / 100; /* Mhz */
1176
1177	r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1178	if (r)
1179		goto done;
1180	cg_scratch &= 0x0000ffff;
1181	cg_scratch |= (dclk / 100) << 16; /* Mhz */
1182
1183done:
1184	WREG32(CG_SCRATCH1, cg_scratch);
1185
1186	return r;
1187}
1188
1189int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1190{
1191	/* start off with something large */
1192	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1193	int r;
1194
1195	/* bypass vclk and dclk with bclk */
1196	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1197		VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1198		~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1199
1200	/* put PLL in bypass mode */
1201	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1202
1203	if (!vclk || !dclk) {
1204		/* keep the Bypass mode, put PLL to sleep */
1205		WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1206		return 0;
1207	}
1208
1209	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1210					  16384, 0x03FFFFFF, 0, 128, 5,
1211					  &fb_div, &vclk_div, &dclk_div);
1212	if (r)
1213		return r;
1214
1215	/* set VCO_MODE to 1 */
1216	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1217
1218	/* toggle UPLL_SLEEP to 1 then back to 0 */
1219	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1220	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1221
1222	/* deassert UPLL_RESET */
1223	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1224
1225	mdelay(1);
1226
1227	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1228	if (r)
1229		return r;
1230
1231	/* assert UPLL_RESET again */
1232	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1233
1234	/* disable spread spectrum. */
1235	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1236
1237	/* set feedback divider */
1238	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1239
1240	/* set ref divider to 0 */
1241	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1242
1243	if (fb_div < 307200)
1244		WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1245	else
1246		WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1247
1248	/* set PDIV_A and PDIV_B */
1249	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1250		UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1251		~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1252
1253	/* give the PLL some time to settle */
1254	mdelay(15);
1255
1256	/* deassert PLL_RESET */
1257	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1258
1259	mdelay(15);
1260
1261	/* switch from bypass mode to normal mode */
1262	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1263
1264	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1265	if (r)
1266		return r;
1267
1268	/* switch VCLK and DCLK selection */
1269	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1270		VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1271		~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1272
1273	mdelay(100);
1274
1275	return 0;
1276}
1277
1278void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1279{
1280	int readrq;
1281	u16 v;
1282
1283	readrq = pcie_get_readrq(rdev->pdev);
1284	v = ffs(readrq) - 8;
1285	/* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1286	 * to avoid hangs or perfomance issues
1287	 */
1288	if ((v == 0) || (v == 6) || (v == 7))
1289		pcie_set_readrq(rdev->pdev, 512);
1290}
1291
1292void dce4_program_fmt(struct drm_encoder *encoder)
1293{
1294	struct drm_device *dev = encoder->dev;
1295	struct radeon_device *rdev = dev->dev_private;
1296	struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1297	struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1298	struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1299	int bpc = 0;
1300	u32 tmp = 0;
1301	enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1302
1303	if (connector) {
1304		struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1305		bpc = radeon_get_monitor_bpc(connector);
1306		dither = radeon_connector->dither;
1307	}
1308
1309	/* LVDS/eDP FMT is set up by atom */
1310	if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1311		return;
1312
1313	/* not needed for analog */
1314	if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1315	    (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1316		return;
1317
1318	if (bpc == 0)
1319		return;
1320
1321	switch (bpc) {
1322	case 6:
1323		if (dither == RADEON_FMT_DITHER_ENABLE)
1324			/* XXX sort out optimal dither settings */
1325			tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1326				FMT_SPATIAL_DITHER_EN);
1327		else
1328			tmp |= FMT_TRUNCATE_EN;
1329		break;
1330	case 8:
1331		if (dither == RADEON_FMT_DITHER_ENABLE)
1332			/* XXX sort out optimal dither settings */
1333			tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1334				FMT_RGB_RANDOM_ENABLE |
1335				FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1336		else
1337			tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1338		break;
1339	case 10:
1340	default:
1341		/* not needed */
1342		break;
1343	}
1344
1345	WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1346}
1347
1348static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1349{
1350	if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1351		return true;
1352	else
1353		return false;
1354}
1355
1356static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1357{
1358	u32 pos1, pos2;
1359
1360	pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1361	pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1362
1363	if (pos1 != pos2)
1364		return true;
1365	else
1366		return false;
1367}
1368
1369/**
1370 * dce4_wait_for_vblank - vblank wait asic callback.
1371 *
1372 * @rdev: radeon_device pointer
1373 * @crtc: crtc to wait for vblank on
1374 *
1375 * Wait for vblank on the requested crtc (evergreen+).
1376 */
1377void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1378{
1379	unsigned i = 0;
1380
1381	if (crtc >= rdev->num_crtc)
1382		return;
1383
1384	if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1385		return;
1386
1387	/* depending on when we hit vblank, we may be close to active; if so,
1388	 * wait for another frame.
1389	 */
1390	while (dce4_is_in_vblank(rdev, crtc)) {
1391		if (i++ % 100 == 0) {
1392			if (!dce4_is_counter_moving(rdev, crtc))
1393				break;
1394		}
1395	}
1396
1397	while (!dce4_is_in_vblank(rdev, crtc)) {
1398		if (i++ % 100 == 0) {
1399			if (!dce4_is_counter_moving(rdev, crtc))
1400				break;
1401		}
1402	}
1403}
1404
1405/**
1406 * evergreen_page_flip - pageflip callback.
1407 *
1408 * @rdev: radeon_device pointer
1409 * @crtc_id: crtc to cleanup pageflip on
1410 * @crtc_base: new address of the crtc (GPU MC address)
1411 * @async: asynchronous flip
1412 *
1413 * Triggers the actual pageflip by updating the primary
1414 * surface base address (evergreen+).
1415 */
1416void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1417			 bool async)
1418{
1419	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1420	struct drm_framebuffer *fb = radeon_crtc->base.primary->fb;
1421
1422	/* flip at hsync for async, default is vsync */
1423	WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1424	       async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1425	/* update pitch */
1426	WREG32(EVERGREEN_GRPH_PITCH + radeon_crtc->crtc_offset,
1427	       fb->pitches[0] / fb->format->cpp[0]);
1428	/* update the scanout addresses */
1429	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1430	       upper_32_bits(crtc_base));
1431	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1432	       (u32)crtc_base);
1433	/* post the write */
1434	RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1435}
1436
1437/**
1438 * evergreen_page_flip_pending - check if page flip is still pending
1439 *
1440 * @rdev: radeon_device pointer
1441 * @crtc_id: crtc to check
1442 *
1443 * Returns the current update pending status.
1444 */
1445bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1446{
1447	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1448
1449	/* Return current update_pending status: */
1450	return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1451		EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1452}
1453
1454/* get temperature in millidegrees */
1455int evergreen_get_temp(struct radeon_device *rdev)
1456{
1457	u32 temp, toffset;
1458	int actual_temp = 0;
1459
1460	if (rdev->family == CHIP_JUNIPER) {
1461		toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1462			TOFFSET_SHIFT;
1463		temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1464			TS0_ADC_DOUT_SHIFT;
1465
1466		if (toffset & 0x100)
1467			actual_temp = temp / 2 - (0x200 - toffset);
1468		else
1469			actual_temp = temp / 2 + toffset;
1470
1471		actual_temp = actual_temp * 1000;
1472
1473	} else {
1474		temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1475			ASIC_T_SHIFT;
1476
1477		if (temp & 0x400)
1478			actual_temp = -256;
1479		else if (temp & 0x200)
1480			actual_temp = 255;
1481		else if (temp & 0x100) {
1482			actual_temp = temp & 0x1ff;
1483			actual_temp |= ~0x1ff;
1484		} else
1485			actual_temp = temp & 0xff;
1486
1487		actual_temp = (actual_temp * 1000) / 2;
1488	}
1489
1490	return actual_temp;
1491}
1492
1493int sumo_get_temp(struct radeon_device *rdev)
1494{
1495	u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1496	int actual_temp = temp - 49;
1497
1498	return actual_temp * 1000;
1499}
1500
1501/**
1502 * sumo_pm_init_profile - Initialize power profiles callback.
1503 *
1504 * @rdev: radeon_device pointer
1505 *
1506 * Initialize the power states used in profile mode
1507 * (sumo, trinity, SI).
1508 * Used for profile mode only.
1509 */
1510void sumo_pm_init_profile(struct radeon_device *rdev)
1511{
1512	int idx;
1513
1514	/* default */
1515	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1516	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1517	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1518	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1519
1520	/* low,mid sh/mh */
1521	if (rdev->flags & RADEON_IS_MOBILITY)
1522		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1523	else
1524		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1525
1526	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1527	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1528	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1529	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1530
1531	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1532	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1533	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1534	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1535
1536	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1537	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1538	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1539	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1540
1541	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1542	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1543	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1544	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1545
1546	/* high sh/mh */
1547	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1548	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1549	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1550	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1551	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1552		rdev->pm.power_state[idx].num_clock_modes - 1;
1553
1554	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1555	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1556	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1557	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1558		rdev->pm.power_state[idx].num_clock_modes - 1;
1559}
1560
1561/**
1562 * btc_pm_init_profile - Initialize power profiles callback.
1563 *
1564 * @rdev: radeon_device pointer
1565 *
1566 * Initialize the power states used in profile mode
1567 * (BTC, cayman).
1568 * Used for profile mode only.
1569 */
1570void btc_pm_init_profile(struct radeon_device *rdev)
1571{
1572	int idx;
1573
1574	/* default */
1575	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1576	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1577	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1578	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1579	/* starting with BTC, there is one state that is used for both
1580	 * MH and SH.  Difference is that we always use the high clock index for
1581	 * mclk.
1582	 */
1583	if (rdev->flags & RADEON_IS_MOBILITY)
1584		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1585	else
1586		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1587	/* low sh */
1588	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1589	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1590	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1591	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1592	/* mid sh */
1593	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1594	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1595	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1596	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1597	/* high sh */
1598	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1599	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1600	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1601	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1602	/* low mh */
1603	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1604	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1605	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1606	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1607	/* mid mh */
1608	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1609	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1610	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1611	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1612	/* high mh */
1613	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1614	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1615	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1616	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1617}
1618
1619/**
1620 * evergreen_pm_misc - set additional pm hw parameters callback.
1621 *
1622 * @rdev: radeon_device pointer
1623 *
1624 * Set non-clock parameters associated with a power state
1625 * (voltage, etc.) (evergreen+).
1626 */
1627void evergreen_pm_misc(struct radeon_device *rdev)
1628{
1629	int req_ps_idx = rdev->pm.requested_power_state_index;
1630	int req_cm_idx = rdev->pm.requested_clock_mode_index;
1631	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1632	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1633
1634	if (voltage->type == VOLTAGE_SW) {
1635		/* 0xff0x are flags rather then an actual voltage */
1636		if ((voltage->voltage & 0xff00) == 0xff00)
1637			return;
1638		if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1639			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1640			rdev->pm.current_vddc = voltage->voltage;
1641			DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1642		}
1643
1644		/* starting with BTC, there is one state that is used for both
1645		 * MH and SH.  Difference is that we always use the high clock index for
1646		 * mclk and vddci.
1647		 */
1648		if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1649		    (rdev->family >= CHIP_BARTS) &&
1650		    rdev->pm.active_crtc_count &&
1651		    ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1652		     (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1653			voltage = &rdev->pm.power_state[req_ps_idx].
1654				clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1655
1656		/* 0xff0x are flags rather then an actual voltage */
1657		if ((voltage->vddci & 0xff00) == 0xff00)
1658			return;
1659		if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1660			radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1661			rdev->pm.current_vddci = voltage->vddci;
1662			DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1663		}
1664	}
1665}
1666
1667/**
1668 * evergreen_pm_prepare - pre-power state change callback.
1669 *
1670 * @rdev: radeon_device pointer
1671 *
1672 * Prepare for a power state change (evergreen+).
1673 */
1674void evergreen_pm_prepare(struct radeon_device *rdev)
1675{
1676	struct drm_device *ddev = rdev_to_drm(rdev);
1677	struct drm_crtc *crtc;
1678	struct radeon_crtc *radeon_crtc;
1679	u32 tmp;
1680
1681	/* disable any active CRTCs */
1682	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1683		radeon_crtc = to_radeon_crtc(crtc);
1684		if (radeon_crtc->enabled) {
1685			tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1686			tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1687			WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1688		}
1689	}
1690}
1691
1692/**
1693 * evergreen_pm_finish - post-power state change callback.
1694 *
1695 * @rdev: radeon_device pointer
1696 *
1697 * Clean up after a power state change (evergreen+).
1698 */
1699void evergreen_pm_finish(struct radeon_device *rdev)
1700{
1701	struct drm_device *ddev = rdev_to_drm(rdev);
1702	struct drm_crtc *crtc;
1703	struct radeon_crtc *radeon_crtc;
1704	u32 tmp;
1705
1706	/* enable any active CRTCs */
1707	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1708		radeon_crtc = to_radeon_crtc(crtc);
1709		if (radeon_crtc->enabled) {
1710			tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1711			tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1712			WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1713		}
1714	}
1715}
1716
1717/**
1718 * evergreen_hpd_sense - hpd sense callback.
1719 *
1720 * @rdev: radeon_device pointer
1721 * @hpd: hpd (hotplug detect) pin
1722 *
1723 * Checks if a digital monitor is connected (evergreen+).
1724 * Returns true if connected, false if not connected.
1725 */
1726bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1727{
1728	if (hpd == RADEON_HPD_NONE)
1729		return false;
1730
1731	return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1732}
1733
1734/**
1735 * evergreen_hpd_set_polarity - hpd set polarity callback.
1736 *
1737 * @rdev: radeon_device pointer
1738 * @hpd: hpd (hotplug detect) pin
1739 *
1740 * Set the polarity of the hpd pin (evergreen+).
1741 */
1742void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1743				enum radeon_hpd_id hpd)
1744{
1745	bool connected = evergreen_hpd_sense(rdev, hpd);
1746
1747	if (hpd == RADEON_HPD_NONE)
1748		return;
1749
1750	if (connected)
1751		WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1752	else
1753		WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1754}
1755
1756/**
1757 * evergreen_hpd_init - hpd setup callback.
1758 *
1759 * @rdev: radeon_device pointer
1760 *
1761 * Setup the hpd pins used by the card (evergreen+).
1762 * Enable the pin, set the polarity, and enable the hpd interrupts.
1763 */
1764void evergreen_hpd_init(struct radeon_device *rdev)
1765{
1766	struct drm_device *dev = rdev_to_drm(rdev);
1767	struct drm_connector *connector;
1768	unsigned enabled = 0;
1769	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1770		DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1771
1772	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1773		enum radeon_hpd_id hpd =
1774			to_radeon_connector(connector)->hpd.hpd;
1775
1776		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1777		    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1778			/* don't try to enable hpd on eDP or LVDS avoid breaking the
1779			 * aux dp channel on imac and help (but not completely fix)
1780			 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1781			 * also avoid interrupt storms during dpms.
1782			 */
1783			continue;
1784		}
1785
1786		if (hpd == RADEON_HPD_NONE)
1787			continue;
1788
1789		WREG32(DC_HPDx_CONTROL(hpd), tmp);
1790		enabled |= 1 << hpd;
1791
1792		radeon_hpd_set_polarity(rdev, hpd);
1793	}
1794	radeon_irq_kms_enable_hpd(rdev, enabled);
1795}
1796
1797/**
1798 * evergreen_hpd_fini - hpd tear down callback.
1799 *
1800 * @rdev: radeon_device pointer
1801 *
1802 * Tear down the hpd pins used by the card (evergreen+).
1803 * Disable the hpd interrupts.
1804 */
1805void evergreen_hpd_fini(struct radeon_device *rdev)
1806{
1807	struct drm_device *dev = rdev_to_drm(rdev);
1808	struct drm_connector *connector;
1809	unsigned disabled = 0;
1810
1811	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1812		enum radeon_hpd_id hpd =
1813			to_radeon_connector(connector)->hpd.hpd;
1814
1815		if (hpd == RADEON_HPD_NONE)
1816			continue;
1817
1818		WREG32(DC_HPDx_CONTROL(hpd), 0);
1819		disabled |= 1 << hpd;
1820	}
1821	radeon_irq_kms_disable_hpd(rdev, disabled);
1822}
1823
1824/* watermark setup */
1825
1826static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1827					struct radeon_crtc *radeon_crtc,
1828					struct drm_display_mode *mode,
1829					struct drm_display_mode *other_mode)
1830{
1831	u32 tmp, buffer_alloc, i;
1832	u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1833	/*
1834	 * Line Buffer Setup
1835	 * There are 3 line buffers, each one shared by 2 display controllers.
1836	 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1837	 * the display controllers.  The paritioning is done via one of four
1838	 * preset allocations specified in bits 2:0:
1839	 * first display controller
1840	 *  0 - first half of lb (3840 * 2)
1841	 *  1 - first 3/4 of lb (5760 * 2)
1842	 *  2 - whole lb (7680 * 2), other crtc must be disabled
1843	 *  3 - first 1/4 of lb (1920 * 2)
1844	 * second display controller
1845	 *  4 - second half of lb (3840 * 2)
1846	 *  5 - second 3/4 of lb (5760 * 2)
1847	 *  6 - whole lb (7680 * 2), other crtc must be disabled
1848	 *  7 - last 1/4 of lb (1920 * 2)
1849	 */
1850	/* this can get tricky if we have two large displays on a paired group
1851	 * of crtcs.  Ideally for multiple large displays we'd assign them to
1852	 * non-linked crtcs for maximum line buffer allocation.
1853	 */
1854	if (radeon_crtc->base.enabled && mode) {
1855		if (other_mode) {
1856			tmp = 0; /* 1/2 */
1857			buffer_alloc = 1;
1858		} else {
1859			tmp = 2; /* whole */
1860			buffer_alloc = 2;
1861		}
1862	} else {
1863		tmp = 0;
1864		buffer_alloc = 0;
1865	}
1866
1867	/* second controller of the pair uses second half of the lb */
1868	if (radeon_crtc->crtc_id % 2)
1869		tmp += 4;
1870	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1871
1872	if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1873		WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1874		       DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1875		for (i = 0; i < rdev->usec_timeout; i++) {
1876			if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1877			    DMIF_BUFFERS_ALLOCATED_COMPLETED)
1878				break;
1879			udelay(1);
1880		}
1881	}
1882
1883	if (radeon_crtc->base.enabled && mode) {
1884		switch (tmp) {
1885		case 0:
1886		case 4:
1887		default:
1888			if (ASIC_IS_DCE5(rdev))
1889				return 4096 * 2;
1890			else
1891				return 3840 * 2;
1892		case 1:
1893		case 5:
1894			if (ASIC_IS_DCE5(rdev))
1895				return 6144 * 2;
1896			else
1897				return 5760 * 2;
1898		case 2:
1899		case 6:
1900			if (ASIC_IS_DCE5(rdev))
1901				return 8192 * 2;
1902			else
1903				return 7680 * 2;
1904		case 3:
1905		case 7:
1906			if (ASIC_IS_DCE5(rdev))
1907				return 2048 * 2;
1908			else
1909				return 1920 * 2;
1910		}
1911	}
1912
1913	/* controller not enabled, so no lb used */
1914	return 0;
1915}
1916
1917u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1918{
1919	u32 tmp = RREG32(MC_SHARED_CHMAP);
1920
1921	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1922	case 0:
1923	default:
1924		return 1;
1925	case 1:
1926		return 2;
1927	case 2:
1928		return 4;
1929	case 3:
1930		return 8;
1931	}
1932}
1933
1934struct evergreen_wm_params {
1935	u32 dram_channels; /* number of dram channels */
1936	u32 yclk;          /* bandwidth per dram data pin in kHz */
1937	u32 sclk;          /* engine clock in kHz */
1938	u32 disp_clk;      /* display clock in kHz */
1939	u32 src_width;     /* viewport width */
1940	u32 active_time;   /* active display time in ns */
1941	u32 blank_time;    /* blank time in ns */
1942	bool interlaced;    /* mode is interlaced */
1943	fixed20_12 vsc;    /* vertical scale ratio */
1944	u32 num_heads;     /* number of active crtcs */
1945	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1946	u32 lb_size;       /* line buffer allocated to pipe */
1947	u32 vtaps;         /* vertical scaler taps */
1948};
1949
1950static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1951{
1952	/* Calculate DRAM Bandwidth and the part allocated to display. */
1953	fixed20_12 dram_efficiency; /* 0.7 */
1954	fixed20_12 yclk, dram_channels, bandwidth;
1955	fixed20_12 a;
1956
1957	a.full = dfixed_const(1000);
1958	yclk.full = dfixed_const(wm->yclk);
1959	yclk.full = dfixed_div(yclk, a);
1960	dram_channels.full = dfixed_const(wm->dram_channels * 4);
1961	a.full = dfixed_const(10);
1962	dram_efficiency.full = dfixed_const(7);
1963	dram_efficiency.full = dfixed_div(dram_efficiency, a);
1964	bandwidth.full = dfixed_mul(dram_channels, yclk);
1965	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1966
1967	return dfixed_trunc(bandwidth);
1968}
1969
1970static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1971{
1972	/* Calculate DRAM Bandwidth and the part allocated to display. */
1973	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1974	fixed20_12 yclk, dram_channels, bandwidth;
1975	fixed20_12 a;
1976
1977	a.full = dfixed_const(1000);
1978	yclk.full = dfixed_const(wm->yclk);
1979	yclk.full = dfixed_div(yclk, a);
1980	dram_channels.full = dfixed_const(wm->dram_channels * 4);
1981	a.full = dfixed_const(10);
1982	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1983	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1984	bandwidth.full = dfixed_mul(dram_channels, yclk);
1985	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1986
1987	return dfixed_trunc(bandwidth);
1988}
1989
1990static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1991{
1992	/* Calculate the display Data return Bandwidth */
1993	fixed20_12 return_efficiency; /* 0.8 */
1994	fixed20_12 sclk, bandwidth;
1995	fixed20_12 a;
1996
1997	a.full = dfixed_const(1000);
1998	sclk.full = dfixed_const(wm->sclk);
1999	sclk.full = dfixed_div(sclk, a);
2000	a.full = dfixed_const(10);
2001	return_efficiency.full = dfixed_const(8);
2002	return_efficiency.full = dfixed_div(return_efficiency, a);
2003	a.full = dfixed_const(32);
2004	bandwidth.full = dfixed_mul(a, sclk);
2005	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2006
2007	return dfixed_trunc(bandwidth);
2008}
2009
2010static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2011{
2012	/* Calculate the DMIF Request Bandwidth */
2013	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2014	fixed20_12 disp_clk, bandwidth;
2015	fixed20_12 a;
2016
2017	a.full = dfixed_const(1000);
2018	disp_clk.full = dfixed_const(wm->disp_clk);
2019	disp_clk.full = dfixed_div(disp_clk, a);
2020	a.full = dfixed_const(10);
2021	disp_clk_request_efficiency.full = dfixed_const(8);
2022	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2023	a.full = dfixed_const(32);
2024	bandwidth.full = dfixed_mul(a, disp_clk);
2025	bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2026
2027	return dfixed_trunc(bandwidth);
2028}
2029
2030static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2031{
2032	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2033	u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2034	u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2035	u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2036
2037	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2038}
2039
2040static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2041{
2042	/* Calculate the display mode Average Bandwidth
2043	 * DisplayMode should contain the source and destination dimensions,
2044	 * timing, etc.
2045	 */
2046	fixed20_12 bpp;
2047	fixed20_12 line_time;
2048	fixed20_12 src_width;
2049	fixed20_12 bandwidth;
2050	fixed20_12 a;
2051
2052	a.full = dfixed_const(1000);
2053	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2054	line_time.full = dfixed_div(line_time, a);
2055	bpp.full = dfixed_const(wm->bytes_per_pixel);
2056	src_width.full = dfixed_const(wm->src_width);
2057	bandwidth.full = dfixed_mul(src_width, bpp);
2058	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2059	bandwidth.full = dfixed_div(bandwidth, line_time);
2060
2061	return dfixed_trunc(bandwidth);
2062}
2063
2064static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2065{
2066	/* First calcualte the latency in ns */
2067	u32 mc_latency = 2000; /* 2000 ns. */
2068	u32 available_bandwidth = evergreen_available_bandwidth(wm);
2069	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2070	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2071	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2072	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2073		(wm->num_heads * cursor_line_pair_return_time);
2074	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2075	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2076	fixed20_12 a, b, c;
2077
2078	if (wm->num_heads == 0)
2079		return 0;
2080
2081	a.full = dfixed_const(2);
2082	b.full = dfixed_const(1);
2083	if ((wm->vsc.full > a.full) ||
2084	    ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2085	    (wm->vtaps >= 5) ||
2086	    ((wm->vsc.full >= a.full) && wm->interlaced))
2087		max_src_lines_per_dst_line = 4;
2088	else
2089		max_src_lines_per_dst_line = 2;
2090
2091	a.full = dfixed_const(available_bandwidth);
2092	b.full = dfixed_const(wm->num_heads);
2093	a.full = dfixed_div(a, b);
2094
2095	lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2096
2097	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2098	b.full = dfixed_const(1000);
2099	c.full = dfixed_const(lb_fill_bw);
2100	b.full = dfixed_div(c, b);
2101	a.full = dfixed_div(a, b);
2102	line_fill_time = dfixed_trunc(a);
2103
2104	if (line_fill_time < wm->active_time)
2105		return latency;
2106	else
2107		return latency + (line_fill_time - wm->active_time);
2108
2109}
2110
2111static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2112{
2113	if (evergreen_average_bandwidth(wm) <=
2114	    (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2115		return true;
2116	else
2117		return false;
2118};
2119
2120static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2121{
2122	if (evergreen_average_bandwidth(wm) <=
2123	    (evergreen_available_bandwidth(wm) / wm->num_heads))
2124		return true;
2125	else
2126		return false;
2127};
2128
2129static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2130{
2131	u32 lb_partitions = wm->lb_size / wm->src_width;
2132	u32 line_time = wm->active_time + wm->blank_time;
2133	u32 latency_tolerant_lines;
2134	u32 latency_hiding;
2135	fixed20_12 a;
2136
2137	a.full = dfixed_const(1);
2138	if (wm->vsc.full > a.full)
2139		latency_tolerant_lines = 1;
2140	else {
2141		if (lb_partitions <= (wm->vtaps + 1))
2142			latency_tolerant_lines = 1;
2143		else
2144			latency_tolerant_lines = 2;
2145	}
2146
2147	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2148
2149	if (evergreen_latency_watermark(wm) <= latency_hiding)
2150		return true;
2151	else
2152		return false;
2153}
2154
2155static void evergreen_program_watermarks(struct radeon_device *rdev,
2156					 struct radeon_crtc *radeon_crtc,
2157					 u32 lb_size, u32 num_heads)
2158{
2159	struct drm_display_mode *mode = &radeon_crtc->base.mode;
2160	struct evergreen_wm_params wm_low, wm_high;
2161	u32 dram_channels;
2162	u32 active_time;
2163	u32 line_time = 0;
2164	u32 latency_watermark_a = 0, latency_watermark_b = 0;
2165	u32 priority_a_mark = 0, priority_b_mark = 0;
2166	u32 priority_a_cnt = PRIORITY_OFF;
2167	u32 priority_b_cnt = PRIORITY_OFF;
2168	u32 pipe_offset = radeon_crtc->crtc_id * 16;
2169	u32 tmp, arb_control3;
2170	fixed20_12 a, b, c;
2171
2172	if (radeon_crtc->base.enabled && num_heads && mode) {
2173		active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2174					    (u32)mode->clock);
2175		line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2176					  (u32)mode->clock);
2177		line_time = min(line_time, (u32)65535);
2178		priority_a_cnt = 0;
2179		priority_b_cnt = 0;
2180		dram_channels = evergreen_get_number_of_dram_channels(rdev);
2181
2182		/* watermark for high clocks */
2183		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2184			wm_high.yclk =
2185				radeon_dpm_get_mclk(rdev, false) * 10;
2186			wm_high.sclk =
2187				radeon_dpm_get_sclk(rdev, false) * 10;
2188		} else {
2189			wm_high.yclk = rdev->pm.current_mclk * 10;
2190			wm_high.sclk = rdev->pm.current_sclk * 10;
2191		}
2192
2193		wm_high.disp_clk = mode->clock;
2194		wm_high.src_width = mode->crtc_hdisplay;
2195		wm_high.active_time = active_time;
2196		wm_high.blank_time = line_time - wm_high.active_time;
2197		wm_high.interlaced = false;
2198		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2199			wm_high.interlaced = true;
2200		wm_high.vsc = radeon_crtc->vsc;
2201		wm_high.vtaps = 1;
2202		if (radeon_crtc->rmx_type != RMX_OFF)
2203			wm_high.vtaps = 2;
2204		wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2205		wm_high.lb_size = lb_size;
2206		wm_high.dram_channels = dram_channels;
2207		wm_high.num_heads = num_heads;
2208
2209		/* watermark for low clocks */
2210		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2211			wm_low.yclk =
2212				radeon_dpm_get_mclk(rdev, true) * 10;
2213			wm_low.sclk =
2214				radeon_dpm_get_sclk(rdev, true) * 10;
2215		} else {
2216			wm_low.yclk = rdev->pm.current_mclk * 10;
2217			wm_low.sclk = rdev->pm.current_sclk * 10;
2218		}
2219
2220		wm_low.disp_clk = mode->clock;
2221		wm_low.src_width = mode->crtc_hdisplay;
2222		wm_low.active_time = active_time;
2223		wm_low.blank_time = line_time - wm_low.active_time;
2224		wm_low.interlaced = false;
2225		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2226			wm_low.interlaced = true;
2227		wm_low.vsc = radeon_crtc->vsc;
2228		wm_low.vtaps = 1;
2229		if (radeon_crtc->rmx_type != RMX_OFF)
2230			wm_low.vtaps = 2;
2231		wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2232		wm_low.lb_size = lb_size;
2233		wm_low.dram_channels = dram_channels;
2234		wm_low.num_heads = num_heads;
2235
2236		/* set for high clocks */
2237		latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2238		/* set for low clocks */
2239		latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2240
2241		/* possibly force display priority to high */
2242		/* should really do this at mode validation time... */
2243		if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2244		    !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2245		    !evergreen_check_latency_hiding(&wm_high) ||
2246		    (rdev->disp_priority == 2)) {
2247			DRM_DEBUG_KMS("force priority a to high\n");
2248			priority_a_cnt |= PRIORITY_ALWAYS_ON;
2249		}
2250		if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2251		    !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2252		    !evergreen_check_latency_hiding(&wm_low) ||
2253		    (rdev->disp_priority == 2)) {
2254			DRM_DEBUG_KMS("force priority b to high\n");
2255			priority_b_cnt |= PRIORITY_ALWAYS_ON;
2256		}
2257
2258		a.full = dfixed_const(1000);
2259		b.full = dfixed_const(mode->clock);
2260		b.full = dfixed_div(b, a);
2261		c.full = dfixed_const(latency_watermark_a);
2262		c.full = dfixed_mul(c, b);
2263		c.full = dfixed_mul(c, radeon_crtc->hsc);
2264		c.full = dfixed_div(c, a);
2265		a.full = dfixed_const(16);
2266		c.full = dfixed_div(c, a);
2267		priority_a_mark = dfixed_trunc(c);
2268		priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2269
2270		a.full = dfixed_const(1000);
2271		b.full = dfixed_const(mode->clock);
2272		b.full = dfixed_div(b, a);
2273		c.full = dfixed_const(latency_watermark_b);
2274		c.full = dfixed_mul(c, b);
2275		c.full = dfixed_mul(c, radeon_crtc->hsc);
2276		c.full = dfixed_div(c, a);
2277		a.full = dfixed_const(16);
2278		c.full = dfixed_div(c, a);
2279		priority_b_mark = dfixed_trunc(c);
2280		priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2281
2282		/* Save number of lines the linebuffer leads before the scanout */
2283		radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2284	}
2285
2286	/* select wm A */
2287	arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2288	tmp = arb_control3;
2289	tmp &= ~LATENCY_WATERMARK_MASK(3);
2290	tmp |= LATENCY_WATERMARK_MASK(1);
2291	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2292	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2293	       (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2294		LATENCY_HIGH_WATERMARK(line_time)));
2295	/* select wm B */
2296	tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2297	tmp &= ~LATENCY_WATERMARK_MASK(3);
2298	tmp |= LATENCY_WATERMARK_MASK(2);
2299	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2300	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2301	       (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2302		LATENCY_HIGH_WATERMARK(line_time)));
2303	/* restore original selection */
2304	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2305
2306	/* write the priority marks */
2307	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2308	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2309
2310	/* save values for DPM */
2311	radeon_crtc->line_time = line_time;
2312	radeon_crtc->wm_high = latency_watermark_a;
2313	radeon_crtc->wm_low = latency_watermark_b;
2314}
2315
2316/**
2317 * evergreen_bandwidth_update - update display watermarks callback.
2318 *
2319 * @rdev: radeon_device pointer
2320 *
2321 * Update the display watermarks based on the requested mode(s)
2322 * (evergreen+).
2323 */
2324void evergreen_bandwidth_update(struct radeon_device *rdev)
2325{
2326	struct drm_display_mode *mode0 = NULL;
2327	struct drm_display_mode *mode1 = NULL;
2328	u32 num_heads = 0, lb_size;
2329	int i;
2330
2331	if (!rdev->mode_info.mode_config_initialized)
2332		return;
2333
2334	radeon_update_display_priority(rdev);
2335
2336	for (i = 0; i < rdev->num_crtc; i++) {
2337		if (rdev->mode_info.crtcs[i]->base.enabled)
2338			num_heads++;
2339	}
2340	for (i = 0; i < rdev->num_crtc; i += 2) {
2341		mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2342		mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2343		lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2344		evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2345		lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2346		evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2347	}
2348}
2349
2350/**
2351 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2352 *
2353 * @rdev: radeon_device pointer
2354 *
2355 * Wait for the MC (memory controller) to be idle.
2356 * (evergreen+).
2357 * Returns 0 if the MC is idle, -1 if not.
2358 */
2359int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2360{
2361	unsigned i;
2362	u32 tmp;
2363
2364	for (i = 0; i < rdev->usec_timeout; i++) {
2365		/* read MC_STATUS */
2366		tmp = RREG32(SRBM_STATUS) & 0x1F00;
2367		if (!tmp)
2368			return 0;
2369		udelay(1);
2370	}
2371	return -1;
2372}
2373
2374/*
2375 * GART
2376 */
2377void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2378{
2379	unsigned i;
2380	u32 tmp;
2381
2382	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2383
2384	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2385	for (i = 0; i < rdev->usec_timeout; i++) {
2386		/* read MC_STATUS */
2387		tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2388		tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2389		if (tmp == 2) {
2390			pr_warn("[drm] r600 flush TLB failed\n");
2391			return;
2392		}
2393		if (tmp) {
2394			return;
2395		}
2396		udelay(1);
2397	}
2398}
2399
2400static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2401{
2402	u32 tmp;
2403	int r;
2404
2405	if (rdev->gart.robj == NULL) {
2406		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2407		return -EINVAL;
2408	}
2409	r = radeon_gart_table_vram_pin(rdev);
2410	if (r)
2411		return r;
2412	/* Setup L2 cache */
2413	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2414				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2415				EFFECTIVE_L2_QUEUE_SIZE(7));
2416	WREG32(VM_L2_CNTL2, 0);
2417	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2418	/* Setup TLB control */
2419	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2420		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2421		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2422		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2423	if (rdev->flags & RADEON_IS_IGP) {
2424		WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2425		WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2426		WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2427	} else {
2428		WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2429		WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2430		WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2431		if ((rdev->family == CHIP_JUNIPER) ||
2432		    (rdev->family == CHIP_CYPRESS) ||
2433		    (rdev->family == CHIP_HEMLOCK) ||
2434		    (rdev->family == CHIP_BARTS))
2435			WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2436	}
2437	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2438	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2439	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2440	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2441	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2442	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2443	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2444	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2445				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2446	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2447			(u32)(rdev->dummy_page.addr >> 12));
2448	WREG32(VM_CONTEXT1_CNTL, 0);
2449
2450	evergreen_pcie_gart_tlb_flush(rdev);
2451	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2452		 (unsigned)(rdev->mc.gtt_size >> 20),
2453		 (unsigned long long)rdev->gart.table_addr);
2454	rdev->gart.ready = true;
2455	return 0;
2456}
2457
2458static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2459{
2460	u32 tmp;
2461
2462	/* Disable all tables */
2463	WREG32(VM_CONTEXT0_CNTL, 0);
2464	WREG32(VM_CONTEXT1_CNTL, 0);
2465
2466	/* Setup L2 cache */
2467	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2468				EFFECTIVE_L2_QUEUE_SIZE(7));
2469	WREG32(VM_L2_CNTL2, 0);
2470	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2471	/* Setup TLB control */
2472	tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2473	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2474	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2475	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2476	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2477	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2478	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2479	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2480	radeon_gart_table_vram_unpin(rdev);
2481}
2482
2483static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2484{
2485	evergreen_pcie_gart_disable(rdev);
2486	radeon_gart_table_vram_free(rdev);
2487	radeon_gart_fini(rdev);
2488}
2489
2490
2491static void evergreen_agp_enable(struct radeon_device *rdev)
2492{
2493	u32 tmp;
2494
2495	/* Setup L2 cache */
2496	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2497				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2498				EFFECTIVE_L2_QUEUE_SIZE(7));
2499	WREG32(VM_L2_CNTL2, 0);
2500	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2501	/* Setup TLB control */
2502	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2503		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2504		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2505		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2506	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2507	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2508	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2509	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2510	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2511	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2512	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2513	WREG32(VM_CONTEXT0_CNTL, 0);
2514	WREG32(VM_CONTEXT1_CNTL, 0);
2515}
2516
2517static const unsigned ni_dig_offsets[] = {
 
2518	NI_DIG0_REGISTER_OFFSET,
2519	NI_DIG1_REGISTER_OFFSET,
2520	NI_DIG2_REGISTER_OFFSET,
2521	NI_DIG3_REGISTER_OFFSET,
2522	NI_DIG4_REGISTER_OFFSET,
2523	NI_DIG5_REGISTER_OFFSET
2524};
2525
2526static const unsigned ni_tx_offsets[] = {
 
2527	NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2528	NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2529	NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2530	NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2531	NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2532	NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2533};
2534
2535static const unsigned evergreen_dp_offsets[] = {
 
2536	EVERGREEN_DP0_REGISTER_OFFSET,
2537	EVERGREEN_DP1_REGISTER_OFFSET,
2538	EVERGREEN_DP2_REGISTER_OFFSET,
2539	EVERGREEN_DP3_REGISTER_OFFSET,
2540	EVERGREEN_DP4_REGISTER_OFFSET,
2541	EVERGREEN_DP5_REGISTER_OFFSET
2542};
2543
2544static const unsigned evergreen_disp_int_status[] = {
 
2545	DISP_INTERRUPT_STATUS,
2546	DISP_INTERRUPT_STATUS_CONTINUE,
2547	DISP_INTERRUPT_STATUS_CONTINUE2,
2548	DISP_INTERRUPT_STATUS_CONTINUE3,
2549	DISP_INTERRUPT_STATUS_CONTINUE4,
2550	DISP_INTERRUPT_STATUS_CONTINUE5
2551};
2552
2553/*
2554 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2555 * We go from crtc to connector and it is not relible  since it
2556 * should be an opposite direction .If crtc is enable then
2557 * find the dig_fe which selects this crtc and insure that it enable.
2558 * if such dig_fe is found then find dig_be which selects found dig_be and
2559 * insure that it enable and in DP_SST mode.
2560 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2561 * from dp symbols clocks .
2562 */
2563static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2564					       unsigned crtc_id, unsigned *ret_dig_fe)
2565{
2566	unsigned i;
2567	unsigned dig_fe;
2568	unsigned dig_be;
2569	unsigned dig_en_be;
2570	unsigned uniphy_pll;
2571	unsigned digs_fe_selected;
2572	unsigned dig_be_mode;
2573	unsigned dig_fe_mask;
2574	bool is_enabled = false;
2575	bool found_crtc = false;
2576
2577	/* loop through all running dig_fe to find selected crtc */
2578	for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2579		dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2580		if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2581		    crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2582			/* found running pipe */
2583			found_crtc = true;
2584			dig_fe_mask = 1 << i;
2585			dig_fe = i;
2586			break;
2587		}
2588	}
2589
2590	if (found_crtc) {
2591		/* loop through all running dig_be to find selected dig_fe */
2592		for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2593			dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2594			/* if dig_fe_selected by dig_be? */
2595			digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2596			dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2597			if (dig_fe_mask &  digs_fe_selected &&
2598			    /* if dig_be in sst mode? */
2599			    dig_be_mode == NI_DIG_BE_DPSST) {
2600				dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2601						   ni_dig_offsets[i]);
2602				uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2603						    ni_tx_offsets[i]);
2604				/* dig_be enable and tx is running */
2605				if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2606				    dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2607				    uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2608					is_enabled = true;
2609					*ret_dig_fe = dig_fe;
2610					break;
2611				}
2612			}
2613		}
2614	}
2615
2616	return is_enabled;
2617}
2618
2619/*
2620 * Blank dig when in dp sst mode
2621 * Dig ignores crtc timing
2622 */
2623static void evergreen_blank_dp_output(struct radeon_device *rdev,
2624				      unsigned dig_fe)
2625{
2626	unsigned stream_ctrl;
2627	unsigned fifo_ctrl;
2628	unsigned counter = 0;
2629
2630	if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2631		DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2632		return;
2633	}
2634
2635	stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2636			     evergreen_dp_offsets[dig_fe]);
2637	if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2638		DRM_ERROR("dig %d , should be enable\n", dig_fe);
2639		return;
2640	}
2641
2642	stream_ctrl &= ~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2643	WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2644	       evergreen_dp_offsets[dig_fe], stream_ctrl);
2645
2646	stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2647			     evergreen_dp_offsets[dig_fe]);
2648	while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2649		msleep(1);
2650		counter++;
2651		stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2652				     evergreen_dp_offsets[dig_fe]);
2653	}
2654	if (counter >= 32)
2655		DRM_ERROR("counter exceeds %d\n", counter);
2656
2657	fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2658	fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2659	WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2660
2661}
2662
2663void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2664{
2665	u32 crtc_enabled, tmp, frame_count, blackout;
2666	int i, j;
2667	unsigned dig_fe;
2668
2669	if (!ASIC_IS_NODCE(rdev)) {
2670		save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2671		save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2672
2673		/* disable VGA render */
2674		WREG32(VGA_RENDER_CONTROL, 0);
2675	}
2676	/* blank the display controllers */
2677	for (i = 0; i < rdev->num_crtc; i++) {
2678		crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2679		if (crtc_enabled) {
2680			save->crtc_enabled[i] = true;
2681			if (ASIC_IS_DCE6(rdev)) {
2682				tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2683				if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2684					radeon_wait_for_vblank(rdev, i);
2685					WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2686					tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2687					WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2688					WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2689				}
2690			} else {
2691				tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2692				if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2693					radeon_wait_for_vblank(rdev, i);
2694					WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2695					tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2696					WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2697					WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2698				}
2699			}
2700			/* wait for the next frame */
2701			frame_count = radeon_get_vblank_counter(rdev, i);
2702			for (j = 0; j < rdev->usec_timeout; j++) {
2703				if (radeon_get_vblank_counter(rdev, i) != frame_count)
2704					break;
2705				udelay(1);
2706			}
2707			/*we should disable dig if it drives dp sst*/
2708			/*but we are in radeon_device_init and the topology is unknown*/
2709			/*and it is available after radeon_modeset_init*/
2710			/*the following method radeon_atom_encoder_dpms_dig*/
2711			/*does the job if we initialize it properly*/
2712			/*for now we do it this manually*/
2713			/**/
2714			if (ASIC_IS_DCE5(rdev) &&
2715			    evergreen_is_dp_sst_stream_enabled(rdev, i, &dig_fe))
2716				evergreen_blank_dp_output(rdev, dig_fe);
2717			/*we could remove 6 lines below*/
2718			/* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2719			WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2720			tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2721			tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2722			WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2723			WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2724			save->crtc_enabled[i] = false;
2725			/* ***** */
2726		} else {
2727			save->crtc_enabled[i] = false;
2728		}
2729	}
2730
2731	radeon_mc_wait_for_idle(rdev);
2732
2733	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2734	if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2735		/* Block CPU access */
2736		WREG32(BIF_FB_EN, 0);
2737		/* blackout the MC */
2738		blackout &= ~BLACKOUT_MODE_MASK;
2739		WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2740	}
2741	/* wait for the MC to settle */
2742	udelay(100);
2743
2744	/* lock double buffered regs */
2745	for (i = 0; i < rdev->num_crtc; i++) {
2746		if (save->crtc_enabled[i]) {
2747			tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2748			if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2749				tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2750				WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2751			}
2752			tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2753			if (!(tmp & 1)) {
2754				tmp |= 1;
2755				WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2756			}
2757		}
2758	}
2759}
2760
2761void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2762{
2763	u32 tmp, frame_count;
2764	int i, j;
2765
2766	/* update crtc base addresses */
2767	for (i = 0; i < rdev->num_crtc; i++) {
2768		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2769		       upper_32_bits(rdev->mc.vram_start));
2770		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2771		       upper_32_bits(rdev->mc.vram_start));
2772		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2773		       (u32)rdev->mc.vram_start);
2774		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2775		       (u32)rdev->mc.vram_start);
2776	}
2777
2778	if (!ASIC_IS_NODCE(rdev)) {
2779		WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2780		WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2781	}
2782
2783	/* unlock regs and wait for update */
2784	for (i = 0; i < rdev->num_crtc; i++) {
2785		if (save->crtc_enabled[i]) {
2786			tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2787			if ((tmp & 0x7) != 0) {
2788				tmp &= ~0x7;
2789				WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2790			}
2791			tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2792			if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2793				tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2794				WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2795			}
2796			tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2797			if (tmp & 1) {
2798				tmp &= ~1;
2799				WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2800			}
2801			for (j = 0; j < rdev->usec_timeout; j++) {
2802				tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2803				if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2804					break;
2805				udelay(1);
2806			}
2807		}
2808	}
2809
2810	/* unblackout the MC */
2811	tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2812	tmp &= ~BLACKOUT_MODE_MASK;
2813	WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2814	/* allow CPU access */
2815	WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2816
2817	for (i = 0; i < rdev->num_crtc; i++) {
2818		if (save->crtc_enabled[i]) {
2819			if (ASIC_IS_DCE6(rdev)) {
2820				tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2821				tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2822				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2823				WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2824				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2825			} else {
2826				tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2827				tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2828				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2829				WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2830				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2831			}
2832			/* wait for the next frame */
2833			frame_count = radeon_get_vblank_counter(rdev, i);
2834			for (j = 0; j < rdev->usec_timeout; j++) {
2835				if (radeon_get_vblank_counter(rdev, i) != frame_count)
2836					break;
2837				udelay(1);
2838			}
2839		}
2840	}
2841	if (!ASIC_IS_NODCE(rdev)) {
2842		/* Unlock vga access */
2843		WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2844		mdelay(1);
2845		WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2846	}
2847}
2848
2849void evergreen_mc_program(struct radeon_device *rdev)
2850{
2851	struct evergreen_mc_save save;
2852	u32 tmp;
2853	int i, j;
2854
2855	/* Initialize HDP */
2856	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2857		WREG32((0x2c14 + j), 0x00000000);
2858		WREG32((0x2c18 + j), 0x00000000);
2859		WREG32((0x2c1c + j), 0x00000000);
2860		WREG32((0x2c20 + j), 0x00000000);
2861		WREG32((0x2c24 + j), 0x00000000);
2862	}
2863	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2864
2865	evergreen_mc_stop(rdev, &save);
2866	if (evergreen_mc_wait_for_idle(rdev)) {
2867		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2868	}
2869	/* Lockout access through VGA aperture*/
2870	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2871	/* Update configuration */
2872	if (rdev->flags & RADEON_IS_AGP) {
2873		if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2874			/* VRAM before AGP */
2875			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2876				rdev->mc.vram_start >> 12);
2877			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2878				rdev->mc.gtt_end >> 12);
2879		} else {
2880			/* VRAM after AGP */
2881			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2882				rdev->mc.gtt_start >> 12);
2883			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2884				rdev->mc.vram_end >> 12);
2885		}
2886	} else {
2887		WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2888			rdev->mc.vram_start >> 12);
2889		WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2890			rdev->mc.vram_end >> 12);
2891	}
2892	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2893	/* llano/ontario only */
2894	if ((rdev->family == CHIP_PALM) ||
2895	    (rdev->family == CHIP_SUMO) ||
2896	    (rdev->family == CHIP_SUMO2)) {
2897		tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2898		tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2899		tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2900		WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2901	}
2902	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2903	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2904	WREG32(MC_VM_FB_LOCATION, tmp);
2905	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2906	WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2907	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2908	if (rdev->flags & RADEON_IS_AGP) {
2909		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2910		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2911		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2912	} else {
2913		WREG32(MC_VM_AGP_BASE, 0);
2914		WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2915		WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2916	}
2917	if (evergreen_mc_wait_for_idle(rdev)) {
2918		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2919	}
2920	evergreen_mc_resume(rdev, &save);
2921	/* we need to own VRAM, so turn off the VGA renderer here
2922	 * to stop it overwriting our objects */
2923	rv515_vga_render_disable(rdev);
2924}
2925
2926/*
2927 * CP.
2928 */
2929void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2930{
2931	struct radeon_ring *ring = &rdev->ring[ib->ring];
2932	u32 next_rptr;
2933
2934	/* set to DX10/11 mode */
2935	radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2936	radeon_ring_write(ring, 1);
2937
2938	if (ring->rptr_save_reg) {
2939		next_rptr = ring->wptr + 3 + 4;
2940		radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2941		radeon_ring_write(ring, ((ring->rptr_save_reg - 
2942					  PACKET3_SET_CONFIG_REG_START) >> 2));
2943		radeon_ring_write(ring, next_rptr);
2944	} else if (rdev->wb.enabled) {
2945		next_rptr = ring->wptr + 5 + 4;
2946		radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2947		radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2948		radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2949		radeon_ring_write(ring, next_rptr);
2950		radeon_ring_write(ring, 0);
2951	}
2952
2953	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2954	radeon_ring_write(ring,
2955#ifdef __BIG_ENDIAN
2956			  (2 << 0) |
2957#endif
2958			  (ib->gpu_addr & 0xFFFFFFFC));
2959	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2960	radeon_ring_write(ring, ib->length_dw);
2961}
2962
2963
2964static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2965{
2966	const __be32 *fw_data;
2967	int i;
2968
2969	if (!rdev->me_fw || !rdev->pfp_fw)
2970		return -EINVAL;
2971
2972	r700_cp_stop(rdev);
2973	WREG32(CP_RB_CNTL,
2974#ifdef __BIG_ENDIAN
2975	       BUF_SWAP_32BIT |
2976#endif
2977	       RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2978
2979	fw_data = (const __be32 *)rdev->pfp_fw->data;
2980	WREG32(CP_PFP_UCODE_ADDR, 0);
2981	for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2982		WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2983	WREG32(CP_PFP_UCODE_ADDR, 0);
2984
2985	fw_data = (const __be32 *)rdev->me_fw->data;
2986	WREG32(CP_ME_RAM_WADDR, 0);
2987	for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2988		WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2989
2990	WREG32(CP_PFP_UCODE_ADDR, 0);
2991	WREG32(CP_ME_RAM_WADDR, 0);
2992	WREG32(CP_ME_RAM_RADDR, 0);
2993	return 0;
2994}
2995
2996static int evergreen_cp_start(struct radeon_device *rdev)
2997{
2998	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2999	int r, i;
3000	uint32_t cp_me;
3001
3002	r = radeon_ring_lock(rdev, ring, 7);
3003	if (r) {
3004		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3005		return r;
3006	}
3007	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3008	radeon_ring_write(ring, 0x1);
3009	radeon_ring_write(ring, 0x0);
3010	radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3011	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3012	radeon_ring_write(ring, 0);
3013	radeon_ring_write(ring, 0);
3014	radeon_ring_unlock_commit(rdev, ring, false);
3015
3016	cp_me = 0xff;
3017	WREG32(CP_ME_CNTL, cp_me);
3018
3019	r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3020	if (r) {
3021		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3022		return r;
3023	}
3024
3025	/* setup clear context state */
3026	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3027	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3028
3029	for (i = 0; i < evergreen_default_size; i++)
3030		radeon_ring_write(ring, evergreen_default_state[i]);
3031
3032	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3033	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3034
3035	/* set clear context state */
3036	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3037	radeon_ring_write(ring, 0);
3038
3039	/* SQ_VTX_BASE_VTX_LOC */
3040	radeon_ring_write(ring, 0xc0026f00);
3041	radeon_ring_write(ring, 0x00000000);
3042	radeon_ring_write(ring, 0x00000000);
3043	radeon_ring_write(ring, 0x00000000);
3044
3045	/* Clear consts */
3046	radeon_ring_write(ring, 0xc0036f00);
3047	radeon_ring_write(ring, 0x00000bc4);
3048	radeon_ring_write(ring, 0xffffffff);
3049	radeon_ring_write(ring, 0xffffffff);
3050	radeon_ring_write(ring, 0xffffffff);
3051
3052	radeon_ring_write(ring, 0xc0026900);
3053	radeon_ring_write(ring, 0x00000316);
3054	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3055	radeon_ring_write(ring, 0x00000010); /*  */
3056
3057	radeon_ring_unlock_commit(rdev, ring, false);
3058
3059	return 0;
3060}
3061
3062static int evergreen_cp_resume(struct radeon_device *rdev)
3063{
3064	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3065	u32 tmp;
3066	u32 rb_bufsz;
3067	int r;
3068
3069	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3070	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3071				 SOFT_RESET_PA |
3072				 SOFT_RESET_SH |
3073				 SOFT_RESET_VGT |
3074				 SOFT_RESET_SPI |
3075				 SOFT_RESET_SX));
3076	RREG32(GRBM_SOFT_RESET);
3077	mdelay(15);
3078	WREG32(GRBM_SOFT_RESET, 0);
3079	RREG32(GRBM_SOFT_RESET);
3080
3081	/* Set ring buffer size */
3082	rb_bufsz = order_base_2(ring->ring_size / 8);
3083	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3084#ifdef __BIG_ENDIAN
3085	tmp |= BUF_SWAP_32BIT;
3086#endif
3087	WREG32(CP_RB_CNTL, tmp);
3088	WREG32(CP_SEM_WAIT_TIMER, 0x0);
3089	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3090
3091	/* Set the write pointer delay */
3092	WREG32(CP_RB_WPTR_DELAY, 0);
3093
3094	/* Initialize the ring buffer's read and write pointers */
3095	WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3096	WREG32(CP_RB_RPTR_WR, 0);
3097	ring->wptr = 0;
3098	WREG32(CP_RB_WPTR, ring->wptr);
3099
3100	/* set the wb address whether it's enabled or not */
3101	WREG32(CP_RB_RPTR_ADDR,
3102	       ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3103	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3104	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3105
3106	if (rdev->wb.enabled)
3107		WREG32(SCRATCH_UMSK, 0xff);
3108	else {
3109		tmp |= RB_NO_UPDATE;
3110		WREG32(SCRATCH_UMSK, 0);
3111	}
3112
3113	mdelay(1);
3114	WREG32(CP_RB_CNTL, tmp);
3115
3116	WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3117	WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3118
3119	evergreen_cp_start(rdev);
3120	ring->ready = true;
3121	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3122	if (r) {
3123		ring->ready = false;
3124		return r;
3125	}
3126	return 0;
3127}
3128
3129/*
3130 * Core functions
3131 */
3132static void evergreen_gpu_init(struct radeon_device *rdev)
3133{
3134	u32 gb_addr_config;
3135	u32 mc_arb_ramcfg;
3136	u32 sx_debug_1;
3137	u32 smx_dc_ctl0;
3138	u32 sq_config;
3139	u32 sq_lds_resource_mgmt;
3140	u32 sq_gpr_resource_mgmt_1;
3141	u32 sq_gpr_resource_mgmt_2;
3142	u32 sq_gpr_resource_mgmt_3;
3143	u32 sq_thread_resource_mgmt;
3144	u32 sq_thread_resource_mgmt_2;
3145	u32 sq_stack_resource_mgmt_1;
3146	u32 sq_stack_resource_mgmt_2;
3147	u32 sq_stack_resource_mgmt_3;
3148	u32 vgt_cache_invalidation;
3149	u32 hdp_host_path_cntl, tmp;
3150	u32 disabled_rb_mask;
3151	int i, j, ps_thread_count;
3152
3153	switch (rdev->family) {
3154	case CHIP_CYPRESS:
3155	case CHIP_HEMLOCK:
3156		rdev->config.evergreen.num_ses = 2;
3157		rdev->config.evergreen.max_pipes = 4;
3158		rdev->config.evergreen.max_tile_pipes = 8;
3159		rdev->config.evergreen.max_simds = 10;
3160		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3161		rdev->config.evergreen.max_gprs = 256;
3162		rdev->config.evergreen.max_threads = 248;
3163		rdev->config.evergreen.max_gs_threads = 32;
3164		rdev->config.evergreen.max_stack_entries = 512;
3165		rdev->config.evergreen.sx_num_of_sets = 4;
3166		rdev->config.evergreen.sx_max_export_size = 256;
3167		rdev->config.evergreen.sx_max_export_pos_size = 64;
3168		rdev->config.evergreen.sx_max_export_smx_size = 192;
3169		rdev->config.evergreen.max_hw_contexts = 8;
3170		rdev->config.evergreen.sq_num_cf_insts = 2;
3171
3172		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3173		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3174		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3175		gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3176		break;
3177	case CHIP_JUNIPER:
3178		rdev->config.evergreen.num_ses = 1;
3179		rdev->config.evergreen.max_pipes = 4;
3180		rdev->config.evergreen.max_tile_pipes = 4;
3181		rdev->config.evergreen.max_simds = 10;
3182		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3183		rdev->config.evergreen.max_gprs = 256;
3184		rdev->config.evergreen.max_threads = 248;
3185		rdev->config.evergreen.max_gs_threads = 32;
3186		rdev->config.evergreen.max_stack_entries = 512;
3187		rdev->config.evergreen.sx_num_of_sets = 4;
3188		rdev->config.evergreen.sx_max_export_size = 256;
3189		rdev->config.evergreen.sx_max_export_pos_size = 64;
3190		rdev->config.evergreen.sx_max_export_smx_size = 192;
3191		rdev->config.evergreen.max_hw_contexts = 8;
3192		rdev->config.evergreen.sq_num_cf_insts = 2;
3193
3194		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3195		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3196		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3197		gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3198		break;
3199	case CHIP_REDWOOD:
3200		rdev->config.evergreen.num_ses = 1;
3201		rdev->config.evergreen.max_pipes = 4;
3202		rdev->config.evergreen.max_tile_pipes = 4;
3203		rdev->config.evergreen.max_simds = 5;
3204		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3205		rdev->config.evergreen.max_gprs = 256;
3206		rdev->config.evergreen.max_threads = 248;
3207		rdev->config.evergreen.max_gs_threads = 32;
3208		rdev->config.evergreen.max_stack_entries = 256;
3209		rdev->config.evergreen.sx_num_of_sets = 4;
3210		rdev->config.evergreen.sx_max_export_size = 256;
3211		rdev->config.evergreen.sx_max_export_pos_size = 64;
3212		rdev->config.evergreen.sx_max_export_smx_size = 192;
3213		rdev->config.evergreen.max_hw_contexts = 8;
3214		rdev->config.evergreen.sq_num_cf_insts = 2;
3215
3216		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3217		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3218		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3219		gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3220		break;
3221	case CHIP_CEDAR:
3222	default:
3223		rdev->config.evergreen.num_ses = 1;
3224		rdev->config.evergreen.max_pipes = 2;
3225		rdev->config.evergreen.max_tile_pipes = 2;
3226		rdev->config.evergreen.max_simds = 2;
3227		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3228		rdev->config.evergreen.max_gprs = 256;
3229		rdev->config.evergreen.max_threads = 192;
3230		rdev->config.evergreen.max_gs_threads = 16;
3231		rdev->config.evergreen.max_stack_entries = 256;
3232		rdev->config.evergreen.sx_num_of_sets = 4;
3233		rdev->config.evergreen.sx_max_export_size = 128;
3234		rdev->config.evergreen.sx_max_export_pos_size = 32;
3235		rdev->config.evergreen.sx_max_export_smx_size = 96;
3236		rdev->config.evergreen.max_hw_contexts = 4;
3237		rdev->config.evergreen.sq_num_cf_insts = 1;
3238
3239		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3240		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3241		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3242		gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3243		break;
3244	case CHIP_PALM:
3245		rdev->config.evergreen.num_ses = 1;
3246		rdev->config.evergreen.max_pipes = 2;
3247		rdev->config.evergreen.max_tile_pipes = 2;
3248		rdev->config.evergreen.max_simds = 2;
3249		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3250		rdev->config.evergreen.max_gprs = 256;
3251		rdev->config.evergreen.max_threads = 192;
3252		rdev->config.evergreen.max_gs_threads = 16;
3253		rdev->config.evergreen.max_stack_entries = 256;
3254		rdev->config.evergreen.sx_num_of_sets = 4;
3255		rdev->config.evergreen.sx_max_export_size = 128;
3256		rdev->config.evergreen.sx_max_export_pos_size = 32;
3257		rdev->config.evergreen.sx_max_export_smx_size = 96;
3258		rdev->config.evergreen.max_hw_contexts = 4;
3259		rdev->config.evergreen.sq_num_cf_insts = 1;
3260
3261		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3262		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3263		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3264		gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3265		break;
3266	case CHIP_SUMO:
3267		rdev->config.evergreen.num_ses = 1;
3268		rdev->config.evergreen.max_pipes = 4;
3269		rdev->config.evergreen.max_tile_pipes = 4;
3270		if (rdev->pdev->device == 0x9648)
3271			rdev->config.evergreen.max_simds = 3;
3272		else if ((rdev->pdev->device == 0x9647) ||
3273			 (rdev->pdev->device == 0x964a))
3274			rdev->config.evergreen.max_simds = 4;
3275		else
3276			rdev->config.evergreen.max_simds = 5;
3277		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3278		rdev->config.evergreen.max_gprs = 256;
3279		rdev->config.evergreen.max_threads = 248;
3280		rdev->config.evergreen.max_gs_threads = 32;
3281		rdev->config.evergreen.max_stack_entries = 256;
3282		rdev->config.evergreen.sx_num_of_sets = 4;
3283		rdev->config.evergreen.sx_max_export_size = 256;
3284		rdev->config.evergreen.sx_max_export_pos_size = 64;
3285		rdev->config.evergreen.sx_max_export_smx_size = 192;
3286		rdev->config.evergreen.max_hw_contexts = 8;
3287		rdev->config.evergreen.sq_num_cf_insts = 2;
3288
3289		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3290		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3291		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3292		gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3293		break;
3294	case CHIP_SUMO2:
3295		rdev->config.evergreen.num_ses = 1;
3296		rdev->config.evergreen.max_pipes = 4;
3297		rdev->config.evergreen.max_tile_pipes = 4;
3298		rdev->config.evergreen.max_simds = 2;
3299		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3300		rdev->config.evergreen.max_gprs = 256;
3301		rdev->config.evergreen.max_threads = 248;
3302		rdev->config.evergreen.max_gs_threads = 32;
3303		rdev->config.evergreen.max_stack_entries = 512;
3304		rdev->config.evergreen.sx_num_of_sets = 4;
3305		rdev->config.evergreen.sx_max_export_size = 256;
3306		rdev->config.evergreen.sx_max_export_pos_size = 64;
3307		rdev->config.evergreen.sx_max_export_smx_size = 192;
3308		rdev->config.evergreen.max_hw_contexts = 4;
3309		rdev->config.evergreen.sq_num_cf_insts = 2;
3310
3311		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3312		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3313		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3314		gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3315		break;
3316	case CHIP_BARTS:
3317		rdev->config.evergreen.num_ses = 2;
3318		rdev->config.evergreen.max_pipes = 4;
3319		rdev->config.evergreen.max_tile_pipes = 8;
3320		rdev->config.evergreen.max_simds = 7;
3321		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3322		rdev->config.evergreen.max_gprs = 256;
3323		rdev->config.evergreen.max_threads = 248;
3324		rdev->config.evergreen.max_gs_threads = 32;
3325		rdev->config.evergreen.max_stack_entries = 512;
3326		rdev->config.evergreen.sx_num_of_sets = 4;
3327		rdev->config.evergreen.sx_max_export_size = 256;
3328		rdev->config.evergreen.sx_max_export_pos_size = 64;
3329		rdev->config.evergreen.sx_max_export_smx_size = 192;
3330		rdev->config.evergreen.max_hw_contexts = 8;
3331		rdev->config.evergreen.sq_num_cf_insts = 2;
3332
3333		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3334		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3335		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3336		gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3337		break;
3338	case CHIP_TURKS:
3339		rdev->config.evergreen.num_ses = 1;
3340		rdev->config.evergreen.max_pipes = 4;
3341		rdev->config.evergreen.max_tile_pipes = 4;
3342		rdev->config.evergreen.max_simds = 6;
3343		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3344		rdev->config.evergreen.max_gprs = 256;
3345		rdev->config.evergreen.max_threads = 248;
3346		rdev->config.evergreen.max_gs_threads = 32;
3347		rdev->config.evergreen.max_stack_entries = 256;
3348		rdev->config.evergreen.sx_num_of_sets = 4;
3349		rdev->config.evergreen.sx_max_export_size = 256;
3350		rdev->config.evergreen.sx_max_export_pos_size = 64;
3351		rdev->config.evergreen.sx_max_export_smx_size = 192;
3352		rdev->config.evergreen.max_hw_contexts = 8;
3353		rdev->config.evergreen.sq_num_cf_insts = 2;
3354
3355		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3356		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3357		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3358		gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3359		break;
3360	case CHIP_CAICOS:
3361		rdev->config.evergreen.num_ses = 1;
3362		rdev->config.evergreen.max_pipes = 2;
3363		rdev->config.evergreen.max_tile_pipes = 2;
3364		rdev->config.evergreen.max_simds = 2;
3365		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3366		rdev->config.evergreen.max_gprs = 256;
3367		rdev->config.evergreen.max_threads = 192;
3368		rdev->config.evergreen.max_gs_threads = 16;
3369		rdev->config.evergreen.max_stack_entries = 256;
3370		rdev->config.evergreen.sx_num_of_sets = 4;
3371		rdev->config.evergreen.sx_max_export_size = 128;
3372		rdev->config.evergreen.sx_max_export_pos_size = 32;
3373		rdev->config.evergreen.sx_max_export_smx_size = 96;
3374		rdev->config.evergreen.max_hw_contexts = 4;
3375		rdev->config.evergreen.sq_num_cf_insts = 1;
3376
3377		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3378		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3379		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3380		gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3381		break;
3382	}
3383
3384	/* Initialize HDP */
3385	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3386		WREG32((0x2c14 + j), 0x00000000);
3387		WREG32((0x2c18 + j), 0x00000000);
3388		WREG32((0x2c1c + j), 0x00000000);
3389		WREG32((0x2c20 + j), 0x00000000);
3390		WREG32((0x2c24 + j), 0x00000000);
3391	}
3392
3393	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3394	WREG32(SRBM_INT_CNTL, 0x1);
3395	WREG32(SRBM_INT_ACK, 0x1);
3396
3397	evergreen_fix_pci_max_read_req_size(rdev);
3398
3399	RREG32(MC_SHARED_CHMAP);
3400	if ((rdev->family == CHIP_PALM) ||
3401	    (rdev->family == CHIP_SUMO) ||
3402	    (rdev->family == CHIP_SUMO2))
3403		mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3404	else
3405		mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3406
3407	/* setup tiling info dword.  gb_addr_config is not adequate since it does
3408	 * not have bank info, so create a custom tiling dword.
3409	 * bits 3:0   num_pipes
3410	 * bits 7:4   num_banks
3411	 * bits 11:8  group_size
3412	 * bits 15:12 row_size
3413	 */
3414	rdev->config.evergreen.tile_config = 0;
3415	switch (rdev->config.evergreen.max_tile_pipes) {
3416	case 1:
3417	default:
3418		rdev->config.evergreen.tile_config |= (0 << 0);
3419		break;
3420	case 2:
3421		rdev->config.evergreen.tile_config |= (1 << 0);
3422		break;
3423	case 4:
3424		rdev->config.evergreen.tile_config |= (2 << 0);
3425		break;
3426	case 8:
3427		rdev->config.evergreen.tile_config |= (3 << 0);
3428		break;
3429	}
3430	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3431	if (rdev->flags & RADEON_IS_IGP)
3432		rdev->config.evergreen.tile_config |= 1 << 4;
3433	else {
3434		switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3435		case 0: /* four banks */
3436			rdev->config.evergreen.tile_config |= 0 << 4;
3437			break;
3438		case 1: /* eight banks */
3439			rdev->config.evergreen.tile_config |= 1 << 4;
3440			break;
3441		case 2: /* sixteen banks */
3442		default:
3443			rdev->config.evergreen.tile_config |= 2 << 4;
3444			break;
3445		}
3446	}
3447	rdev->config.evergreen.tile_config |= 0 << 8;
3448	rdev->config.evergreen.tile_config |=
3449		((gb_addr_config & 0x30000000) >> 28) << 12;
3450
3451	if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3452		u32 efuse_straps_4;
3453		u32 efuse_straps_3;
3454
3455		efuse_straps_4 = RREG32_RCU(0x204);
3456		efuse_straps_3 = RREG32_RCU(0x203);
3457		tmp = (((efuse_straps_4 & 0xf) << 4) |
3458		      ((efuse_straps_3 & 0xf0000000) >> 28));
3459	} else {
3460		tmp = 0;
3461		for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3462			u32 rb_disable_bitmap;
3463
3464			WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3465			WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3466			rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3467			tmp <<= 4;
3468			tmp |= rb_disable_bitmap;
3469		}
3470	}
3471	/* enabled rb are just the one not disabled :) */
3472	disabled_rb_mask = tmp;
3473	tmp = 0;
3474	for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3475		tmp |= (1 << i);
3476	/* if all the backends are disabled, fix it up here */
3477	if ((disabled_rb_mask & tmp) == tmp) {
3478		for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3479			disabled_rb_mask &= ~(1 << i);
3480	}
3481
3482	for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3483		u32 simd_disable_bitmap;
3484
3485		WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3486		WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3487		simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3488		simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3489		tmp <<= 16;
3490		tmp |= simd_disable_bitmap;
3491	}
3492	rdev->config.evergreen.active_simds = hweight32(~tmp);
3493
3494	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3495	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3496
3497	WREG32(GB_ADDR_CONFIG, gb_addr_config);
3498	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3499	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3500	WREG32(DMA_TILING_CONFIG, gb_addr_config);
3501	WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3502	WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3503	WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3504
3505	if ((rdev->config.evergreen.max_backends == 1) &&
3506	    (rdev->flags & RADEON_IS_IGP)) {
3507		if ((disabled_rb_mask & 3) == 1) {
3508			/* RB0 disabled, RB1 enabled */
3509			tmp = 0x11111111;
3510		} else {
3511			/* RB1 disabled, RB0 enabled */
3512			tmp = 0x00000000;
3513		}
3514	} else {
3515		tmp = gb_addr_config & NUM_PIPES_MASK;
3516		tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3517						EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3518	}
3519	rdev->config.evergreen.backend_map = tmp;
3520	WREG32(GB_BACKEND_MAP, tmp);
3521
3522	WREG32(CGTS_SYS_TCC_DISABLE, 0);
3523	WREG32(CGTS_TCC_DISABLE, 0);
3524	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3525	WREG32(CGTS_USER_TCC_DISABLE, 0);
3526
3527	/* set HW defaults for 3D engine */
3528	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3529				     ROQ_IB2_START(0x2b)));
3530
3531	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3532
3533	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3534			     SYNC_GRADIENT |
3535			     SYNC_WALKER |
3536			     SYNC_ALIGNER));
3537
3538	sx_debug_1 = RREG32(SX_DEBUG_1);
3539	sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3540	WREG32(SX_DEBUG_1, sx_debug_1);
3541
3542
3543	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3544	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3545	smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3546	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3547
3548	if (rdev->family <= CHIP_SUMO2)
3549		WREG32(SMX_SAR_CTL0, 0x00010000);
3550
3551	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3552					POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3553					SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3554
3555	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3556				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3557				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3558
3559	WREG32(VGT_NUM_INSTANCES, 1);
3560	WREG32(SPI_CONFIG_CNTL, 0);
3561	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3562	WREG32(CP_PERFMON_CNTL, 0);
3563
3564	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3565				  FETCH_FIFO_HIWATER(0x4) |
3566				  DONE_FIFO_HIWATER(0xe0) |
3567				  ALU_UPDATE_FIFO_HIWATER(0x8)));
3568
3569	sq_config = RREG32(SQ_CONFIG);
3570	sq_config &= ~(PS_PRIO(3) |
3571		       VS_PRIO(3) |
3572		       GS_PRIO(3) |
3573		       ES_PRIO(3));
3574	sq_config |= (VC_ENABLE |
3575		      EXPORT_SRC_C |
3576		      PS_PRIO(0) |
3577		      VS_PRIO(1) |
3578		      GS_PRIO(2) |
3579		      ES_PRIO(3));
3580
3581	switch (rdev->family) {
3582	case CHIP_CEDAR:
3583	case CHIP_PALM:
3584	case CHIP_SUMO:
3585	case CHIP_SUMO2:
3586	case CHIP_CAICOS:
3587		/* no vertex cache */
3588		sq_config &= ~VC_ENABLE;
3589		break;
3590	default:
3591		break;
3592	}
3593
3594	sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3595
3596	sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 12 / 32);
3597	sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3598	sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3599	sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3600	sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3601	sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3602	sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3603
3604	switch (rdev->family) {
3605	case CHIP_CEDAR:
3606	case CHIP_PALM:
3607	case CHIP_SUMO:
3608	case CHIP_SUMO2:
3609		ps_thread_count = 96;
3610		break;
3611	default:
3612		ps_thread_count = 128;
3613		break;
3614	}
3615
3616	sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3617	sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3618	sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3619	sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3620	sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3621	sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3622
3623	sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3624	sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3625	sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3626	sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3627	sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3628	sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3629
3630	WREG32(SQ_CONFIG, sq_config);
3631	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3632	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3633	WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3634	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3635	WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3636	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3637	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3638	WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3639	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3640	WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3641
3642	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3643					  FORCE_EOV_MAX_REZ_CNT(255)));
3644
3645	switch (rdev->family) {
3646	case CHIP_CEDAR:
3647	case CHIP_PALM:
3648	case CHIP_SUMO:
3649	case CHIP_SUMO2:
3650	case CHIP_CAICOS:
3651		vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3652		break;
3653	default:
3654		vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3655		break;
3656	}
3657	vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3658	WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3659
3660	WREG32(VGT_GS_VERTEX_REUSE, 16);
3661	WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3662	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3663
3664	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3665	WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3666
3667	WREG32(CB_PERF_CTR0_SEL_0, 0);
3668	WREG32(CB_PERF_CTR0_SEL_1, 0);
3669	WREG32(CB_PERF_CTR1_SEL_0, 0);
3670	WREG32(CB_PERF_CTR1_SEL_1, 0);
3671	WREG32(CB_PERF_CTR2_SEL_0, 0);
3672	WREG32(CB_PERF_CTR2_SEL_1, 0);
3673	WREG32(CB_PERF_CTR3_SEL_0, 0);
3674	WREG32(CB_PERF_CTR3_SEL_1, 0);
3675
3676	/* clear render buffer base addresses */
3677	WREG32(CB_COLOR0_BASE, 0);
3678	WREG32(CB_COLOR1_BASE, 0);
3679	WREG32(CB_COLOR2_BASE, 0);
3680	WREG32(CB_COLOR3_BASE, 0);
3681	WREG32(CB_COLOR4_BASE, 0);
3682	WREG32(CB_COLOR5_BASE, 0);
3683	WREG32(CB_COLOR6_BASE, 0);
3684	WREG32(CB_COLOR7_BASE, 0);
3685	WREG32(CB_COLOR8_BASE, 0);
3686	WREG32(CB_COLOR9_BASE, 0);
3687	WREG32(CB_COLOR10_BASE, 0);
3688	WREG32(CB_COLOR11_BASE, 0);
3689
3690	/* set the shader const cache sizes to 0 */
3691	for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3692		WREG32(i, 0);
3693	for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3694		WREG32(i, 0);
3695
3696	tmp = RREG32(HDP_MISC_CNTL);
3697	tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3698	WREG32(HDP_MISC_CNTL, tmp);
3699
3700	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3701	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3702
3703	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3704
3705	udelay(50);
3706
3707}
3708
3709int evergreen_mc_init(struct radeon_device *rdev)
3710{
3711	u32 tmp;
3712	int chansize, numchan;
3713
3714	/* Get VRAM informations */
3715	rdev->mc.vram_is_ddr = true;
3716	if ((rdev->family == CHIP_PALM) ||
3717	    (rdev->family == CHIP_SUMO) ||
3718	    (rdev->family == CHIP_SUMO2))
3719		tmp = RREG32(FUS_MC_ARB_RAMCFG);
3720	else
3721		tmp = RREG32(MC_ARB_RAMCFG);
3722	if (tmp & CHANSIZE_OVERRIDE) {
3723		chansize = 16;
3724	} else if (tmp & CHANSIZE_MASK) {
3725		chansize = 64;
3726	} else {
3727		chansize = 32;
3728	}
3729	tmp = RREG32(MC_SHARED_CHMAP);
3730	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3731	case 0:
3732	default:
3733		numchan = 1;
3734		break;
3735	case 1:
3736		numchan = 2;
3737		break;
3738	case 2:
3739		numchan = 4;
3740		break;
3741	case 3:
3742		numchan = 8;
3743		break;
3744	}
3745	rdev->mc.vram_width = numchan * chansize;
3746	/* Could aper size report 0 ? */
3747	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3748	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3749	/* Setup GPU memory space */
3750	if ((rdev->family == CHIP_PALM) ||
3751	    (rdev->family == CHIP_SUMO) ||
3752	    (rdev->family == CHIP_SUMO2)) {
3753		/* size in bytes on fusion */
3754		rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3755		rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3756	} else {
3757		/* size in MB on evergreen/cayman/tn */
3758		rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3759		rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3760	}
3761	rdev->mc.visible_vram_size = rdev->mc.aper_size;
3762	r700_vram_gtt_location(rdev, &rdev->mc);
3763	radeon_update_bandwidth_info(rdev);
3764
3765	return 0;
3766}
3767
3768void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3769{
3770	dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3771		RREG32(GRBM_STATUS));
3772	dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3773		RREG32(GRBM_STATUS_SE0));
3774	dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3775		RREG32(GRBM_STATUS_SE1));
3776	dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3777		RREG32(SRBM_STATUS));
3778	dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3779		RREG32(SRBM_STATUS2));
3780	dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3781		RREG32(CP_STALLED_STAT1));
3782	dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3783		RREG32(CP_STALLED_STAT2));
3784	dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3785		RREG32(CP_BUSY_STAT));
3786	dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3787		RREG32(CP_STAT));
3788	dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3789		RREG32(DMA_STATUS_REG));
3790	if (rdev->family >= CHIP_CAYMAN) {
3791		dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3792			 RREG32(DMA_STATUS_REG + 0x800));
3793	}
3794}
3795
3796bool evergreen_is_display_hung(struct radeon_device *rdev)
3797{
3798	u32 crtc_hung = 0;
3799	u32 crtc_status[6];
3800	u32 i, j, tmp;
3801
3802	for (i = 0; i < rdev->num_crtc; i++) {
3803		if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3804			crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3805			crtc_hung |= (1 << i);
3806		}
3807	}
3808
3809	for (j = 0; j < 10; j++) {
3810		for (i = 0; i < rdev->num_crtc; i++) {
3811			if (crtc_hung & (1 << i)) {
3812				tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3813				if (tmp != crtc_status[i])
3814					crtc_hung &= ~(1 << i);
3815			}
3816		}
3817		if (crtc_hung == 0)
3818			return false;
3819		udelay(100);
3820	}
3821
3822	return true;
3823}
3824
3825u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3826{
3827	u32 reset_mask = 0;
3828	u32 tmp;
3829
3830	/* GRBM_STATUS */
3831	tmp = RREG32(GRBM_STATUS);
3832	if (tmp & (PA_BUSY | SC_BUSY |
3833		   SH_BUSY | SX_BUSY |
3834		   TA_BUSY | VGT_BUSY |
3835		   DB_BUSY | CB_BUSY |
3836		   SPI_BUSY | VGT_BUSY_NO_DMA))
3837		reset_mask |= RADEON_RESET_GFX;
3838
3839	if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3840		   CP_BUSY | CP_COHERENCY_BUSY))
3841		reset_mask |= RADEON_RESET_CP;
3842
3843	if (tmp & GRBM_EE_BUSY)
3844		reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3845
3846	/* DMA_STATUS_REG */
3847	tmp = RREG32(DMA_STATUS_REG);
3848	if (!(tmp & DMA_IDLE))
3849		reset_mask |= RADEON_RESET_DMA;
3850
3851	/* SRBM_STATUS2 */
3852	tmp = RREG32(SRBM_STATUS2);
3853	if (tmp & DMA_BUSY)
3854		reset_mask |= RADEON_RESET_DMA;
3855
3856	/* SRBM_STATUS */
3857	tmp = RREG32(SRBM_STATUS);
3858	if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3859		reset_mask |= RADEON_RESET_RLC;
3860
3861	if (tmp & IH_BUSY)
3862		reset_mask |= RADEON_RESET_IH;
3863
3864	if (tmp & SEM_BUSY)
3865		reset_mask |= RADEON_RESET_SEM;
3866
3867	if (tmp & GRBM_RQ_PENDING)
3868		reset_mask |= RADEON_RESET_GRBM;
3869
3870	if (tmp & VMC_BUSY)
3871		reset_mask |= RADEON_RESET_VMC;
3872
3873	if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3874		   MCC_BUSY | MCD_BUSY))
3875		reset_mask |= RADEON_RESET_MC;
3876
3877	if (evergreen_is_display_hung(rdev))
3878		reset_mask |= RADEON_RESET_DISPLAY;
3879
3880	/* VM_L2_STATUS */
3881	tmp = RREG32(VM_L2_STATUS);
3882	if (tmp & L2_BUSY)
3883		reset_mask |= RADEON_RESET_VMC;
3884
3885	/* Skip MC reset as it's mostly likely not hung, just busy */
3886	if (reset_mask & RADEON_RESET_MC) {
3887		DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3888		reset_mask &= ~RADEON_RESET_MC;
3889	}
3890
3891	return reset_mask;
3892}
3893
3894static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3895{
3896	struct evergreen_mc_save save;
3897	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3898	u32 tmp;
3899
3900	if (reset_mask == 0)
3901		return;
3902
3903	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3904
3905	evergreen_print_gpu_status_regs(rdev);
3906
3907	/* Disable CP parsing/prefetching */
3908	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3909
3910	if (reset_mask & RADEON_RESET_DMA) {
3911		/* Disable DMA */
3912		tmp = RREG32(DMA_RB_CNTL);
3913		tmp &= ~DMA_RB_ENABLE;
3914		WREG32(DMA_RB_CNTL, tmp);
3915	}
3916
3917	udelay(50);
3918
3919	evergreen_mc_stop(rdev, &save);
3920	if (evergreen_mc_wait_for_idle(rdev)) {
3921		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3922	}
3923
3924	if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3925		grbm_soft_reset |= SOFT_RESET_DB |
3926			SOFT_RESET_CB |
3927			SOFT_RESET_PA |
3928			SOFT_RESET_SC |
3929			SOFT_RESET_SPI |
3930			SOFT_RESET_SX |
3931			SOFT_RESET_SH |
3932			SOFT_RESET_TC |
3933			SOFT_RESET_TA |
3934			SOFT_RESET_VC |
3935			SOFT_RESET_VGT;
3936	}
3937
3938	if (reset_mask & RADEON_RESET_CP) {
3939		grbm_soft_reset |= SOFT_RESET_CP |
3940			SOFT_RESET_VGT;
3941
3942		srbm_soft_reset |= SOFT_RESET_GRBM;
3943	}
3944
3945	if (reset_mask & RADEON_RESET_DMA)
3946		srbm_soft_reset |= SOFT_RESET_DMA;
3947
3948	if (reset_mask & RADEON_RESET_DISPLAY)
3949		srbm_soft_reset |= SOFT_RESET_DC;
3950
3951	if (reset_mask & RADEON_RESET_RLC)
3952		srbm_soft_reset |= SOFT_RESET_RLC;
3953
3954	if (reset_mask & RADEON_RESET_SEM)
3955		srbm_soft_reset |= SOFT_RESET_SEM;
3956
3957	if (reset_mask & RADEON_RESET_IH)
3958		srbm_soft_reset |= SOFT_RESET_IH;
3959
3960	if (reset_mask & RADEON_RESET_GRBM)
3961		srbm_soft_reset |= SOFT_RESET_GRBM;
3962
3963	if (reset_mask & RADEON_RESET_VMC)
3964		srbm_soft_reset |= SOFT_RESET_VMC;
3965
3966	if (!(rdev->flags & RADEON_IS_IGP)) {
3967		if (reset_mask & RADEON_RESET_MC)
3968			srbm_soft_reset |= SOFT_RESET_MC;
3969	}
3970
3971	if (grbm_soft_reset) {
3972		tmp = RREG32(GRBM_SOFT_RESET);
3973		tmp |= grbm_soft_reset;
3974		dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3975		WREG32(GRBM_SOFT_RESET, tmp);
3976		tmp = RREG32(GRBM_SOFT_RESET);
3977
3978		udelay(50);
3979
3980		tmp &= ~grbm_soft_reset;
3981		WREG32(GRBM_SOFT_RESET, tmp);
3982		tmp = RREG32(GRBM_SOFT_RESET);
3983	}
3984
3985	if (srbm_soft_reset) {
3986		tmp = RREG32(SRBM_SOFT_RESET);
3987		tmp |= srbm_soft_reset;
3988		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3989		WREG32(SRBM_SOFT_RESET, tmp);
3990		tmp = RREG32(SRBM_SOFT_RESET);
3991
3992		udelay(50);
3993
3994		tmp &= ~srbm_soft_reset;
3995		WREG32(SRBM_SOFT_RESET, tmp);
3996		tmp = RREG32(SRBM_SOFT_RESET);
3997	}
3998
3999	/* Wait a little for things to settle down */
4000	udelay(50);
4001
4002	evergreen_mc_resume(rdev, &save);
4003	udelay(50);
4004
4005	evergreen_print_gpu_status_regs(rdev);
4006}
4007
4008void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4009{
4010	struct evergreen_mc_save save;
4011	u32 tmp, i;
4012
4013	dev_info(rdev->dev, "GPU pci config reset\n");
4014
4015	/* disable dpm? */
4016
4017	/* Disable CP parsing/prefetching */
4018	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4019	udelay(50);
4020	/* Disable DMA */
4021	tmp = RREG32(DMA_RB_CNTL);
4022	tmp &= ~DMA_RB_ENABLE;
4023	WREG32(DMA_RB_CNTL, tmp);
4024	/* XXX other engines? */
4025
4026	/* halt the rlc */
4027	r600_rlc_stop(rdev);
4028
4029	udelay(50);
4030
4031	/* set mclk/sclk to bypass */
4032	rv770_set_clk_bypass_mode(rdev);
4033	/* disable BM */
4034	pci_clear_master(rdev->pdev);
4035	/* disable mem access */
4036	evergreen_mc_stop(rdev, &save);
4037	if (evergreen_mc_wait_for_idle(rdev)) {
4038		dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4039	}
4040	/* reset */
4041	radeon_pci_config_reset(rdev);
4042	/* wait for asic to come out of reset */
4043	for (i = 0; i < rdev->usec_timeout; i++) {
4044		if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4045			break;
4046		udelay(1);
4047	}
4048}
4049
4050int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4051{
4052	u32 reset_mask;
4053
4054	if (hard) {
4055		evergreen_gpu_pci_config_reset(rdev);
4056		return 0;
4057	}
4058
4059	reset_mask = evergreen_gpu_check_soft_reset(rdev);
4060
4061	if (reset_mask)
4062		r600_set_bios_scratch_engine_hung(rdev, true);
4063
4064	/* try soft reset */
4065	evergreen_gpu_soft_reset(rdev, reset_mask);
4066
4067	reset_mask = evergreen_gpu_check_soft_reset(rdev);
4068
4069	/* try pci config reset */
4070	if (reset_mask && radeon_hard_reset)
4071		evergreen_gpu_pci_config_reset(rdev);
4072
4073	reset_mask = evergreen_gpu_check_soft_reset(rdev);
4074
4075	if (!reset_mask)
4076		r600_set_bios_scratch_engine_hung(rdev, false);
4077
4078	return 0;
4079}
4080
4081/**
4082 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4083 *
4084 * @rdev: radeon_device pointer
4085 * @ring: radeon_ring structure holding ring information
4086 *
4087 * Check if the GFX engine is locked up.
4088 * Returns true if the engine appears to be locked up, false if not.
4089 */
4090bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4091{
4092	u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4093
4094	if (!(reset_mask & (RADEON_RESET_GFX |
4095			    RADEON_RESET_COMPUTE |
4096			    RADEON_RESET_CP))) {
4097		radeon_ring_lockup_update(rdev, ring);
4098		return false;
4099	}
4100	return radeon_ring_test_lockup(rdev, ring);
4101}
4102
4103/*
4104 * RLC
4105 */
4106#define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
4107#define RLC_CLEAR_STATE_END_MARKER          0x00000001
4108
4109void sumo_rlc_fini(struct radeon_device *rdev)
4110{
4111	int r;
4112
4113	/* save restore block */
4114	if (rdev->rlc.save_restore_obj) {
4115		r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4116		if (unlikely(r != 0))
4117			dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4118		radeon_bo_unpin(rdev->rlc.save_restore_obj);
4119		radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4120
4121		radeon_bo_unref(&rdev->rlc.save_restore_obj);
4122		rdev->rlc.save_restore_obj = NULL;
4123	}
4124
4125	/* clear state block */
4126	if (rdev->rlc.clear_state_obj) {
4127		r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4128		if (unlikely(r != 0))
4129			dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4130		radeon_bo_unpin(rdev->rlc.clear_state_obj);
4131		radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4132
4133		radeon_bo_unref(&rdev->rlc.clear_state_obj);
4134		rdev->rlc.clear_state_obj = NULL;
4135	}
4136
4137	/* clear state block */
4138	if (rdev->rlc.cp_table_obj) {
4139		r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4140		if (unlikely(r != 0))
4141			dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4142		radeon_bo_unpin(rdev->rlc.cp_table_obj);
4143		radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4144
4145		radeon_bo_unref(&rdev->rlc.cp_table_obj);
4146		rdev->rlc.cp_table_obj = NULL;
4147	}
4148}
4149
4150#define CP_ME_TABLE_SIZE    96
4151
4152int sumo_rlc_init(struct radeon_device *rdev)
4153{
4154	const u32 *src_ptr;
4155	volatile u32 *dst_ptr;
4156	u32 dws, data, i, j, k, reg_num;
4157	u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4158	u64 reg_list_mc_addr;
4159	const struct cs_section_def *cs_data;
4160	int r;
4161
4162	src_ptr = rdev->rlc.reg_list;
4163	dws = rdev->rlc.reg_list_size;
4164	if (rdev->family >= CHIP_BONAIRE) {
4165		dws += (5 * 16) + 48 + 48 + 64;
4166	}
4167	cs_data = rdev->rlc.cs_data;
4168
4169	if (src_ptr) {
4170		/* save restore block */
4171		if (rdev->rlc.save_restore_obj == NULL) {
4172			r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4173					     RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4174					     NULL, &rdev->rlc.save_restore_obj);
4175			if (r) {
4176				dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4177				return r;
4178			}
4179		}
4180
4181		r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4182		if (unlikely(r != 0)) {
4183			sumo_rlc_fini(rdev);
4184			return r;
4185		}
4186		r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4187				  &rdev->rlc.save_restore_gpu_addr);
4188		if (r) {
4189			radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4190			dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4191			sumo_rlc_fini(rdev);
4192			return r;
4193		}
4194
4195		r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4196		if (r) {
4197			dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4198			sumo_rlc_fini(rdev);
4199			return r;
4200		}
4201		/* write the sr buffer */
4202		dst_ptr = rdev->rlc.sr_ptr;
4203		if (rdev->family >= CHIP_TAHITI) {
4204			/* SI */
4205			for (i = 0; i < rdev->rlc.reg_list_size; i++)
4206				dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4207		} else {
4208			/* ON/LN/TN */
4209			/* format:
4210			 * dw0: (reg2 << 16) | reg1
4211			 * dw1: reg1 save space
4212			 * dw2: reg2 save space
4213			 */
4214			for (i = 0; i < dws; i++) {
4215				data = src_ptr[i] >> 2;
4216				i++;
4217				if (i < dws)
4218					data |= (src_ptr[i] >> 2) << 16;
4219				j = (((i - 1) * 3) / 2);
4220				dst_ptr[j] = cpu_to_le32(data);
4221			}
4222			j = ((i * 3) / 2);
4223			dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4224		}
4225		radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4226		radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4227	}
4228
4229	if (cs_data) {
4230		/* clear state block */
4231		if (rdev->family >= CHIP_BONAIRE) {
4232			rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4233		} else if (rdev->family >= CHIP_TAHITI) {
4234			rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4235			dws = rdev->rlc.clear_state_size + (256 / 4);
4236		} else {
4237			reg_list_num = 0;
4238			dws = 0;
4239			for (i = 0; cs_data[i].section != NULL; i++) {
4240				for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4241					reg_list_num++;
4242					dws += cs_data[i].section[j].reg_count;
4243				}
4244			}
4245			reg_list_blk_index = (3 * reg_list_num + 2);
4246			dws += reg_list_blk_index;
4247			rdev->rlc.clear_state_size = dws;
4248		}
4249
4250		if (rdev->rlc.clear_state_obj == NULL) {
4251			r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4252					     RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4253					     NULL, &rdev->rlc.clear_state_obj);
4254			if (r) {
4255				dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4256				sumo_rlc_fini(rdev);
4257				return r;
4258			}
4259		}
4260		r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4261		if (unlikely(r != 0)) {
4262			sumo_rlc_fini(rdev);
4263			return r;
4264		}
4265		r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4266				  &rdev->rlc.clear_state_gpu_addr);
4267		if (r) {
4268			radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4269			dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4270			sumo_rlc_fini(rdev);
4271			return r;
4272		}
4273
4274		r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4275		if (r) {
4276			dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4277			sumo_rlc_fini(rdev);
4278			return r;
4279		}
4280		/* set up the cs buffer */
4281		dst_ptr = rdev->rlc.cs_ptr;
4282		if (rdev->family >= CHIP_BONAIRE) {
4283			cik_get_csb_buffer(rdev, dst_ptr);
4284		} else if (rdev->family >= CHIP_TAHITI) {
4285			reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4286			dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4287			dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4288			dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4289			si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4290		} else {
4291			reg_list_hdr_blk_index = 0;
4292			reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4293			data = upper_32_bits(reg_list_mc_addr);
4294			dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4295			reg_list_hdr_blk_index++;
4296			for (i = 0; cs_data[i].section != NULL; i++) {
4297				for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4298					reg_num = cs_data[i].section[j].reg_count;
4299					data = reg_list_mc_addr & 0xffffffff;
4300					dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4301					reg_list_hdr_blk_index++;
4302
4303					data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4304					dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4305					reg_list_hdr_blk_index++;
4306
4307					data = 0x08000000 | (reg_num * 4);
4308					dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4309					reg_list_hdr_blk_index++;
4310
4311					for (k = 0; k < reg_num; k++) {
4312						data = cs_data[i].section[j].extent[k];
4313						dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4314					}
4315					reg_list_mc_addr += reg_num * 4;
4316					reg_list_blk_index += reg_num;
4317				}
4318			}
4319			dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4320		}
4321		radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4322		radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4323	}
4324
4325	if (rdev->rlc.cp_table_size) {
4326		if (rdev->rlc.cp_table_obj == NULL) {
4327			r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4328					     PAGE_SIZE, true,
4329					     RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4330					     NULL, &rdev->rlc.cp_table_obj);
4331			if (r) {
4332				dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4333				sumo_rlc_fini(rdev);
4334				return r;
4335			}
4336		}
4337
4338		r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4339		if (unlikely(r != 0)) {
4340			dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4341			sumo_rlc_fini(rdev);
4342			return r;
4343		}
4344		r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4345				  &rdev->rlc.cp_table_gpu_addr);
4346		if (r) {
4347			radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4348			dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4349			sumo_rlc_fini(rdev);
4350			return r;
4351		}
4352		r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4353		if (r) {
4354			dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4355			sumo_rlc_fini(rdev);
4356			return r;
4357		}
4358
4359		cik_init_cp_pg_table(rdev);
4360
4361		radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4362		radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4363
4364	}
4365
4366	return 0;
4367}
4368
4369static void evergreen_rlc_start(struct radeon_device *rdev)
4370{
4371	u32 mask = RLC_ENABLE;
4372
4373	if (rdev->flags & RADEON_IS_IGP) {
4374		mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4375	}
4376
4377	WREG32(RLC_CNTL, mask);
4378}
4379
4380int evergreen_rlc_resume(struct radeon_device *rdev)
4381{
4382	u32 i;
4383	const __be32 *fw_data;
4384
4385	if (!rdev->rlc_fw)
4386		return -EINVAL;
4387
4388	r600_rlc_stop(rdev);
4389
4390	WREG32(RLC_HB_CNTL, 0);
4391
4392	if (rdev->flags & RADEON_IS_IGP) {
4393		if (rdev->family == CHIP_ARUBA) {
4394			u32 always_on_bitmap =
4395				3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4396			/* find out the number of active simds */
4397			u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4398			tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4399			tmp = hweight32(~tmp);
4400			if (tmp == rdev->config.cayman.max_simds_per_se) {
4401				WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4402				WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4403				WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4404				WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4405				WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4406			}
4407		} else {
4408			WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4409			WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4410		}
4411		WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4412		WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4413	} else {
4414		WREG32(RLC_HB_BASE, 0);
4415		WREG32(RLC_HB_RPTR, 0);
4416		WREG32(RLC_HB_WPTR, 0);
4417		WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4418		WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4419	}
4420	WREG32(RLC_MC_CNTL, 0);
4421	WREG32(RLC_UCODE_CNTL, 0);
4422
4423	fw_data = (const __be32 *)rdev->rlc_fw->data;
4424	if (rdev->family >= CHIP_ARUBA) {
4425		for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4426			WREG32(RLC_UCODE_ADDR, i);
4427			WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4428		}
4429	} else if (rdev->family >= CHIP_CAYMAN) {
4430		for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4431			WREG32(RLC_UCODE_ADDR, i);
4432			WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4433		}
4434	} else {
4435		for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4436			WREG32(RLC_UCODE_ADDR, i);
4437			WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4438		}
4439	}
4440	WREG32(RLC_UCODE_ADDR, 0);
4441
4442	evergreen_rlc_start(rdev);
4443
4444	return 0;
4445}
4446
4447/* Interrupts */
4448
4449u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4450{
4451	if (crtc >= rdev->num_crtc)
4452		return 0;
4453	else
4454		return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4455}
4456
4457void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4458{
4459	int i;
4460	u32 tmp;
4461
4462	if (rdev->family >= CHIP_CAYMAN) {
4463		cayman_cp_int_cntl_setup(rdev, 0,
4464					 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4465		cayman_cp_int_cntl_setup(rdev, 1, 0);
4466		cayman_cp_int_cntl_setup(rdev, 2, 0);
4467		tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4468		WREG32(CAYMAN_DMA1_CNTL, tmp);
4469	} else
4470		WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4471	tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4472	WREG32(DMA_CNTL, tmp);
4473	WREG32(GRBM_INT_CNTL, 0);
4474	WREG32(SRBM_INT_CNTL, 0);
4475	for (i = 0; i < rdev->num_crtc; i++)
4476		WREG32(INT_MASK + crtc_offsets[i], 0);
4477	for (i = 0; i < rdev->num_crtc; i++)
4478		WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4479
4480	/* only one DAC on DCE5 */
4481	if (!ASIC_IS_DCE5(rdev))
4482		WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4483	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4484
4485	for (i = 0; i < 6; i++)
4486		WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4487}
4488
4489/* Note that the order we write back regs here is important */
4490int evergreen_irq_set(struct radeon_device *rdev)
4491{
4492	int i;
4493	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4494	u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4495	u32 grbm_int_cntl = 0;
4496	u32 dma_cntl, dma_cntl1 = 0;
4497	u32 thermal_int = 0;
4498
4499	if (!rdev->irq.installed) {
4500		WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4501		return -EINVAL;
4502	}
4503	/* don't enable anything if the ih is disabled */
4504	if (!rdev->ih.enabled) {
4505		r600_disable_interrupts(rdev);
4506		/* force the active interrupt state to all disabled */
4507		evergreen_disable_interrupt_state(rdev);
4508		return 0;
4509	}
4510
4511	if (rdev->family == CHIP_ARUBA)
4512		thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4513			~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4514	else
4515		thermal_int = RREG32(CG_THERMAL_INT) &
4516			~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4517
4518	dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4519
4520	if (rdev->family >= CHIP_CAYMAN) {
4521		/* enable CP interrupts on all rings */
4522		if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4523			DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4524			cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4525		}
4526		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4527			DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4528			cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4529		}
4530		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4531			DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4532			cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4533		}
4534	} else {
4535		if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4536			DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4537			cp_int_cntl |= RB_INT_ENABLE;
4538			cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4539		}
4540	}
4541
4542	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4543		DRM_DEBUG("r600_irq_set: sw int dma\n");
4544		dma_cntl |= TRAP_ENABLE;
4545	}
4546
4547	if (rdev->family >= CHIP_CAYMAN) {
4548		dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4549		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4550			DRM_DEBUG("r600_irq_set: sw int dma1\n");
4551			dma_cntl1 |= TRAP_ENABLE;
4552		}
4553	}
4554
4555	if (rdev->irq.dpm_thermal) {
4556		DRM_DEBUG("dpm thermal\n");
4557		thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4558	}
4559
4560	if (rdev->family >= CHIP_CAYMAN) {
4561		cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4562		cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4563		cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4564	} else
4565		WREG32(CP_INT_CNTL, cp_int_cntl);
4566
4567	WREG32(DMA_CNTL, dma_cntl);
4568
4569	if (rdev->family >= CHIP_CAYMAN)
4570		WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4571
4572	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4573
4574	for (i = 0; i < rdev->num_crtc; i++) {
4575		radeon_irq_kms_set_irq_n_enabled(
4576		    rdev, INT_MASK + crtc_offsets[i],
4577		    VBLANK_INT_MASK,
4578		    rdev->irq.crtc_vblank_int[i] ||
4579		    atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4580	}
4581
4582	for (i = 0; i < rdev->num_crtc; i++)
4583		WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4584
4585	for (i = 0; i < 6; i++) {
4586		radeon_irq_kms_set_irq_n_enabled(
4587		    rdev, DC_HPDx_INT_CONTROL(i),
4588		    DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4589		    rdev->irq.hpd[i], "HPD", i);
4590	}
4591
4592	if (rdev->family == CHIP_ARUBA)
4593		WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4594	else
4595		WREG32(CG_THERMAL_INT, thermal_int);
4596
4597	for (i = 0; i < 6; i++) {
4598		radeon_irq_kms_set_irq_n_enabled(
4599		    rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4600		    AFMT_AZ_FORMAT_WTRIG_MASK,
4601		    rdev->irq.afmt[i], "HDMI", i);
4602	}
4603
4604	/* posting read */
4605	RREG32(SRBM_STATUS);
4606
4607	return 0;
4608}
4609
4610/* Note that the order we write back regs here is important */
4611static void evergreen_irq_ack(struct radeon_device *rdev)
4612{
4613	int i, j;
4614	u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4615	u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4616	u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4617
4618	for (i = 0; i < 6; i++) {
4619		disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4620		afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4621		if (i < rdev->num_crtc)
4622			grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4623	}
4624
4625	/* We write back each interrupt register in pairs of two */
4626	for (i = 0; i < rdev->num_crtc; i += 2) {
4627		for (j = i; j < (i + 2); j++) {
4628			if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4629				WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4630				       GRPH_PFLIP_INT_CLEAR);
4631		}
4632
4633		for (j = i; j < (i + 2); j++) {
4634			if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4635				WREG32(VBLANK_STATUS + crtc_offsets[j],
4636				       VBLANK_ACK);
4637			if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4638				WREG32(VLINE_STATUS + crtc_offsets[j],
4639				       VLINE_ACK);
4640		}
4641	}
4642
4643	for (i = 0; i < 6; i++) {
4644		if (disp_int[i] & DC_HPD1_INTERRUPT)
4645			WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4646	}
4647
4648	for (i = 0; i < 6; i++) {
4649		if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4650			WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4651	}
4652
4653	for (i = 0; i < 6; i++) {
4654		if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4655			WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4656				  AFMT_AZ_FORMAT_WTRIG_ACK);
4657	}
4658}
4659
4660static void evergreen_irq_disable(struct radeon_device *rdev)
4661{
4662	r600_disable_interrupts(rdev);
4663	/* Wait and acknowledge irq */
4664	mdelay(1);
4665	evergreen_irq_ack(rdev);
4666	evergreen_disable_interrupt_state(rdev);
4667}
4668
4669void evergreen_irq_suspend(struct radeon_device *rdev)
4670{
4671	evergreen_irq_disable(rdev);
4672	r600_rlc_stop(rdev);
4673}
4674
4675static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4676{
4677	u32 wptr, tmp;
4678
4679	if (rdev->wb.enabled)
4680		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4681	else
4682		wptr = RREG32(IH_RB_WPTR);
4683
4684	if (wptr & RB_OVERFLOW) {
4685		wptr &= ~RB_OVERFLOW;
4686		/* When a ring buffer overflow happen start parsing interrupt
4687		 * from the last not overwritten vector (wptr + 16). Hopefully
4688		 * this should allow us to catchup.
4689		 */
4690		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4691			 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4692		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4693		tmp = RREG32(IH_RB_CNTL);
4694		tmp |= IH_WPTR_OVERFLOW_CLEAR;
4695		WREG32(IH_RB_CNTL, tmp);
4696	}
4697	return (wptr & rdev->ih.ptr_mask);
4698}
4699
4700int evergreen_irq_process(struct radeon_device *rdev)
4701{
4702	u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4703	u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4704	u32 crtc_idx, hpd_idx, afmt_idx;
4705	u32 mask;
4706	u32 wptr;
4707	u32 rptr;
4708	u32 src_id, src_data;
4709	u32 ring_index;
4710	bool queue_hotplug = false;
4711	bool queue_hdmi = false;
4712	bool queue_dp = false;
4713	bool queue_thermal = false;
4714	u32 status, addr;
4715	const char *event_name;
4716
4717	if (!rdev->ih.enabled || rdev->shutdown)
4718		return IRQ_NONE;
4719
4720	wptr = evergreen_get_ih_wptr(rdev);
4721
4722restart_ih:
4723	/* is somebody else already processing irqs? */
4724	if (atomic_xchg(&rdev->ih.lock, 1))
4725		return IRQ_NONE;
4726
4727	rptr = rdev->ih.rptr;
4728	DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4729
4730	/* Order reading of wptr vs. reading of IH ring data */
4731	rmb();
4732
4733	/* display interrupts */
4734	evergreen_irq_ack(rdev);
4735
4736	while (rptr != wptr) {
4737		/* wptr/rptr are in bytes! */
4738		ring_index = rptr / 4;
4739		src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4740		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4741
4742		switch (src_id) {
4743		case 1: /* D1 vblank/vline */
4744		case 2: /* D2 vblank/vline */
4745		case 3: /* D3 vblank/vline */
4746		case 4: /* D4 vblank/vline */
4747		case 5: /* D5 vblank/vline */
4748		case 6: /* D6 vblank/vline */
4749			crtc_idx = src_id - 1;
4750
4751			if (src_data == 0) { /* vblank */
4752				mask = LB_D1_VBLANK_INTERRUPT;
4753				event_name = "vblank";
4754
4755				if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4756					drm_handle_vblank(rdev_to_drm(rdev), crtc_idx);
4757					rdev->pm.vblank_sync = true;
4758					wake_up(&rdev->irq.vblank_queue);
4759				}
4760				if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4761					radeon_crtc_handle_vblank(rdev,
4762								  crtc_idx);
4763				}
4764
4765			} else if (src_data == 1) { /* vline */
4766				mask = LB_D1_VLINE_INTERRUPT;
4767				event_name = "vline";
4768			} else {
4769				DRM_DEBUG("Unhandled interrupt: %d %d\n",
4770					  src_id, src_data);
4771				break;
4772			}
4773
4774			if (!(disp_int[crtc_idx] & mask)) {
4775				DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4776					  crtc_idx + 1, event_name);
4777			}
4778
4779			disp_int[crtc_idx] &= ~mask;
4780			DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4781
4782			break;
4783		case 8: /* D1 page flip */
4784		case 10: /* D2 page flip */
4785		case 12: /* D3 page flip */
4786		case 14: /* D4 page flip */
4787		case 16: /* D5 page flip */
4788		case 18: /* D6 page flip */
4789			DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4790			if (radeon_use_pflipirq > 0)
4791				radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4792			break;
4793		case 42: /* HPD hotplug */
4794			if (src_data <= 5) {
4795				hpd_idx = src_data;
4796				mask = DC_HPD1_INTERRUPT;
4797				queue_hotplug = true;
4798				event_name = "HPD";
4799
4800			} else if (src_data <= 11) {
4801				hpd_idx = src_data - 6;
4802				mask = DC_HPD1_RX_INTERRUPT;
4803				queue_dp = true;
4804				event_name = "HPD_RX";
4805
4806			} else {
4807				DRM_DEBUG("Unhandled interrupt: %d %d\n",
4808					  src_id, src_data);
4809				break;
4810			}
4811
4812			if (!(disp_int[hpd_idx] & mask))
4813				DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4814
4815			disp_int[hpd_idx] &= ~mask;
4816			DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4817
4818			break;
4819		case 44: /* hdmi */
4820			afmt_idx = src_data;
 
 
 
4821			if (afmt_idx > 5) {
4822				DRM_ERROR("Unhandled interrupt: %d %d\n",
4823					  src_id, src_data);
4824				break;
4825			}
4826
4827			if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4828				DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4829
4830			afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4831			queue_hdmi = true;
4832			DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4833			break;
4834		case 96:
4835			DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4836			WREG32(SRBM_INT_ACK, 0x1);
4837			break;
4838		case 124: /* UVD */
4839			DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4840			radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4841			break;
4842		case 146:
4843		case 147:
4844			addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4845			status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4846			/* reset addr and status */
4847			WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4848			if (addr == 0x0 && status == 0x0)
4849				break;
4850			dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4851			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4852				addr);
4853			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4854				status);
4855			cayman_vm_decode_fault(rdev, status, addr);
4856			break;
4857		case 176: /* CP_INT in ring buffer */
4858		case 177: /* CP_INT in IB1 */
4859		case 178: /* CP_INT in IB2 */
4860			DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4861			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4862			break;
4863		case 181: /* CP EOP event */
4864			DRM_DEBUG("IH: CP EOP\n");
4865			if (rdev->family >= CHIP_CAYMAN) {
4866				switch (src_data) {
4867				case 0:
4868					radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4869					break;
4870				case 1:
4871					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4872					break;
4873				case 2:
4874					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4875					break;
4876				}
4877			} else
4878				radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4879			break;
4880		case 224: /* DMA trap event */
4881			DRM_DEBUG("IH: DMA trap\n");
4882			radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4883			break;
4884		case 230: /* thermal low to high */
4885			DRM_DEBUG("IH: thermal low to high\n");
4886			rdev->pm.dpm.thermal.high_to_low = false;
4887			queue_thermal = true;
4888			break;
4889		case 231: /* thermal high to low */
4890			DRM_DEBUG("IH: thermal high to low\n");
4891			rdev->pm.dpm.thermal.high_to_low = true;
4892			queue_thermal = true;
4893			break;
4894		case 233: /* GUI IDLE */
4895			DRM_DEBUG("IH: GUI idle\n");
4896			break;
4897		case 244: /* DMA trap event */
4898			if (rdev->family >= CHIP_CAYMAN) {
4899				DRM_DEBUG("IH: DMA1 trap\n");
4900				radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4901			}
4902			break;
4903		default:
4904			DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4905			break;
4906		}
4907
4908		/* wptr/rptr are in bytes! */
4909		rptr += 16;
4910		rptr &= rdev->ih.ptr_mask;
4911		WREG32(IH_RB_RPTR, rptr);
4912	}
4913	if (queue_dp)
4914		schedule_work(&rdev->dp_work);
4915	if (queue_hotplug)
4916		schedule_delayed_work(&rdev->hotplug_work, 0);
4917	if (queue_hdmi)
4918		schedule_work(&rdev->audio_work);
4919	if (queue_thermal && rdev->pm.dpm_enabled)
4920		schedule_work(&rdev->pm.dpm.thermal.work);
4921	rdev->ih.rptr = rptr;
4922	atomic_set(&rdev->ih.lock, 0);
4923
4924	/* make sure wptr hasn't changed while processing */
4925	wptr = evergreen_get_ih_wptr(rdev);
4926	if (wptr != rptr)
4927		goto restart_ih;
4928
4929	return IRQ_HANDLED;
4930}
4931
4932static void evergreen_uvd_init(struct radeon_device *rdev)
4933{
4934	int r;
4935
4936	if (!rdev->has_uvd)
4937		return;
4938
4939	r = radeon_uvd_init(rdev);
4940	if (r) {
4941		dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4942		/*
4943		 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4944		 * to early fails uvd_v2_2_resume() and thus nothing happens
4945		 * there. So it is pointless to try to go through that code
4946		 * hence why we disable uvd here.
4947		 */
4948		rdev->has_uvd = false;
4949		return;
4950	}
4951	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4952	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4953}
4954
4955static void evergreen_uvd_start(struct radeon_device *rdev)
4956{
4957	int r;
4958
4959	if (!rdev->has_uvd)
4960		return;
4961
4962	r = uvd_v2_2_resume(rdev);
4963	if (r) {
4964		dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4965		goto error;
4966	}
4967	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4968	if (r) {
4969		dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4970		goto error;
4971	}
4972	return;
4973
4974error:
4975	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4976}
4977
4978static void evergreen_uvd_resume(struct radeon_device *rdev)
4979{
4980	struct radeon_ring *ring;
4981	int r;
4982
4983	if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4984		return;
4985
4986	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4987	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4988	if (r) {
4989		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4990		return;
4991	}
4992	r = uvd_v1_0_init(rdev);
4993	if (r) {
4994		dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4995		return;
4996	}
4997}
4998
4999static int evergreen_startup(struct radeon_device *rdev)
5000{
5001	struct radeon_ring *ring;
5002	int r;
5003
5004	/* enable pcie gen2 link */
5005	evergreen_pcie_gen2_enable(rdev);
5006	/* enable aspm */
5007	evergreen_program_aspm(rdev);
5008
5009	/* scratch needs to be initialized before MC */
5010	r = r600_vram_scratch_init(rdev);
5011	if (r)
5012		return r;
5013
5014	evergreen_mc_program(rdev);
5015
5016	if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5017		r = ni_mc_load_microcode(rdev);
5018		if (r) {
5019			DRM_ERROR("Failed to load MC firmware!\n");
5020			return r;
5021		}
5022	}
5023
5024	if (rdev->flags & RADEON_IS_AGP) {
5025		evergreen_agp_enable(rdev);
5026	} else {
5027		r = evergreen_pcie_gart_enable(rdev);
5028		if (r)
5029			return r;
5030	}
5031	evergreen_gpu_init(rdev);
5032
5033	/* allocate rlc buffers */
5034	if (rdev->flags & RADEON_IS_IGP) {
5035		rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5036		rdev->rlc.reg_list_size =
5037			(u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5038		rdev->rlc.cs_data = evergreen_cs_data;
5039		r = sumo_rlc_init(rdev);
5040		if (r) {
5041			DRM_ERROR("Failed to init rlc BOs!\n");
5042			return r;
5043		}
5044	}
5045
5046	/* allocate wb buffer */
5047	r = radeon_wb_init(rdev);
5048	if (r)
5049		return r;
5050
5051	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5052	if (r) {
5053		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5054		return r;
5055	}
5056
5057	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5058	if (r) {
5059		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5060		return r;
5061	}
5062
5063	evergreen_uvd_start(rdev);
5064
5065	/* Enable IRQ */
5066	if (!rdev->irq.installed) {
5067		r = radeon_irq_kms_init(rdev);
5068		if (r)
5069			return r;
5070	}
5071
5072	r = r600_irq_init(rdev);
5073	if (r) {
5074		DRM_ERROR("radeon: IH init failed (%d).\n", r);
5075		radeon_irq_kms_fini(rdev);
5076		return r;
5077	}
5078	evergreen_irq_set(rdev);
5079
5080	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5081	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5082			     RADEON_CP_PACKET2);
5083	if (r)
5084		return r;
5085
5086	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5087	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5088			     DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5089	if (r)
5090		return r;
5091
5092	r = evergreen_cp_load_microcode(rdev);
5093	if (r)
5094		return r;
5095	r = evergreen_cp_resume(rdev);
5096	if (r)
5097		return r;
5098	r = r600_dma_resume(rdev);
5099	if (r)
5100		return r;
5101
5102	evergreen_uvd_resume(rdev);
5103
5104	r = radeon_ib_pool_init(rdev);
5105	if (r) {
5106		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5107		return r;
5108	}
5109
5110	r = radeon_audio_init(rdev);
5111	if (r) {
5112		DRM_ERROR("radeon: audio init failed\n");
5113		return r;
5114	}
5115
5116	return 0;
5117}
5118
5119int evergreen_resume(struct radeon_device *rdev)
5120{
5121	int r;
5122
5123	/* reset the asic, the gfx blocks are often in a bad state
5124	 * after the driver is unloaded or after a resume
5125	 */
5126	if (radeon_asic_reset(rdev))
5127		dev_warn(rdev->dev, "GPU reset failed !\n");
5128	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5129	 * posting will perform necessary task to bring back GPU into good
5130	 * shape.
5131	 */
5132	/* post card */
5133	atom_asic_init(rdev->mode_info.atom_context);
5134
5135	/* init golden registers */
5136	evergreen_init_golden_registers(rdev);
5137
5138	if (rdev->pm.pm_method == PM_METHOD_DPM)
5139		radeon_pm_resume(rdev);
5140
5141	rdev->accel_working = true;
5142	r = evergreen_startup(rdev);
5143	if (r) {
5144		DRM_ERROR("evergreen startup failed on resume\n");
5145		rdev->accel_working = false;
5146		return r;
5147	}
5148
5149	return r;
5150
5151}
5152
5153int evergreen_suspend(struct radeon_device *rdev)
5154{
5155	radeon_pm_suspend(rdev);
5156	radeon_audio_fini(rdev);
5157	if (rdev->has_uvd) {
5158		radeon_uvd_suspend(rdev);
5159		uvd_v1_0_fini(rdev);
 
5160	}
5161	r700_cp_stop(rdev);
5162	r600_dma_stop(rdev);
5163	evergreen_irq_suspend(rdev);
5164	radeon_wb_disable(rdev);
5165	evergreen_pcie_gart_disable(rdev);
5166
5167	return 0;
5168}
5169
5170/* Plan is to move initialization in that function and use
5171 * helper function so that radeon_device_init pretty much
5172 * do nothing more than calling asic specific function. This
5173 * should also allow to remove a bunch of callback function
5174 * like vram_info.
5175 */
5176int evergreen_init(struct radeon_device *rdev)
5177{
5178	int r;
5179
5180	/* Read BIOS */
5181	if (!radeon_get_bios(rdev)) {
5182		if (ASIC_IS_AVIVO(rdev))
5183			return -EINVAL;
5184	}
5185	/* Must be an ATOMBIOS */
5186	if (!rdev->is_atom_bios) {
5187		dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5188		return -EINVAL;
5189	}
5190	r = radeon_atombios_init(rdev);
5191	if (r)
5192		return r;
5193	/* reset the asic, the gfx blocks are often in a bad state
5194	 * after the driver is unloaded or after a resume
5195	 */
5196	if (radeon_asic_reset(rdev))
5197		dev_warn(rdev->dev, "GPU reset failed !\n");
5198	/* Post card if necessary */
5199	if (!radeon_card_posted(rdev)) {
5200		if (!rdev->bios) {
5201			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5202			return -EINVAL;
5203		}
5204		DRM_INFO("GPU not posted. posting now...\n");
5205		atom_asic_init(rdev->mode_info.atom_context);
5206	}
5207	/* init golden registers */
5208	evergreen_init_golden_registers(rdev);
5209	/* Initialize scratch registers */
5210	r600_scratch_init(rdev);
5211	/* Initialize surface registers */
5212	radeon_surface_init(rdev);
5213	/* Initialize clocks */
5214	radeon_get_clock_info(rdev_to_drm(rdev));
5215	/* Fence driver */
5216	radeon_fence_driver_init(rdev);
5217	/* initialize AGP */
5218	if (rdev->flags & RADEON_IS_AGP) {
5219		r = radeon_agp_init(rdev);
5220		if (r)
5221			radeon_agp_disable(rdev);
5222	}
5223	/* initialize memory controller */
5224	r = evergreen_mc_init(rdev);
5225	if (r)
5226		return r;
5227	/* Memory manager */
5228	r = radeon_bo_init(rdev);
5229	if (r)
5230		return r;
5231
5232	if (ASIC_IS_DCE5(rdev)) {
5233		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5234			r = ni_init_microcode(rdev);
5235			if (r) {
5236				DRM_ERROR("Failed to load firmware!\n");
5237				return r;
5238			}
5239		}
5240	} else {
5241		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5242			r = r600_init_microcode(rdev);
5243			if (r) {
5244				DRM_ERROR("Failed to load firmware!\n");
5245				return r;
5246			}
5247		}
5248	}
5249
5250	/* Initialize power management */
5251	radeon_pm_init(rdev);
5252
5253	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5254	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5255
5256	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5257	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5258
5259	evergreen_uvd_init(rdev);
5260
5261	rdev->ih.ring_obj = NULL;
5262	r600_ih_ring_init(rdev, 64 * 1024);
5263
5264	r = r600_pcie_gart_init(rdev);
5265	if (r)
5266		return r;
5267
5268	rdev->accel_working = true;
5269	r = evergreen_startup(rdev);
5270	if (r) {
5271		dev_err(rdev->dev, "disabling GPU acceleration\n");
5272		r700_cp_fini(rdev);
5273		r600_dma_fini(rdev);
5274		r600_irq_fini(rdev);
5275		if (rdev->flags & RADEON_IS_IGP)
5276			sumo_rlc_fini(rdev);
5277		radeon_wb_fini(rdev);
5278		radeon_ib_pool_fini(rdev);
5279		radeon_irq_kms_fini(rdev);
5280		evergreen_pcie_gart_fini(rdev);
5281		rdev->accel_working = false;
5282	}
5283
5284	/* Don't start up if the MC ucode is missing on BTC parts.
5285	 * The default clocks and voltages before the MC ucode
5286	 * is loaded are not suffient for advanced operations.
5287	 */
5288	if (ASIC_IS_DCE5(rdev)) {
5289		if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5290			DRM_ERROR("radeon: MC ucode required for NI+.\n");
5291			return -EINVAL;
5292		}
5293	}
5294
5295	return 0;
5296}
5297
5298void evergreen_fini(struct radeon_device *rdev)
5299{
5300	radeon_pm_fini(rdev);
5301	radeon_audio_fini(rdev);
5302	r700_cp_fini(rdev);
5303	r600_dma_fini(rdev);
5304	r600_irq_fini(rdev);
5305	if (rdev->flags & RADEON_IS_IGP)
5306		sumo_rlc_fini(rdev);
5307	radeon_wb_fini(rdev);
5308	radeon_ib_pool_fini(rdev);
5309	radeon_irq_kms_fini(rdev);
5310	uvd_v1_0_fini(rdev);
5311	radeon_uvd_fini(rdev);
5312	evergreen_pcie_gart_fini(rdev);
5313	r600_vram_scratch_fini(rdev);
5314	radeon_gem_fini(rdev);
5315	radeon_fence_driver_fini(rdev);
5316	radeon_agp_fini(rdev);
5317	radeon_bo_fini(rdev);
5318	radeon_atombios_fini(rdev);
5319	kfree(rdev->bios);
5320	rdev->bios = NULL;
5321}
5322
5323void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5324{
5325	u32 link_width_cntl, speed_cntl;
5326
5327	if (radeon_pcie_gen2 == 0)
5328		return;
5329
5330	if (rdev->flags & RADEON_IS_IGP)
5331		return;
5332
5333	if (!(rdev->flags & RADEON_IS_PCIE))
5334		return;
5335
5336	/* x2 cards have a special sequence */
5337	if (ASIC_IS_X2(rdev))
5338		return;
5339
5340	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5341		(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5342		return;
5343
5344	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5345	if (speed_cntl & LC_CURRENT_DATA_RATE) {
5346		DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5347		return;
5348	}
5349
5350	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5351
5352	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5353	    (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5354
5355		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5356		link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5357		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5358
5359		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5360		speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5361		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5362
5363		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5364		speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5365		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5366
5367		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5368		speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5369		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5370
5371		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5372		speed_cntl |= LC_GEN2_EN_STRAP;
5373		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5374
5375	} else {
5376		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5377		/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5378		if (1)
5379			link_width_cntl |= LC_UPCONFIGURE_DIS;
5380		else
5381			link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5382		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5383	}
5384}
5385
5386void evergreen_program_aspm(struct radeon_device *rdev)
5387{
5388	u32 data, orig;
5389	u32 pcie_lc_cntl, pcie_lc_cntl_old;
5390	bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5391	/* fusion_platform = true
5392	 * if the system is a fusion system
5393	 * (APU or DGPU in a fusion system).
5394	 * todo: check if the system is a fusion platform.
5395	 */
5396	bool fusion_platform = false;
5397
5398	if (radeon_aspm == 0)
5399		return;
5400
5401	if (!(rdev->flags & RADEON_IS_PCIE))
5402		return;
5403
5404	switch (rdev->family) {
5405	case CHIP_CYPRESS:
5406	case CHIP_HEMLOCK:
5407	case CHIP_JUNIPER:
5408	case CHIP_REDWOOD:
5409	case CHIP_CEDAR:
5410	case CHIP_SUMO:
5411	case CHIP_SUMO2:
5412	case CHIP_PALM:
5413	case CHIP_ARUBA:
5414		disable_l0s = true;
5415		break;
5416	default:
5417		disable_l0s = false;
5418		break;
5419	}
5420
5421	if (rdev->flags & RADEON_IS_IGP)
5422		fusion_platform = true; /* XXX also dGPUs in a fusion system */
5423
5424	data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5425	if (fusion_platform)
5426		data &= ~MULTI_PIF;
5427	else
5428		data |= MULTI_PIF;
5429	if (data != orig)
5430		WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5431
5432	data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5433	if (fusion_platform)
5434		data &= ~MULTI_PIF;
5435	else
5436		data |= MULTI_PIF;
5437	if (data != orig)
5438		WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5439
5440	pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5441	pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5442	if (!disable_l0s) {
5443		if (rdev->family >= CHIP_BARTS)
5444			pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5445		else
5446			pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5447	}
5448
5449	if (!disable_l1) {
5450		if (rdev->family >= CHIP_BARTS)
5451			pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5452		else
5453			pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5454
5455		if (!disable_plloff_in_l1) {
5456			data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5457			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5458			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5459			if (data != orig)
5460				WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5461
5462			data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5463			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5464			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5465			if (data != orig)
5466				WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5467
5468			data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5469			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5470			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5471			if (data != orig)
5472				WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5473
5474			data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5475			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5476			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5477			if (data != orig)
5478				WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5479
5480			if (rdev->family >= CHIP_BARTS) {
5481				data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5482				data &= ~PLL_RAMP_UP_TIME_0_MASK;
5483				data |= PLL_RAMP_UP_TIME_0(4);
5484				if (data != orig)
5485					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5486
5487				data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5488				data &= ~PLL_RAMP_UP_TIME_1_MASK;
5489				data |= PLL_RAMP_UP_TIME_1(4);
5490				if (data != orig)
5491					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5492
5493				data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5494				data &= ~PLL_RAMP_UP_TIME_0_MASK;
5495				data |= PLL_RAMP_UP_TIME_0(4);
5496				if (data != orig)
5497					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5498
5499				data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5500				data &= ~PLL_RAMP_UP_TIME_1_MASK;
5501				data |= PLL_RAMP_UP_TIME_1(4);
5502				if (data != orig)
5503					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5504			}
5505
5506			data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5507			data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5508			data |= LC_DYN_LANES_PWR_STATE(3);
5509			if (data != orig)
5510				WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5511
5512			if (rdev->family >= CHIP_BARTS) {
5513				data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5514				data &= ~LS2_EXIT_TIME_MASK;
5515				data |= LS2_EXIT_TIME(1);
5516				if (data != orig)
5517					WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5518
5519				data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5520				data &= ~LS2_EXIT_TIME_MASK;
5521				data |= LS2_EXIT_TIME(1);
5522				if (data != orig)
5523					WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5524			}
5525		}
5526	}
5527
5528	/* evergreen parts only */
5529	if (rdev->family < CHIP_BARTS)
5530		pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5531
5532	if (pcie_lc_cntl != pcie_lc_cntl_old)
5533		WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5534}