1/*
   2 * Copyright 2008 Advanced Micro Devices, Inc.
   3 * Copyright 2008 Red Hat Inc.
   4 * Copyright 2009 Jerome Glisse.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 *
  24 * Authors: Dave Airlie
  25 *          Alex Deucher
  26 *          Jerome Glisse
  27 */
  28
  29#include <linux/firmware.h>
  30#include <linux/slab.h>
  31
  32#include <drm/drm_device.h>
  33#include <drm/drm_pci.h>
  34#include <drm/radeon_drm.h>
  35
  36#include "atom.h"
  37#include "avivod.h"
  38#include "radeon.h"
  39#include "radeon_asic.h"
  40#include "radeon_audio.h"
 
  41#include "rv770d.h"
 
 
  42
  43#define R700_PFP_UCODE_SIZE 848
  44#define R700_PM4_UCODE_SIZE 1360
  45
  46static void rv770_gpu_init(struct radeon_device *rdev);
  47void rv770_fini(struct radeon_device *rdev);
  48static void rv770_pcie_gen2_enable(struct radeon_device *rdev);
  49int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk);
  50
  51int rv770_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
  52{
  53	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
  54	int r;
  55
  56	/* RV740 uses evergreen uvd clk programming */
  57	if (rdev->family == CHIP_RV740)
  58		return evergreen_set_uvd_clocks(rdev, vclk, dclk);
  59
  60	/* bypass vclk and dclk with bclk */
  61	WREG32_P(CG_UPLL_FUNC_CNTL_2,
  62		 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
  63		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
  64
  65	if (!vclk || !dclk) {
  66		/* keep the Bypass mode, put PLL to sleep */
  67		WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
  68		return 0;
  69	}
  70
  71	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000,
  72					  43663, 0x03FFFFFE, 1, 30, ~0,
  73					  &fb_div, &vclk_div, &dclk_div);
  74	if (r)
  75		return r;
  76
  77	fb_div |= 1;
  78	vclk_div -= 1;
  79	dclk_div -= 1;
  80
  81	/* set UPLL_FB_DIV to 0x50000 */
  82	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(0x50000), ~UPLL_FB_DIV_MASK);
  83
  84	/* deassert UPLL_RESET and UPLL_SLEEP */
  85	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~(UPLL_RESET_MASK | UPLL_SLEEP_MASK));
  86
  87	/* assert BYPASS EN and FB_DIV[0] <- ??? why? */
  88	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
  89	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(1), ~UPLL_FB_DIV(1));
  90
  91	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
  92	if (r)
  93		return r;
  94
  95	/* assert PLL_RESET */
  96	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
  97
  98	/* set the required FB_DIV, REF_DIV, Post divder values */
  99	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_REF_DIV(1), ~UPLL_REF_DIV_MASK);
 100	WREG32_P(CG_UPLL_FUNC_CNTL_2,
 101		 UPLL_SW_HILEN(vclk_div >> 1) |
 102		 UPLL_SW_LOLEN((vclk_div >> 1) + (vclk_div & 1)) |
 103		 UPLL_SW_HILEN2(dclk_div >> 1) |
 104		 UPLL_SW_LOLEN2((dclk_div >> 1) + (dclk_div & 1)),
 105		 ~UPLL_SW_MASK);
 106
 107	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div),
 108		 ~UPLL_FB_DIV_MASK);
 109
 110	/* give the PLL some time to settle */
 111	mdelay(15);
 112
 113	/* deassert PLL_RESET */
 114	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
 115
 116	mdelay(15);
 117
 118	/* deassert BYPASS EN and FB_DIV[0] <- ??? why? */
 119	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
 120	WREG32_P(CG_UPLL_FUNC_CNTL_3, 0, ~UPLL_FB_DIV(1));
 121
 122	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
 123	if (r)
 124		return r;
 125
 126	/* switch VCLK and DCLK selection */
 127	WREG32_P(CG_UPLL_FUNC_CNTL_2,
 128		 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
 129		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
 130
 131	mdelay(100);
 132
 133	return 0;
 134}
 135
 136static const u32 r7xx_golden_registers[] =
 137{
 138	0x8d00, 0xffffffff, 0x0e0e0074,
 139	0x8d04, 0xffffffff, 0x013a2b34,
 140	0x9508, 0xffffffff, 0x00000002,
 141	0x8b20, 0xffffffff, 0,
 142	0x88c4, 0xffffffff, 0x000000c2,
 143	0x28350, 0xffffffff, 0,
 144	0x9058, 0xffffffff, 0x0fffc40f,
 145	0x240c, 0xffffffff, 0x00000380,
 146	0x733c, 0xffffffff, 0x00000002,
 147	0x2650, 0x00040000, 0,
 148	0x20bc, 0x00040000, 0,
 149	0x7300, 0xffffffff, 0x001000f0
 150};
 151
 152static const u32 r7xx_golden_dyn_gpr_registers[] =
 153{
 154	0x8db0, 0xffffffff, 0x98989898,
 155	0x8db4, 0xffffffff, 0x98989898,
 156	0x8db8, 0xffffffff, 0x98989898,
 157	0x8dbc, 0xffffffff, 0x98989898,
 158	0x8dc0, 0xffffffff, 0x98989898,
 159	0x8dc4, 0xffffffff, 0x98989898,
 160	0x8dc8, 0xffffffff, 0x98989898,
 161	0x8dcc, 0xffffffff, 0x98989898,
 162	0x88c4, 0xffffffff, 0x00000082
 163};
 164
 165static const u32 rv770_golden_registers[] =
 166{
 167	0x562c, 0xffffffff, 0,
 168	0x3f90, 0xffffffff, 0,
 169	0x9148, 0xffffffff, 0,
 170	0x3f94, 0xffffffff, 0,
 171	0x914c, 0xffffffff, 0,
 172	0x9698, 0x18000000, 0x18000000
 173};
 174
 175static const u32 rv770ce_golden_registers[] =
 176{
 177	0x562c, 0xffffffff, 0,
 178	0x3f90, 0xffffffff, 0x00cc0000,
 179	0x9148, 0xffffffff, 0x00cc0000,
 180	0x3f94, 0xffffffff, 0x00cc0000,
 181	0x914c, 0xffffffff, 0x00cc0000,
 182	0x9b7c, 0xffffffff, 0x00fa0000,
 183	0x3f8c, 0xffffffff, 0x00fa0000,
 184	0x9698, 0x18000000, 0x18000000
 185};
 186
 187static const u32 rv770_mgcg_init[] =
 188{
 189	0x8bcc, 0xffffffff, 0x130300f9,
 190	0x5448, 0xffffffff, 0x100,
 191	0x55e4, 0xffffffff, 0x100,
 192	0x160c, 0xffffffff, 0x100,
 193	0x5644, 0xffffffff, 0x100,
 194	0xc164, 0xffffffff, 0x100,
 195	0x8a18, 0xffffffff, 0x100,
 196	0x897c, 0xffffffff, 0x8000100,
 197	0x8b28, 0xffffffff, 0x3c000100,
 198	0x9144, 0xffffffff, 0x100,
 199	0x9a1c, 0xffffffff, 0x10000,
 200	0x9a50, 0xffffffff, 0x100,
 201	0x9a1c, 0xffffffff, 0x10001,
 202	0x9a50, 0xffffffff, 0x100,
 203	0x9a1c, 0xffffffff, 0x10002,
 204	0x9a50, 0xffffffff, 0x100,
 205	0x9a1c, 0xffffffff, 0x10003,
 206	0x9a50, 0xffffffff, 0x100,
 207	0x9a1c, 0xffffffff, 0x0,
 208	0x9870, 0xffffffff, 0x100,
 209	0x8d58, 0xffffffff, 0x100,
 210	0x9500, 0xffffffff, 0x0,
 211	0x9510, 0xffffffff, 0x100,
 212	0x9500, 0xffffffff, 0x1,
 213	0x9510, 0xffffffff, 0x100,
 214	0x9500, 0xffffffff, 0x2,
 215	0x9510, 0xffffffff, 0x100,
 216	0x9500, 0xffffffff, 0x3,
 217	0x9510, 0xffffffff, 0x100,
 218	0x9500, 0xffffffff, 0x4,
 219	0x9510, 0xffffffff, 0x100,
 220	0x9500, 0xffffffff, 0x5,
 221	0x9510, 0xffffffff, 0x100,
 222	0x9500, 0xffffffff, 0x6,
 223	0x9510, 0xffffffff, 0x100,
 224	0x9500, 0xffffffff, 0x7,
 225	0x9510, 0xffffffff, 0x100,
 226	0x9500, 0xffffffff, 0x8,
 227	0x9510, 0xffffffff, 0x100,
 228	0x9500, 0xffffffff, 0x9,
 229	0x9510, 0xffffffff, 0x100,
 230	0x9500, 0xffffffff, 0x8000,
 231	0x9490, 0xffffffff, 0x0,
 232	0x949c, 0xffffffff, 0x100,
 233	0x9490, 0xffffffff, 0x1,
 234	0x949c, 0xffffffff, 0x100,
 235	0x9490, 0xffffffff, 0x2,
 236	0x949c, 0xffffffff, 0x100,
 237	0x9490, 0xffffffff, 0x3,
 238	0x949c, 0xffffffff, 0x100,
 239	0x9490, 0xffffffff, 0x4,
 240	0x949c, 0xffffffff, 0x100,
 241	0x9490, 0xffffffff, 0x5,
 242	0x949c, 0xffffffff, 0x100,
 243	0x9490, 0xffffffff, 0x6,
 244	0x949c, 0xffffffff, 0x100,
 245	0x9490, 0xffffffff, 0x7,
 246	0x949c, 0xffffffff, 0x100,
 247	0x9490, 0xffffffff, 0x8,
 248	0x949c, 0xffffffff, 0x100,
 249	0x9490, 0xffffffff, 0x9,
 250	0x949c, 0xffffffff, 0x100,
 251	0x9490, 0xffffffff, 0x8000,
 252	0x9604, 0xffffffff, 0x0,
 253	0x9654, 0xffffffff, 0x100,
 254	0x9604, 0xffffffff, 0x1,
 255	0x9654, 0xffffffff, 0x100,
 256	0x9604, 0xffffffff, 0x2,
 257	0x9654, 0xffffffff, 0x100,
 258	0x9604, 0xffffffff, 0x3,
 259	0x9654, 0xffffffff, 0x100,
 260	0x9604, 0xffffffff, 0x4,
 261	0x9654, 0xffffffff, 0x100,
 262	0x9604, 0xffffffff, 0x5,
 263	0x9654, 0xffffffff, 0x100,
 264	0x9604, 0xffffffff, 0x6,
 265	0x9654, 0xffffffff, 0x100,
 266	0x9604, 0xffffffff, 0x7,
 267	0x9654, 0xffffffff, 0x100,
 268	0x9604, 0xffffffff, 0x8,
 269	0x9654, 0xffffffff, 0x100,
 270	0x9604, 0xffffffff, 0x9,
 271	0x9654, 0xffffffff, 0x100,
 272	0x9604, 0xffffffff, 0x80000000,
 273	0x9030, 0xffffffff, 0x100,
 274	0x9034, 0xffffffff, 0x100,
 275	0x9038, 0xffffffff, 0x100,
 276	0x903c, 0xffffffff, 0x100,
 277	0x9040, 0xffffffff, 0x100,
 278	0xa200, 0xffffffff, 0x100,
 279	0xa204, 0xffffffff, 0x100,
 280	0xa208, 0xffffffff, 0x100,
 281	0xa20c, 0xffffffff, 0x100,
 282	0x971c, 0xffffffff, 0x100,
 283	0x915c, 0xffffffff, 0x00020001,
 284	0x9160, 0xffffffff, 0x00040003,
 285	0x916c, 0xffffffff, 0x00060005,
 286	0x9170, 0xffffffff, 0x00080007,
 287	0x9174, 0xffffffff, 0x000a0009,
 288	0x9178, 0xffffffff, 0x000c000b,
 289	0x917c, 0xffffffff, 0x000e000d,
 290	0x9180, 0xffffffff, 0x0010000f,
 291	0x918c, 0xffffffff, 0x00120011,
 292	0x9190, 0xffffffff, 0x00140013,
 293	0x9194, 0xffffffff, 0x00020001,
 294	0x9198, 0xffffffff, 0x00040003,
 295	0x919c, 0xffffffff, 0x00060005,
 296	0x91a8, 0xffffffff, 0x00080007,
 297	0x91ac, 0xffffffff, 0x000a0009,
 298	0x91b0, 0xffffffff, 0x000c000b,
 299	0x91b4, 0xffffffff, 0x000e000d,
 300	0x91b8, 0xffffffff, 0x0010000f,
 301	0x91c4, 0xffffffff, 0x00120011,
 302	0x91c8, 0xffffffff, 0x00140013,
 303	0x91cc, 0xffffffff, 0x00020001,
 304	0x91d0, 0xffffffff, 0x00040003,
 305	0x91d4, 0xffffffff, 0x00060005,
 306	0x91e0, 0xffffffff, 0x00080007,
 307	0x91e4, 0xffffffff, 0x000a0009,
 308	0x91e8, 0xffffffff, 0x000c000b,
 309	0x91ec, 0xffffffff, 0x00020001,
 310	0x91f0, 0xffffffff, 0x00040003,
 311	0x91f4, 0xffffffff, 0x00060005,
 312	0x9200, 0xffffffff, 0x00080007,
 313	0x9204, 0xffffffff, 0x000a0009,
 314	0x9208, 0xffffffff, 0x000c000b,
 315	0x920c, 0xffffffff, 0x000e000d,
 316	0x9210, 0xffffffff, 0x0010000f,
 317	0x921c, 0xffffffff, 0x00120011,
 318	0x9220, 0xffffffff, 0x00140013,
 319	0x9224, 0xffffffff, 0x00020001,
 320	0x9228, 0xffffffff, 0x00040003,
 321	0x922c, 0xffffffff, 0x00060005,
 322	0x9238, 0xffffffff, 0x00080007,
 323	0x923c, 0xffffffff, 0x000a0009,
 324	0x9240, 0xffffffff, 0x000c000b,
 325	0x9244, 0xffffffff, 0x000e000d,
 326	0x9248, 0xffffffff, 0x0010000f,
 327	0x9254, 0xffffffff, 0x00120011,
 328	0x9258, 0xffffffff, 0x00140013,
 329	0x925c, 0xffffffff, 0x00020001,
 330	0x9260, 0xffffffff, 0x00040003,
 331	0x9264, 0xffffffff, 0x00060005,
 332	0x9270, 0xffffffff, 0x00080007,
 333	0x9274, 0xffffffff, 0x000a0009,
 334	0x9278, 0xffffffff, 0x000c000b,
 335	0x927c, 0xffffffff, 0x000e000d,
 336	0x9280, 0xffffffff, 0x0010000f,
 337	0x928c, 0xffffffff, 0x00120011,
 338	0x9290, 0xffffffff, 0x00140013,
 339	0x9294, 0xffffffff, 0x00020001,
 340	0x929c, 0xffffffff, 0x00040003,
 341	0x92a0, 0xffffffff, 0x00060005,
 342	0x92a4, 0xffffffff, 0x00080007
 343};
 344
 345static const u32 rv710_golden_registers[] =
 346{
 347	0x3f90, 0x00ff0000, 0x00fc0000,
 348	0x9148, 0x00ff0000, 0x00fc0000,
 349	0x3f94, 0x00ff0000, 0x00fc0000,
 350	0x914c, 0x00ff0000, 0x00fc0000,
 351	0xb4c, 0x00000020, 0x00000020,
 352	0xa180, 0xffffffff, 0x00003f3f
 353};
 354
 355static const u32 rv710_mgcg_init[] =
 356{
 357	0x8bcc, 0xffffffff, 0x13030040,
 358	0x5448, 0xffffffff, 0x100,
 359	0x55e4, 0xffffffff, 0x100,
 360	0x160c, 0xffffffff, 0x100,
 361	0x5644, 0xffffffff, 0x100,
 362	0xc164, 0xffffffff, 0x100,
 363	0x8a18, 0xffffffff, 0x100,
 364	0x897c, 0xffffffff, 0x8000100,
 365	0x8b28, 0xffffffff, 0x3c000100,
 366	0x9144, 0xffffffff, 0x100,
 367	0x9a1c, 0xffffffff, 0x10000,
 368	0x9a50, 0xffffffff, 0x100,
 369	0x9a1c, 0xffffffff, 0x0,
 370	0x9870, 0xffffffff, 0x100,
 371	0x8d58, 0xffffffff, 0x100,
 372	0x9500, 0xffffffff, 0x0,
 373	0x9510, 0xffffffff, 0x100,
 374	0x9500, 0xffffffff, 0x1,
 375	0x9510, 0xffffffff, 0x100,
 376	0x9500, 0xffffffff, 0x8000,
 377	0x9490, 0xffffffff, 0x0,
 378	0x949c, 0xffffffff, 0x100,
 379	0x9490, 0xffffffff, 0x1,
 380	0x949c, 0xffffffff, 0x100,
 381	0x9490, 0xffffffff, 0x8000,
 382	0x9604, 0xffffffff, 0x0,
 383	0x9654, 0xffffffff, 0x100,
 384	0x9604, 0xffffffff, 0x1,
 385	0x9654, 0xffffffff, 0x100,
 386	0x9604, 0xffffffff, 0x80000000,
 387	0x9030, 0xffffffff, 0x100,
 388	0x9034, 0xffffffff, 0x100,
 389	0x9038, 0xffffffff, 0x100,
 390	0x903c, 0xffffffff, 0x100,
 391	0x9040, 0xffffffff, 0x100,
 392	0xa200, 0xffffffff, 0x100,
 393	0xa204, 0xffffffff, 0x100,
 394	0xa208, 0xffffffff, 0x100,
 395	0xa20c, 0xffffffff, 0x100,
 396	0x971c, 0xffffffff, 0x100,
 397	0x915c, 0xffffffff, 0x00020001,
 398	0x9174, 0xffffffff, 0x00000003,
 399	0x9178, 0xffffffff, 0x00050001,
 400	0x917c, 0xffffffff, 0x00030002,
 401	0x918c, 0xffffffff, 0x00000004,
 402	0x9190, 0xffffffff, 0x00070006,
 403	0x9194, 0xffffffff, 0x00050001,
 404	0x9198, 0xffffffff, 0x00030002,
 405	0x91a8, 0xffffffff, 0x00000004,
 406	0x91ac, 0xffffffff, 0x00070006,
 407	0x91e8, 0xffffffff, 0x00000001,
 408	0x9294, 0xffffffff, 0x00000001,
 409	0x929c, 0xffffffff, 0x00000002,
 410	0x92a0, 0xffffffff, 0x00040003,
 411	0x9150, 0xffffffff, 0x4d940000
 412};
 413
 414static const u32 rv730_golden_registers[] =
 415{
 416	0x3f90, 0x00ff0000, 0x00f00000,
 417	0x9148, 0x00ff0000, 0x00f00000,
 418	0x3f94, 0x00ff0000, 0x00f00000,
 419	0x914c, 0x00ff0000, 0x00f00000,
 420	0x900c, 0xffffffff, 0x003b033f,
 421	0xb4c, 0x00000020, 0x00000020,
 422	0xa180, 0xffffffff, 0x00003f3f
 423};
 424
 425static const u32 rv730_mgcg_init[] =
 426{
 427	0x8bcc, 0xffffffff, 0x130300f9,
 428	0x5448, 0xffffffff, 0x100,
 429	0x55e4, 0xffffffff, 0x100,
 430	0x160c, 0xffffffff, 0x100,
 431	0x5644, 0xffffffff, 0x100,
 432	0xc164, 0xffffffff, 0x100,
 433	0x8a18, 0xffffffff, 0x100,
 434	0x897c, 0xffffffff, 0x8000100,
 435	0x8b28, 0xffffffff, 0x3c000100,
 436	0x9144, 0xffffffff, 0x100,
 437	0x9a1c, 0xffffffff, 0x10000,
 438	0x9a50, 0xffffffff, 0x100,
 439	0x9a1c, 0xffffffff, 0x10001,
 440	0x9a50, 0xffffffff, 0x100,
 441	0x9a1c, 0xffffffff, 0x0,
 442	0x9870, 0xffffffff, 0x100,
 443	0x8d58, 0xffffffff, 0x100,
 444	0x9500, 0xffffffff, 0x0,
 445	0x9510, 0xffffffff, 0x100,
 446	0x9500, 0xffffffff, 0x1,
 447	0x9510, 0xffffffff, 0x100,
 448	0x9500, 0xffffffff, 0x2,
 449	0x9510, 0xffffffff, 0x100,
 450	0x9500, 0xffffffff, 0x3,
 451	0x9510, 0xffffffff, 0x100,
 452	0x9500, 0xffffffff, 0x4,
 453	0x9510, 0xffffffff, 0x100,
 454	0x9500, 0xffffffff, 0x5,
 455	0x9510, 0xffffffff, 0x100,
 456	0x9500, 0xffffffff, 0x6,
 457	0x9510, 0xffffffff, 0x100,
 458	0x9500, 0xffffffff, 0x7,
 459	0x9510, 0xffffffff, 0x100,
 460	0x9500, 0xffffffff, 0x8000,
 461	0x9490, 0xffffffff, 0x0,
 462	0x949c, 0xffffffff, 0x100,
 463	0x9490, 0xffffffff, 0x1,
 464	0x949c, 0xffffffff, 0x100,
 465	0x9490, 0xffffffff, 0x2,
 466	0x949c, 0xffffffff, 0x100,
 467	0x9490, 0xffffffff, 0x3,
 468	0x949c, 0xffffffff, 0x100,
 469	0x9490, 0xffffffff, 0x4,
 470	0x949c, 0xffffffff, 0x100,
 471	0x9490, 0xffffffff, 0x5,
 472	0x949c, 0xffffffff, 0x100,
 473	0x9490, 0xffffffff, 0x6,
 474	0x949c, 0xffffffff, 0x100,
 475	0x9490, 0xffffffff, 0x7,
 476	0x949c, 0xffffffff, 0x100,
 477	0x9490, 0xffffffff, 0x8000,
 478	0x9604, 0xffffffff, 0x0,
 479	0x9654, 0xffffffff, 0x100,
 480	0x9604, 0xffffffff, 0x1,
 481	0x9654, 0xffffffff, 0x100,
 482	0x9604, 0xffffffff, 0x2,
 483	0x9654, 0xffffffff, 0x100,
 484	0x9604, 0xffffffff, 0x3,
 485	0x9654, 0xffffffff, 0x100,
 486	0x9604, 0xffffffff, 0x4,
 487	0x9654, 0xffffffff, 0x100,
 488	0x9604, 0xffffffff, 0x5,
 489	0x9654, 0xffffffff, 0x100,
 490	0x9604, 0xffffffff, 0x6,
 491	0x9654, 0xffffffff, 0x100,
 492	0x9604, 0xffffffff, 0x7,
 493	0x9654, 0xffffffff, 0x100,
 494	0x9604, 0xffffffff, 0x80000000,
 495	0x9030, 0xffffffff, 0x100,
 496	0x9034, 0xffffffff, 0x100,
 497	0x9038, 0xffffffff, 0x100,
 498	0x903c, 0xffffffff, 0x100,
 499	0x9040, 0xffffffff, 0x100,
 500	0xa200, 0xffffffff, 0x100,
 501	0xa204, 0xffffffff, 0x100,
 502	0xa208, 0xffffffff, 0x100,
 503	0xa20c, 0xffffffff, 0x100,
 504	0x971c, 0xffffffff, 0x100,
 505	0x915c, 0xffffffff, 0x00020001,
 506	0x916c, 0xffffffff, 0x00040003,
 507	0x9170, 0xffffffff, 0x00000005,
 508	0x9178, 0xffffffff, 0x00050001,
 509	0x917c, 0xffffffff, 0x00030002,
 510	0x918c, 0xffffffff, 0x00000004,
 511	0x9190, 0xffffffff, 0x00070006,
 512	0x9194, 0xffffffff, 0x00050001,
 513	0x9198, 0xffffffff, 0x00030002,
 514	0x91a8, 0xffffffff, 0x00000004,
 515	0x91ac, 0xffffffff, 0x00070006,
 516	0x91b0, 0xffffffff, 0x00050001,
 517	0x91b4, 0xffffffff, 0x00030002,
 518	0x91c4, 0xffffffff, 0x00000004,
 519	0x91c8, 0xffffffff, 0x00070006,
 520	0x91cc, 0xffffffff, 0x00050001,
 521	0x91d0, 0xffffffff, 0x00030002,
 522	0x91e0, 0xffffffff, 0x00000004,
 523	0x91e4, 0xffffffff, 0x00070006,
 524	0x91e8, 0xffffffff, 0x00000001,
 525	0x91ec, 0xffffffff, 0x00050001,
 526	0x91f0, 0xffffffff, 0x00030002,
 527	0x9200, 0xffffffff, 0x00000004,
 528	0x9204, 0xffffffff, 0x00070006,
 529	0x9208, 0xffffffff, 0x00050001,
 530	0x920c, 0xffffffff, 0x00030002,
 531	0x921c, 0xffffffff, 0x00000004,
 532	0x9220, 0xffffffff, 0x00070006,
 533	0x9224, 0xffffffff, 0x00050001,
 534	0x9228, 0xffffffff, 0x00030002,
 535	0x9238, 0xffffffff, 0x00000004,
 536	0x923c, 0xffffffff, 0x00070006,
 537	0x9240, 0xffffffff, 0x00050001,
 538	0x9244, 0xffffffff, 0x00030002,
 539	0x9254, 0xffffffff, 0x00000004,
 540	0x9258, 0xffffffff, 0x00070006,
 541	0x9294, 0xffffffff, 0x00000001,
 542	0x929c, 0xffffffff, 0x00000002,
 543	0x92a0, 0xffffffff, 0x00040003,
 544	0x92a4, 0xffffffff, 0x00000005
 545};
 546
 547static const u32 rv740_golden_registers[] =
 548{
 549	0x88c4, 0xffffffff, 0x00000082,
 550	0x28a50, 0xfffffffc, 0x00000004,
 551	0x2650, 0x00040000, 0,
 552	0x20bc, 0x00040000, 0,
 553	0x733c, 0xffffffff, 0x00000002,
 554	0x7300, 0xffffffff, 0x001000f0,
 555	0x3f90, 0x00ff0000, 0,
 556	0x9148, 0x00ff0000, 0,
 557	0x3f94, 0x00ff0000, 0,
 558	0x914c, 0x00ff0000, 0,
 559	0x240c, 0xffffffff, 0x00000380,
 560	0x8a14, 0x00000007, 0x00000007,
 561	0x8b24, 0xffffffff, 0x00ff0fff,
 562	0x28a4c, 0xffffffff, 0x00004000,
 563	0xa180, 0xffffffff, 0x00003f3f,
 564	0x8d00, 0xffffffff, 0x0e0e003a,
 565	0x8d04, 0xffffffff, 0x013a0e2a,
 566	0x8c00, 0xffffffff, 0xe400000f,
 567	0x8db0, 0xffffffff, 0x98989898,
 568	0x8db4, 0xffffffff, 0x98989898,
 569	0x8db8, 0xffffffff, 0x98989898,
 570	0x8dbc, 0xffffffff, 0x98989898,
 571	0x8dc0, 0xffffffff, 0x98989898,
 572	0x8dc4, 0xffffffff, 0x98989898,
 573	0x8dc8, 0xffffffff, 0x98989898,
 574	0x8dcc, 0xffffffff, 0x98989898,
 575	0x9058, 0xffffffff, 0x0fffc40f,
 576	0x900c, 0xffffffff, 0x003b033f,
 577	0x28350, 0xffffffff, 0,
 578	0x8cf0, 0x1fffffff, 0x08e00420,
 579	0x9508, 0xffffffff, 0x00000002,
 580	0x88c4, 0xffffffff, 0x000000c2,
 581	0x9698, 0x18000000, 0x18000000
 582};
 583
 584static const u32 rv740_mgcg_init[] =
 585{
 586	0x8bcc, 0xffffffff, 0x13030100,
 587	0x5448, 0xffffffff, 0x100,
 588	0x55e4, 0xffffffff, 0x100,
 589	0x160c, 0xffffffff, 0x100,
 590	0x5644, 0xffffffff, 0x100,
 591	0xc164, 0xffffffff, 0x100,
 592	0x8a18, 0xffffffff, 0x100,
 593	0x897c, 0xffffffff, 0x100,
 594	0x8b28, 0xffffffff, 0x100,
 595	0x9144, 0xffffffff, 0x100,
 596	0x9a1c, 0xffffffff, 0x10000,
 597	0x9a50, 0xffffffff, 0x100,
 598	0x9a1c, 0xffffffff, 0x10001,
 599	0x9a50, 0xffffffff, 0x100,
 600	0x9a1c, 0xffffffff, 0x10002,
 601	0x9a50, 0xffffffff, 0x100,
 602	0x9a1c, 0xffffffff, 0x10003,
 603	0x9a50, 0xffffffff, 0x100,
 604	0x9a1c, 0xffffffff, 0x0,
 605	0x9870, 0xffffffff, 0x100,
 606	0x8d58, 0xffffffff, 0x100,
 607	0x9500, 0xffffffff, 0x0,
 608	0x9510, 0xffffffff, 0x100,
 609	0x9500, 0xffffffff, 0x1,
 610	0x9510, 0xffffffff, 0x100,
 611	0x9500, 0xffffffff, 0x2,
 612	0x9510, 0xffffffff, 0x100,
 613	0x9500, 0xffffffff, 0x3,
 614	0x9510, 0xffffffff, 0x100,
 615	0x9500, 0xffffffff, 0x4,
 616	0x9510, 0xffffffff, 0x100,
 617	0x9500, 0xffffffff, 0x5,
 618	0x9510, 0xffffffff, 0x100,
 619	0x9500, 0xffffffff, 0x6,
 620	0x9510, 0xffffffff, 0x100,
 621	0x9500, 0xffffffff, 0x7,
 622	0x9510, 0xffffffff, 0x100,
 623	0x9500, 0xffffffff, 0x8000,
 624	0x9490, 0xffffffff, 0x0,
 625	0x949c, 0xffffffff, 0x100,
 626	0x9490, 0xffffffff, 0x1,
 627	0x949c, 0xffffffff, 0x100,
 628	0x9490, 0xffffffff, 0x2,
 629	0x949c, 0xffffffff, 0x100,
 630	0x9490, 0xffffffff, 0x3,
 631	0x949c, 0xffffffff, 0x100,
 632	0x9490, 0xffffffff, 0x4,
 633	0x949c, 0xffffffff, 0x100,
 634	0x9490, 0xffffffff, 0x5,
 635	0x949c, 0xffffffff, 0x100,
 636	0x9490, 0xffffffff, 0x6,
 637	0x949c, 0xffffffff, 0x100,
 638	0x9490, 0xffffffff, 0x7,
 639	0x949c, 0xffffffff, 0x100,
 640	0x9490, 0xffffffff, 0x8000,
 641	0x9604, 0xffffffff, 0x0,
 642	0x9654, 0xffffffff, 0x100,
 643	0x9604, 0xffffffff, 0x1,
 644	0x9654, 0xffffffff, 0x100,
 645	0x9604, 0xffffffff, 0x2,
 646	0x9654, 0xffffffff, 0x100,
 647	0x9604, 0xffffffff, 0x3,
 648	0x9654, 0xffffffff, 0x100,
 649	0x9604, 0xffffffff, 0x4,
 650	0x9654, 0xffffffff, 0x100,
 651	0x9604, 0xffffffff, 0x5,
 652	0x9654, 0xffffffff, 0x100,
 653	0x9604, 0xffffffff, 0x6,
 654	0x9654, 0xffffffff, 0x100,
 655	0x9604, 0xffffffff, 0x7,
 656	0x9654, 0xffffffff, 0x100,
 657	0x9604, 0xffffffff, 0x80000000,
 658	0x9030, 0xffffffff, 0x100,
 659	0x9034, 0xffffffff, 0x100,
 660	0x9038, 0xffffffff, 0x100,
 661	0x903c, 0xffffffff, 0x100,
 662	0x9040, 0xffffffff, 0x100,
 663	0xa200, 0xffffffff, 0x100,
 664	0xa204, 0xffffffff, 0x100,
 665	0xa208, 0xffffffff, 0x100,
 666	0xa20c, 0xffffffff, 0x100,
 667	0x971c, 0xffffffff, 0x100,
 668	0x915c, 0xffffffff, 0x00020001,
 669	0x9160, 0xffffffff, 0x00040003,
 670	0x916c, 0xffffffff, 0x00060005,
 671	0x9170, 0xffffffff, 0x00080007,
 672	0x9174, 0xffffffff, 0x000a0009,
 673	0x9178, 0xffffffff, 0x000c000b,
 674	0x917c, 0xffffffff, 0x000e000d,
 675	0x9180, 0xffffffff, 0x0010000f,
 676	0x918c, 0xffffffff, 0x00120011,
 677	0x9190, 0xffffffff, 0x00140013,
 678	0x9194, 0xffffffff, 0x00020001,
 679	0x9198, 0xffffffff, 0x00040003,
 680	0x919c, 0xffffffff, 0x00060005,
 681	0x91a8, 0xffffffff, 0x00080007,
 682	0x91ac, 0xffffffff, 0x000a0009,
 683	0x91b0, 0xffffffff, 0x000c000b,
 684	0x91b4, 0xffffffff, 0x000e000d,
 685	0x91b8, 0xffffffff, 0x0010000f,
 686	0x91c4, 0xffffffff, 0x00120011,
 687	0x91c8, 0xffffffff, 0x00140013,
 688	0x91cc, 0xffffffff, 0x00020001,
 689	0x91d0, 0xffffffff, 0x00040003,
 690	0x91d4, 0xffffffff, 0x00060005,
 691	0x91e0, 0xffffffff, 0x00080007,
 692	0x91e4, 0xffffffff, 0x000a0009,
 693	0x91e8, 0xffffffff, 0x000c000b,
 694	0x91ec, 0xffffffff, 0x00020001,
 695	0x91f0, 0xffffffff, 0x00040003,
 696	0x91f4, 0xffffffff, 0x00060005,
 697	0x9200, 0xffffffff, 0x00080007,
 698	0x9204, 0xffffffff, 0x000a0009,
 699	0x9208, 0xffffffff, 0x000c000b,
 700	0x920c, 0xffffffff, 0x000e000d,
 701	0x9210, 0xffffffff, 0x0010000f,
 702	0x921c, 0xffffffff, 0x00120011,
 703	0x9220, 0xffffffff, 0x00140013,
 704	0x9224, 0xffffffff, 0x00020001,
 705	0x9228, 0xffffffff, 0x00040003,
 706	0x922c, 0xffffffff, 0x00060005,
 707	0x9238, 0xffffffff, 0x00080007,
 708	0x923c, 0xffffffff, 0x000a0009,
 709	0x9240, 0xffffffff, 0x000c000b,
 710	0x9244, 0xffffffff, 0x000e000d,
 711	0x9248, 0xffffffff, 0x0010000f,
 712	0x9254, 0xffffffff, 0x00120011,
 713	0x9258, 0xffffffff, 0x00140013,
 714	0x9294, 0xffffffff, 0x00020001,
 715	0x929c, 0xffffffff, 0x00040003,
 716	0x92a0, 0xffffffff, 0x00060005,
 717	0x92a4, 0xffffffff, 0x00080007
 718};
 719
 720static void rv770_init_golden_registers(struct radeon_device *rdev)
 721{
 722	switch (rdev->family) {
 723	case CHIP_RV770:
 724		radeon_program_register_sequence(rdev,
 725						 r7xx_golden_registers,
 726						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
 727		radeon_program_register_sequence(rdev,
 728						 r7xx_golden_dyn_gpr_registers,
 729						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 730		if (rdev->pdev->device == 0x994e)
 731			radeon_program_register_sequence(rdev,
 732							 rv770ce_golden_registers,
 733							 (const u32)ARRAY_SIZE(rv770ce_golden_registers));
 734		else
 735			radeon_program_register_sequence(rdev,
 736							 rv770_golden_registers,
 737							 (const u32)ARRAY_SIZE(rv770_golden_registers));
 738		radeon_program_register_sequence(rdev,
 739						 rv770_mgcg_init,
 740						 (const u32)ARRAY_SIZE(rv770_mgcg_init));
 741		break;
 742	case CHIP_RV730:
 743		radeon_program_register_sequence(rdev,
 744						 r7xx_golden_registers,
 745						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
 746		radeon_program_register_sequence(rdev,
 747						 r7xx_golden_dyn_gpr_registers,
 748						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 749		radeon_program_register_sequence(rdev,
 750						 rv730_golden_registers,
 751						 (const u32)ARRAY_SIZE(rv730_golden_registers));
 752		radeon_program_register_sequence(rdev,
 753						 rv730_mgcg_init,
 754						 (const u32)ARRAY_SIZE(rv730_mgcg_init));
 755		break;
 756	case CHIP_RV710:
 757		radeon_program_register_sequence(rdev,
 758						 r7xx_golden_registers,
 759						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
 760		radeon_program_register_sequence(rdev,
 761						 r7xx_golden_dyn_gpr_registers,
 762						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 763		radeon_program_register_sequence(rdev,
 764						 rv710_golden_registers,
 765						 (const u32)ARRAY_SIZE(rv710_golden_registers));
 766		radeon_program_register_sequence(rdev,
 767						 rv710_mgcg_init,
 768						 (const u32)ARRAY_SIZE(rv710_mgcg_init));
 769		break;
 770	case CHIP_RV740:
 771		radeon_program_register_sequence(rdev,
 772						 rv740_golden_registers,
 773						 (const u32)ARRAY_SIZE(rv740_golden_registers));
 774		radeon_program_register_sequence(rdev,
 775						 rv740_mgcg_init,
 776						 (const u32)ARRAY_SIZE(rv740_mgcg_init));
 777		break;
 778	default:
 779		break;
 780	}
 781}
 782
 783#define PCIE_BUS_CLK                10000
 784#define TCLK                        (PCIE_BUS_CLK / 10)
 785
 786/**
 787 * rv770_get_xclk - get the xclk
 788 *
 789 * @rdev: radeon_device pointer
 790 *
 791 * Returns the reference clock used by the gfx engine
 792 * (r7xx-cayman).
 793 */
 794u32 rv770_get_xclk(struct radeon_device *rdev)
 795{
 796	u32 reference_clock = rdev->clock.spll.reference_freq;
 797	u32 tmp = RREG32(CG_CLKPIN_CNTL);
 798
 799	if (tmp & MUX_TCLK_TO_XCLK)
 800		return TCLK;
 801
 802	if (tmp & XTALIN_DIVIDE)
 803		return reference_clock / 4;
 804
 805	return reference_clock;
 806}
 807
 808void rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, bool async)
 809{
 810	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 811	u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
 812	int i;
 813
 814	/* Lock the graphics update lock */
 815	tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
 816	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 817
 818	/* update the scanout addresses */
 819	WREG32(AVIVO_D1GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
 820	       async ? AVIVO_D1GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
 821	if (radeon_crtc->crtc_id) {
 822		WREG32(D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 823		WREG32(D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 824	} else {
 825		WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 826		WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 827	}
 828	WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 829	       (u32)crtc_base);
 830	WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 831	       (u32)crtc_base);
 832
 833	/* Wait for update_pending to go high. */
 834	for (i = 0; i < rdev->usec_timeout; i++) {
 835		if (RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING)
 836			break;
 837		udelay(1);
 838	}
 839	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
 840
 841	/* Unlock the lock, so double-buffering can take place inside vblank */
 842	tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
 843	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 844}
 845
 846bool rv770_page_flip_pending(struct radeon_device *rdev, int crtc_id)
 847{
 848	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 849
 850	/* Return current update_pending status: */
 851	return !!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) &
 852		AVIVO_D1GRPH_SURFACE_UPDATE_PENDING);
 853}
 854
 855/* get temperature in millidegrees */
 856int rv770_get_temp(struct radeon_device *rdev)
 857{
 858	u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
 859		ASIC_T_SHIFT;
 860	int actual_temp;
 861
 862	if (temp & 0x400)
 863		actual_temp = -256;
 864	else if (temp & 0x200)
 865		actual_temp = 255;
 866	else if (temp & 0x100) {
 867		actual_temp = temp & 0x1ff;
 868		actual_temp |= ~0x1ff;
 869	} else
 870		actual_temp = temp & 0xff;
 871
 872	return (actual_temp * 1000) / 2;
 873}
 874
 875void rv770_pm_misc(struct radeon_device *rdev)
 876{
 877	int req_ps_idx = rdev->pm.requested_power_state_index;
 878	int req_cm_idx = rdev->pm.requested_clock_mode_index;
 879	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
 880	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 881
 882	if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
 883		/* 0xff01 is a flag rather then an actual voltage */
 884		if (voltage->voltage == 0xff01)
 885			return;
 886		if (voltage->voltage != rdev->pm.current_vddc) {
 887			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
 888			rdev->pm.current_vddc = voltage->voltage;
 889			DRM_DEBUG("Setting: v: %d\n", voltage->voltage);
 890		}
 891	}
 892}
 893
 894/*
 895 * GART
 896 */
 897static int rv770_pcie_gart_enable(struct radeon_device *rdev)
 898{
 899	u32 tmp;
 900	int r, i;
 901
 902	if (rdev->gart.robj == NULL) {
 903		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 904		return -EINVAL;
 905	}
 906	r = radeon_gart_table_vram_pin(rdev);
 907	if (r)
 908		return r;
 909	/* Setup L2 cache */
 910	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 911				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 912				EFFECTIVE_L2_QUEUE_SIZE(7));
 913	WREG32(VM_L2_CNTL2, 0);
 914	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 915	/* Setup TLB control */
 916	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 917		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 918		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 919		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 920	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 921	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 922	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 923	if (rdev->family == CHIP_RV740)
 924		WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
 925	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 926	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 927	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 928	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 929	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 930	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 931	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 932	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 933				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 934	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 935			(u32)(rdev->dummy_page.addr >> 12));
 936	for (i = 1; i < 7; i++)
 937		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 938
 939	r600_pcie_gart_tlb_flush(rdev);
 940	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
 941		 (unsigned)(rdev->mc.gtt_size >> 20),
 942		 (unsigned long long)rdev->gart.table_addr);
 943	rdev->gart.ready = true;
 944	return 0;
 945}
 946
 947static void rv770_pcie_gart_disable(struct radeon_device *rdev)
 948{
 949	u32 tmp;
 950	int i;
 951
 952	/* Disable all tables */
 953	for (i = 0; i < 7; i++)
 954		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 955
 956	/* Setup L2 cache */
 957	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 958				EFFECTIVE_L2_QUEUE_SIZE(7));
 959	WREG32(VM_L2_CNTL2, 0);
 960	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 961	/* Setup TLB control */
 962	tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 963	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 964	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 965	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 966	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 967	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 968	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 969	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 970	radeon_gart_table_vram_unpin(rdev);
 971}
 972
 973static void rv770_pcie_gart_fini(struct radeon_device *rdev)
 974{
 975	radeon_gart_fini(rdev);
 976	rv770_pcie_gart_disable(rdev);
 977	radeon_gart_table_vram_free(rdev);
 978}
 979
 980
 981static void rv770_agp_enable(struct radeon_device *rdev)
 982{
 983	u32 tmp;
 984	int i;
 985
 986	/* Setup L2 cache */
 987	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 988				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 989				EFFECTIVE_L2_QUEUE_SIZE(7));
 990	WREG32(VM_L2_CNTL2, 0);
 991	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 992	/* Setup TLB control */
 993	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 994		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 995		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 996		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 997	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 998	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 999	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1000	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1001	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1002	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1003	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1004	for (i = 0; i < 7; i++)
1005		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1006}
1007
1008static void rv770_mc_program(struct radeon_device *rdev)
1009{
1010	struct rv515_mc_save save;
1011	u32 tmp;
1012	int i, j;
1013
1014	/* Initialize HDP */
1015	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1016		WREG32((0x2c14 + j), 0x00000000);
1017		WREG32((0x2c18 + j), 0x00000000);
1018		WREG32((0x2c1c + j), 0x00000000);
1019		WREG32((0x2c20 + j), 0x00000000);
1020		WREG32((0x2c24 + j), 0x00000000);
1021	}
1022	/* r7xx hw bug.  Read from HDP_DEBUG1 rather
1023	 * than writing to HDP_REG_COHERENCY_FLUSH_CNTL
1024	 */
1025	tmp = RREG32(HDP_DEBUG1);
1026
1027	rv515_mc_stop(rdev, &save);
1028	if (r600_mc_wait_for_idle(rdev)) {
1029		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1030	}
1031	/* Lockout access through VGA aperture*/
1032	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1033	/* Update configuration */
1034	if (rdev->flags & RADEON_IS_AGP) {
1035		if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1036			/* VRAM before AGP */
1037			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1038				rdev->mc.vram_start >> 12);
1039			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1040				rdev->mc.gtt_end >> 12);
1041		} else {
1042			/* VRAM after AGP */
1043			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1044				rdev->mc.gtt_start >> 12);
1045			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1046				rdev->mc.vram_end >> 12);
1047		}
1048	} else {
1049		WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1050			rdev->mc.vram_start >> 12);
1051		WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1052			rdev->mc.vram_end >> 12);
1053	}
1054	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1055	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1056	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1057	WREG32(MC_VM_FB_LOCATION, tmp);
1058	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1059	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1060	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1061	if (rdev->flags & RADEON_IS_AGP) {
1062		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1063		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1064		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1065	} else {
1066		WREG32(MC_VM_AGP_BASE, 0);
1067		WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1068		WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1069	}
1070	if (r600_mc_wait_for_idle(rdev)) {
1071		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1072	}
1073	rv515_mc_resume(rdev, &save);
1074	/* we need to own VRAM, so turn off the VGA renderer here
1075	 * to stop it overwriting our objects */
1076	rv515_vga_render_disable(rdev);
1077}
1078
1079
1080/*
1081 * CP.
1082 */
1083void r700_cp_stop(struct radeon_device *rdev)
1084{
1085	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1086		radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1087	WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1088	WREG32(SCRATCH_UMSK, 0);
1089	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1090}
1091
1092static int rv770_cp_load_microcode(struct radeon_device *rdev)
1093{
1094	const __be32 *fw_data;
1095	int i;
1096
1097	if (!rdev->me_fw || !rdev->pfp_fw)
1098		return -EINVAL;
1099
1100	r700_cp_stop(rdev);
1101	WREG32(CP_RB_CNTL,
1102#ifdef __BIG_ENDIAN
1103	       BUF_SWAP_32BIT |
1104#endif
1105	       RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1106
1107	/* Reset cp */
1108	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1109	RREG32(GRBM_SOFT_RESET);
1110	mdelay(15);
1111	WREG32(GRBM_SOFT_RESET, 0);
1112
1113	fw_data = (const __be32 *)rdev->pfp_fw->data;
1114	WREG32(CP_PFP_UCODE_ADDR, 0);
1115	for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
1116		WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1117	WREG32(CP_PFP_UCODE_ADDR, 0);
1118
1119	fw_data = (const __be32 *)rdev->me_fw->data;
1120	WREG32(CP_ME_RAM_WADDR, 0);
1121	for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
1122		WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1123
1124	WREG32(CP_PFP_UCODE_ADDR, 0);
1125	WREG32(CP_ME_RAM_WADDR, 0);
1126	WREG32(CP_ME_RAM_RADDR, 0);
1127	return 0;
1128}
1129
1130void r700_cp_fini(struct radeon_device *rdev)
1131{
1132	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1133	r700_cp_stop(rdev);
1134	radeon_ring_fini(rdev, ring);
1135	radeon_scratch_free(rdev, ring->rptr_save_reg);
1136}
1137
1138void rv770_set_clk_bypass_mode(struct radeon_device *rdev)
1139{
1140	u32 tmp, i;
1141
1142	if (rdev->flags & RADEON_IS_IGP)
1143		return;
1144
1145	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
1146	tmp &= SCLK_MUX_SEL_MASK;
1147	tmp |= SCLK_MUX_SEL(1) | SCLK_MUX_UPDATE;
1148	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1149
1150	for (i = 0; i < rdev->usec_timeout; i++) {
1151		if (RREG32(CG_SPLL_STATUS) & SPLL_CHG_STATUS)
1152			break;
1153		udelay(1);
1154	}
1155
1156	tmp &= ~SCLK_MUX_UPDATE;
1157	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1158
1159	tmp = RREG32(MPLL_CNTL_MODE);
1160	if ((rdev->family == CHIP_RV710) || (rdev->family == CHIP_RV730))
1161		tmp &= ~RV730_MPLL_MCLK_SEL;
1162	else
1163		tmp &= ~MPLL_MCLK_SEL;
1164	WREG32(MPLL_CNTL_MODE, tmp);
1165}
1166
1167/*
1168 * Core functions
1169 */
1170static void rv770_gpu_init(struct radeon_device *rdev)
1171{
1172	int i, j, num_qd_pipes;
1173	u32 ta_aux_cntl;
1174	u32 sx_debug_1;
1175	u32 smx_dc_ctl0;
1176	u32 db_debug3;
1177	u32 num_gs_verts_per_thread;
1178	u32 vgt_gs_per_es;
1179	u32 gs_prim_buffer_depth = 0;
1180	u32 sq_ms_fifo_sizes;
1181	u32 sq_config;
1182	u32 sq_thread_resource_mgmt;
1183	u32 hdp_host_path_cntl;
1184	u32 sq_dyn_gpr_size_simd_ab_0;
1185	u32 gb_tiling_config = 0;
1186	u32 cc_gc_shader_pipe_config = 0;
1187	u32 mc_arb_ramcfg;
1188	u32 db_debug4, tmp;
1189	u32 inactive_pipes, shader_pipe_config;
1190	u32 disabled_rb_mask;
1191	unsigned active_number;
1192
1193	/* setup chip specs */
1194	rdev->config.rv770.tiling_group_size = 256;
1195	switch (rdev->family) {
1196	case CHIP_RV770:
1197		rdev->config.rv770.max_pipes = 4;
1198		rdev->config.rv770.max_tile_pipes = 8;
1199		rdev->config.rv770.max_simds = 10;
1200		rdev->config.rv770.max_backends = 4;
1201		rdev->config.rv770.max_gprs = 256;
1202		rdev->config.rv770.max_threads = 248;
1203		rdev->config.rv770.max_stack_entries = 512;
1204		rdev->config.rv770.max_hw_contexts = 8;
1205		rdev->config.rv770.max_gs_threads = 16 * 2;
1206		rdev->config.rv770.sx_max_export_size = 128;
1207		rdev->config.rv770.sx_max_export_pos_size = 16;
1208		rdev->config.rv770.sx_max_export_smx_size = 112;
1209		rdev->config.rv770.sq_num_cf_insts = 2;
1210
1211		rdev->config.rv770.sx_num_of_sets = 7;
1212		rdev->config.rv770.sc_prim_fifo_size = 0xF9;
1213		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1214		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1215		break;
1216	case CHIP_RV730:
1217		rdev->config.rv770.max_pipes = 2;
1218		rdev->config.rv770.max_tile_pipes = 4;
1219		rdev->config.rv770.max_simds = 8;
1220		rdev->config.rv770.max_backends = 2;
1221		rdev->config.rv770.max_gprs = 128;
1222		rdev->config.rv770.max_threads = 248;
1223		rdev->config.rv770.max_stack_entries = 256;
1224		rdev->config.rv770.max_hw_contexts = 8;
1225		rdev->config.rv770.max_gs_threads = 16 * 2;
1226		rdev->config.rv770.sx_max_export_size = 256;
1227		rdev->config.rv770.sx_max_export_pos_size = 32;
1228		rdev->config.rv770.sx_max_export_smx_size = 224;
1229		rdev->config.rv770.sq_num_cf_insts = 2;
1230
1231		rdev->config.rv770.sx_num_of_sets = 7;
1232		rdev->config.rv770.sc_prim_fifo_size = 0xf9;
1233		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1234		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1235		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1236			rdev->config.rv770.sx_max_export_pos_size -= 16;
1237			rdev->config.rv770.sx_max_export_smx_size += 16;
1238		}
1239		break;
1240	case CHIP_RV710:
1241		rdev->config.rv770.max_pipes = 2;
1242		rdev->config.rv770.max_tile_pipes = 2;
1243		rdev->config.rv770.max_simds = 2;
1244		rdev->config.rv770.max_backends = 1;
1245		rdev->config.rv770.max_gprs = 256;
1246		rdev->config.rv770.max_threads = 192;
1247		rdev->config.rv770.max_stack_entries = 256;
1248		rdev->config.rv770.max_hw_contexts = 4;
1249		rdev->config.rv770.max_gs_threads = 8 * 2;
1250		rdev->config.rv770.sx_max_export_size = 128;
1251		rdev->config.rv770.sx_max_export_pos_size = 16;
1252		rdev->config.rv770.sx_max_export_smx_size = 112;
1253		rdev->config.rv770.sq_num_cf_insts = 1;
1254
1255		rdev->config.rv770.sx_num_of_sets = 7;
1256		rdev->config.rv770.sc_prim_fifo_size = 0x40;
1257		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1258		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1259		break;
1260	case CHIP_RV740:
1261		rdev->config.rv770.max_pipes = 4;
1262		rdev->config.rv770.max_tile_pipes = 4;
1263		rdev->config.rv770.max_simds = 8;
1264		rdev->config.rv770.max_backends = 4;
1265		rdev->config.rv770.max_gprs = 256;
1266		rdev->config.rv770.max_threads = 248;
1267		rdev->config.rv770.max_stack_entries = 512;
1268		rdev->config.rv770.max_hw_contexts = 8;
1269		rdev->config.rv770.max_gs_threads = 16 * 2;
1270		rdev->config.rv770.sx_max_export_size = 256;
1271		rdev->config.rv770.sx_max_export_pos_size = 32;
1272		rdev->config.rv770.sx_max_export_smx_size = 224;
1273		rdev->config.rv770.sq_num_cf_insts = 2;
1274
1275		rdev->config.rv770.sx_num_of_sets = 7;
1276		rdev->config.rv770.sc_prim_fifo_size = 0x100;
1277		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1278		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1279
1280		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1281			rdev->config.rv770.sx_max_export_pos_size -= 16;
1282			rdev->config.rv770.sx_max_export_smx_size += 16;
1283		}
1284		break;
1285	default:
1286		break;
1287	}
1288
1289	/* Initialize HDP */
1290	j = 0;
1291	for (i = 0; i < 32; i++) {
1292		WREG32((0x2c14 + j), 0x00000000);
1293		WREG32((0x2c18 + j), 0x00000000);
1294		WREG32((0x2c1c + j), 0x00000000);
1295		WREG32((0x2c20 + j), 0x00000000);
1296		WREG32((0x2c24 + j), 0x00000000);
1297		j += 0x18;
1298	}
1299
1300	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1301
1302	/* setup tiling, simd, pipe config */
1303	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1304
1305	shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG);
1306	inactive_pipes = (shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> INACTIVE_QD_PIPES_SHIFT;
1307	for (i = 0, tmp = 1, active_number = 0; i < R7XX_MAX_PIPES; i++) {
1308		if (!(inactive_pipes & tmp)) {
1309			active_number++;
1310		}
1311		tmp <<= 1;
1312	}
1313	if (active_number == 1) {
1314		WREG32(SPI_CONFIG_CNTL, DISABLE_INTERP_1);
1315	} else {
1316		WREG32(SPI_CONFIG_CNTL, 0);
1317	}
1318
1319	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1320	tmp = rdev->config.rv770.max_simds -
1321		r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R7XX_MAX_SIMDS_MASK);
1322	rdev->config.rv770.active_simds = tmp;
1323
1324	switch (rdev->config.rv770.max_tile_pipes) {
1325	case 1:
1326	default:
1327		gb_tiling_config = PIPE_TILING(0);
1328		break;
1329	case 2:
1330		gb_tiling_config = PIPE_TILING(1);
1331		break;
1332	case 4:
1333		gb_tiling_config = PIPE_TILING(2);
1334		break;
1335	case 8:
1336		gb_tiling_config = PIPE_TILING(3);
1337		break;
1338	}
1339	rdev->config.rv770.tiling_npipes = rdev->config.rv770.max_tile_pipes;
1340
1341	disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R7XX_MAX_BACKENDS_MASK;
1342	tmp = 0;
1343	for (i = 0; i < rdev->config.rv770.max_backends; i++)
1344		tmp |= (1 << i);
1345	/* if all the backends are disabled, fix it up here */
1346	if ((disabled_rb_mask & tmp) == tmp) {
1347		for (i = 0; i < rdev->config.rv770.max_backends; i++)
1348			disabled_rb_mask &= ~(1 << i);
1349	}
1350	tmp = (gb_tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1351	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.rv770.max_backends,
1352					R7XX_MAX_BACKENDS, disabled_rb_mask);
1353	gb_tiling_config |= tmp << 16;
1354	rdev->config.rv770.backend_map = tmp;
1355
1356	if (rdev->family == CHIP_RV770)
1357		gb_tiling_config |= BANK_TILING(1);
1358	else {
1359		if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
1360			gb_tiling_config |= BANK_TILING(1);
1361		else
1362			gb_tiling_config |= BANK_TILING(0);
1363	}
1364	rdev->config.rv770.tiling_nbanks = 4 << ((gb_tiling_config >> 4) & 0x3);
1365	gb_tiling_config |= GROUP_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1366	if (((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT) > 3) {
1367		gb_tiling_config |= ROW_TILING(3);
1368		gb_tiling_config |= SAMPLE_SPLIT(3);
1369	} else {
1370		gb_tiling_config |=
1371			ROW_TILING(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1372		gb_tiling_config |=
1373			SAMPLE_SPLIT(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1374	}
1375
1376	gb_tiling_config |= BANK_SWAPS(1);
1377	rdev->config.rv770.tile_config = gb_tiling_config;
1378
1379	WREG32(GB_TILING_CONFIG, gb_tiling_config);
1380	WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1381	WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1382	WREG32(DMA_TILING_CONFIG, (gb_tiling_config & 0xffff));
1383	WREG32(DMA_TILING_CONFIG2, (gb_tiling_config & 0xffff));
1384	if (rdev->family == CHIP_RV730) {
1385		WREG32(UVD_UDEC_DB_TILING_CONFIG, (gb_tiling_config & 0xffff));
1386		WREG32(UVD_UDEC_DBW_TILING_CONFIG, (gb_tiling_config & 0xffff));
1387		WREG32(UVD_UDEC_TILING_CONFIG, (gb_tiling_config & 0xffff));
1388	}
1389
1390	WREG32(CGTS_SYS_TCC_DISABLE, 0);
1391	WREG32(CGTS_TCC_DISABLE, 0);
1392	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
1393	WREG32(CGTS_USER_TCC_DISABLE, 0);
1394
1395
1396	num_qd_pipes = R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1397	WREG32(VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & DEALLOC_DIST_MASK);
1398	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1399
1400	/* set HW defaults for 3D engine */
1401	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
1402				     ROQ_IB2_START(0x2b)));
1403
1404	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
1405
1406	ta_aux_cntl = RREG32(TA_CNTL_AUX);
1407	WREG32(TA_CNTL_AUX, ta_aux_cntl | DISABLE_CUBE_ANISO);
1408
1409	sx_debug_1 = RREG32(SX_DEBUG_1);
1410	sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1411	WREG32(SX_DEBUG_1, sx_debug_1);
1412
1413	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1414	smx_dc_ctl0 &= ~CACHE_DEPTH(0x1ff);
1415	smx_dc_ctl0 |= CACHE_DEPTH((rdev->config.rv770.sx_num_of_sets * 64) - 1);
1416	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1417
1418	if (rdev->family != CHIP_RV740)
1419		WREG32(SMX_EVENT_CTL, (ES_FLUSH_CTL(4) |
1420				       GS_FLUSH_CTL(4) |
1421				       ACK_FLUSH_CTL(3) |
1422				       SYNC_FLUSH_CTL));
1423
1424	if (rdev->family != CHIP_RV770)
1425		WREG32(SMX_SAR_CTL0, 0x00003f3f);
1426
1427	db_debug3 = RREG32(DB_DEBUG3);
1428	db_debug3 &= ~DB_CLK_OFF_DELAY(0x1f);
1429	switch (rdev->family) {
1430	case CHIP_RV770:
1431	case CHIP_RV740:
1432		db_debug3 |= DB_CLK_OFF_DELAY(0x1f);
1433		break;
1434	case CHIP_RV710:
1435	case CHIP_RV730:
1436	default:
1437		db_debug3 |= DB_CLK_OFF_DELAY(2);
1438		break;
1439	}
1440	WREG32(DB_DEBUG3, db_debug3);
1441
1442	if (rdev->family != CHIP_RV770) {
1443		db_debug4 = RREG32(DB_DEBUG4);
1444		db_debug4 |= DISABLE_TILE_COVERED_FOR_PS_ITER;
1445		WREG32(DB_DEBUG4, db_debug4);
1446	}
1447
1448	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.rv770.sx_max_export_size / 4) - 1) |
1449					POSITION_BUFFER_SIZE((rdev->config.rv770.sx_max_export_pos_size / 4) - 1) |
1450					SMX_BUFFER_SIZE((rdev->config.rv770.sx_max_export_smx_size / 4) - 1)));
1451
1452	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.rv770.sc_prim_fifo_size) |
1453				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_hiz_tile_fifo_size) |
1454				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_earlyz_tile_fifo_fize)));
1455
1456	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1457
1458	WREG32(VGT_NUM_INSTANCES, 1);
1459
1460	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
1461
1462	WREG32(CP_PERFMON_CNTL, 0);
1463
1464	sq_ms_fifo_sizes = (CACHE_FIFO_SIZE(16 * rdev->config.rv770.sq_num_cf_insts) |
1465			    DONE_FIFO_HIWATER(0xe0) |
1466			    ALU_UPDATE_FIFO_HIWATER(0x8));
1467	switch (rdev->family) {
1468	case CHIP_RV770:
1469	case CHIP_RV730:
1470	case CHIP_RV710:
1471		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x1);
1472		break;
1473	case CHIP_RV740:
1474	default:
1475		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x4);
1476		break;
1477	}
1478	WREG32(SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
1479
1480	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1481	 * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1482	 */
1483	sq_config = RREG32(SQ_CONFIG);
1484	sq_config &= ~(PS_PRIO(3) |
1485		       VS_PRIO(3) |
1486		       GS_PRIO(3) |
1487		       ES_PRIO(3));
1488	sq_config |= (DX9_CONSTS |
1489		      VC_ENABLE |
1490		      EXPORT_SRC_C |
1491		      PS_PRIO(0) |
1492		      VS_PRIO(1) |
1493		      GS_PRIO(2) |
1494		      ES_PRIO(3));
1495	if (rdev->family == CHIP_RV710)
1496		/* no vertex cache */
1497		sq_config &= ~VC_ENABLE;
1498
1499	WREG32(SQ_CONFIG, sq_config);
1500
1501	WREG32(SQ_GPR_RESOURCE_MGMT_1,  (NUM_PS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1502					 NUM_VS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1503					 NUM_CLAUSE_TEMP_GPRS(((rdev->config.rv770.max_gprs * 24)/64)/2)));
1504
1505	WREG32(SQ_GPR_RESOURCE_MGMT_2,  (NUM_GS_GPRS((rdev->config.rv770.max_gprs * 7)/64) |
1506					 NUM_ES_GPRS((rdev->config.rv770.max_gprs * 7)/64)));
1507
1508	sq_thread_resource_mgmt = (NUM_PS_THREADS((rdev->config.rv770.max_threads * 4)/8) |
1509				   NUM_VS_THREADS((rdev->config.rv770.max_threads * 2)/8) |
1510				   NUM_ES_THREADS((rdev->config.rv770.max_threads * 1)/8));
1511	if (((rdev->config.rv770.max_threads * 1) / 8) > rdev->config.rv770.max_gs_threads)
1512		sq_thread_resource_mgmt |= NUM_GS_THREADS(rdev->config.rv770.max_gs_threads);
1513	else
1514		sq_thread_resource_mgmt |= NUM_GS_THREADS((rdev->config.rv770.max_gs_threads * 1)/8);
1515	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1516
1517	WREG32(SQ_STACK_RESOURCE_MGMT_1, (NUM_PS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1518						     NUM_VS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1519
1520	WREG32(SQ_STACK_RESOURCE_MGMT_2, (NUM_GS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1521						     NUM_ES_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1522
1523	sq_dyn_gpr_size_simd_ab_0 = (SIMDA_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1524				     SIMDA_RING1((rdev->config.rv770.max_gprs * 38)/64) |
1525				     SIMDB_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1526				     SIMDB_RING1((rdev->config.rv770.max_gprs * 38)/64));
1527
1528	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
1529	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
1530	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
1531	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
1532	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
1533	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
1534	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
1535	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
1536
1537	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1538					  FORCE_EOV_MAX_REZ_CNT(255)));
1539
1540	if (rdev->family == CHIP_RV710)
1541		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(TC_ONLY) |
1542						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1543	else
1544		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(VC_AND_TC) |
1545						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1546
1547	switch (rdev->family) {
1548	case CHIP_RV770:
1549	case CHIP_RV730:
1550	case CHIP_RV740:
1551		gs_prim_buffer_depth = 384;
1552		break;
1553	case CHIP_RV710:
1554		gs_prim_buffer_depth = 128;
1555		break;
1556	default:
1557		break;
1558	}
1559
1560	num_gs_verts_per_thread = rdev->config.rv770.max_pipes * 16;
1561	vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1562	/* Max value for this is 256 */
1563	if (vgt_gs_per_es > 256)
1564		vgt_gs_per_es = 256;
1565
1566	WREG32(VGT_ES_PER_GS, 128);
1567	WREG32(VGT_GS_PER_ES, vgt_gs_per_es);
1568	WREG32(VGT_GS_PER_VS, 2);
1569
1570	/* more default values. 2D/3D driver should adjust as needed */
1571	WREG32(VGT_GS_VERTEX_REUSE, 16);
1572	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1573	WREG32(VGT_STRMOUT_EN, 0);
1574	WREG32(SX_MISC, 0);
1575	WREG32(PA_SC_MODE_CNTL, 0);
1576	WREG32(PA_SC_EDGERULE, 0xaaaaaaaa);
1577	WREG32(PA_SC_AA_CONFIG, 0);
1578	WREG32(PA_SC_CLIPRECT_RULE, 0xffff);
1579	WREG32(PA_SC_LINE_STIPPLE, 0);
1580	WREG32(SPI_INPUT_Z, 0);
1581	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1582	WREG32(CB_COLOR7_FRAG, 0);
1583
1584	/* clear render buffer base addresses */
1585	WREG32(CB_COLOR0_BASE, 0);
1586	WREG32(CB_COLOR1_BASE, 0);
1587	WREG32(CB_COLOR2_BASE, 0);
1588	WREG32(CB_COLOR3_BASE, 0);
1589	WREG32(CB_COLOR4_BASE, 0);
1590	WREG32(CB_COLOR5_BASE, 0);
1591	WREG32(CB_COLOR6_BASE, 0);
1592	WREG32(CB_COLOR7_BASE, 0);
1593
1594	WREG32(TCP_CNTL, 0);
1595
1596	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1597	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1598
1599	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1600
1601	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1602					  NUM_CLIP_SEQ(3)));
1603	WREG32(VC_ENHANCE, 0);
1604}
1605
1606void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1607{
1608	u64 size_bf, size_af;
1609
1610	if (mc->mc_vram_size > 0xE0000000) {
1611		/* leave room for at least 512M GTT */
1612		dev_warn(rdev->dev, "limiting VRAM\n");
1613		mc->real_vram_size = 0xE0000000;
1614		mc->mc_vram_size = 0xE0000000;
1615	}
1616	if (rdev->flags & RADEON_IS_AGP) {
1617		size_bf = mc->gtt_start;
1618		size_af = mc->mc_mask - mc->gtt_end;
1619		if (size_bf > size_af) {
1620			if (mc->mc_vram_size > size_bf) {
1621				dev_warn(rdev->dev, "limiting VRAM\n");
1622				mc->real_vram_size = size_bf;
1623				mc->mc_vram_size = size_bf;
1624			}
1625			mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1626		} else {
1627			if (mc->mc_vram_size > size_af) {
1628				dev_warn(rdev->dev, "limiting VRAM\n");
1629				mc->real_vram_size = size_af;
1630				mc->mc_vram_size = size_af;
1631			}
1632			mc->vram_start = mc->gtt_end + 1;
1633		}
1634		mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1635		dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1636				mc->mc_vram_size >> 20, mc->vram_start,
1637				mc->vram_end, mc->real_vram_size >> 20);
1638	} else {
1639		radeon_vram_location(rdev, &rdev->mc, 0);
1640		rdev->mc.gtt_base_align = 0;
1641		radeon_gtt_location(rdev, mc);
1642	}
1643}
1644
1645static int rv770_mc_init(struct radeon_device *rdev)
1646{
1647	u32 tmp;
1648	int chansize, numchan;
1649
1650	/* Get VRAM informations */
1651	rdev->mc.vram_is_ddr = true;
1652	tmp = RREG32(MC_ARB_RAMCFG);
1653	if (tmp & CHANSIZE_OVERRIDE) {
1654		chansize = 16;
1655	} else if (tmp & CHANSIZE_MASK) {
1656		chansize = 64;
1657	} else {
1658		chansize = 32;
1659	}
1660	tmp = RREG32(MC_SHARED_CHMAP);
1661	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1662	case 0:
1663	default:
1664		numchan = 1;
1665		break;
1666	case 1:
1667		numchan = 2;
1668		break;
1669	case 2:
1670		numchan = 4;
1671		break;
1672	case 3:
1673		numchan = 8;
1674		break;
1675	}
1676	rdev->mc.vram_width = numchan * chansize;
1677	/* Could aper size report 0 ? */
1678	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1679	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1680	/* Setup GPU memory space */
1681	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1682	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1683	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1684	r700_vram_gtt_location(rdev, &rdev->mc);
1685	radeon_update_bandwidth_info(rdev);
1686
1687	return 0;
1688}
1689
1690static void rv770_uvd_init(struct radeon_device *rdev)
1691{
1692	int r;
1693
1694	if (!rdev->has_uvd)
1695		return;
1696
1697	r = radeon_uvd_init(rdev);
1698	if (r) {
1699		dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
1700		/*
1701		 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
1702		 * to early fails uvd_v2_2_resume() and thus nothing happens
1703		 * there. So it is pointless to try to go through that code
1704		 * hence why we disable uvd here.
1705		 */
1706		rdev->has_uvd = 0;
1707		return;
1708	}
1709	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
1710	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
1711}
1712
1713static void rv770_uvd_start(struct radeon_device *rdev)
1714{
1715	int r;
1716
1717	if (!rdev->has_uvd)
1718		return;
1719
1720	r = uvd_v2_2_resume(rdev);
1721	if (r) {
1722		dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
1723		goto error;
1724	}
1725	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
1726	if (r) {
1727		dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
1728		goto error;
1729	}
1730	return;
1731
1732error:
1733	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
1734}
1735
1736static void rv770_uvd_resume(struct radeon_device *rdev)
1737{
1738	struct radeon_ring *ring;
1739	int r;
1740
1741	if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
1742		return;
1743
1744	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
1745	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
1746	if (r) {
1747		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
1748		return;
1749	}
1750	r = uvd_v1_0_init(rdev);
1751	if (r) {
1752		dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
1753		return;
1754	}
1755}
1756
1757static int rv770_startup(struct radeon_device *rdev)
1758{
1759	struct radeon_ring *ring;
1760	int r;
1761
1762	/* enable pcie gen2 link */
1763	rv770_pcie_gen2_enable(rdev);
1764
1765	/* scratch needs to be initialized before MC */
1766	r = r600_vram_scratch_init(rdev);
1767	if (r)
1768		return r;
1769
1770	rv770_mc_program(rdev);
1771
1772	if (rdev->flags & RADEON_IS_AGP) {
1773		rv770_agp_enable(rdev);
1774	} else {
1775		r = rv770_pcie_gart_enable(rdev);
1776		if (r)
1777			return r;
1778	}
1779
1780	rv770_gpu_init(rdev);
1781
1782	/* allocate wb buffer */
1783	r = radeon_wb_init(rdev);
1784	if (r)
1785		return r;
1786
1787	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1788	if (r) {
1789		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1790		return r;
1791	}
1792
1793	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
1794	if (r) {
1795		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1796		return r;
1797	}
1798
1799	rv770_uvd_start(rdev);
1800
1801	/* Enable IRQ */
1802	if (!rdev->irq.installed) {
1803		r = radeon_irq_kms_init(rdev);
1804		if (r)
1805			return r;
1806	}
1807
1808	r = r600_irq_init(rdev);
1809	if (r) {
1810		DRM_ERROR("radeon: IH init failed (%d).\n", r);
1811		radeon_irq_kms_fini(rdev);
1812		return r;
1813	}
1814	r600_irq_set(rdev);
1815
1816	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1817	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1818			     RADEON_CP_PACKET2);
1819	if (r)
1820		return r;
1821
1822	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1823	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
1824			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1825	if (r)
1826		return r;
1827
1828	r = rv770_cp_load_microcode(rdev);
1829	if (r)
1830		return r;
1831	r = r600_cp_resume(rdev);
1832	if (r)
1833		return r;
1834
1835	r = r600_dma_resume(rdev);
1836	if (r)
1837		return r;
1838
1839	rv770_uvd_resume(rdev);
1840
1841	r = radeon_ib_pool_init(rdev);
1842	if (r) {
1843		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1844		return r;
1845	}
1846
1847	r = radeon_audio_init(rdev);
1848	if (r) {
1849		DRM_ERROR("radeon: audio init failed\n");
1850		return r;
1851	}
1852
1853	return 0;
1854}
1855
1856int rv770_resume(struct radeon_device *rdev)
1857{
1858	int r;
1859
1860	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
1861	 * posting will perform necessary task to bring back GPU into good
1862	 * shape.
1863	 */
1864	/* post card */
1865	atom_asic_init(rdev->mode_info.atom_context);
1866
1867	/* init golden registers */
1868	rv770_init_golden_registers(rdev);
1869
1870	if (rdev->pm.pm_method == PM_METHOD_DPM)
1871		radeon_pm_resume(rdev);
1872
1873	rdev->accel_working = true;
1874	r = rv770_startup(rdev);
1875	if (r) {
1876		DRM_ERROR("r600 startup failed on resume\n");
1877		rdev->accel_working = false;
1878		return r;
1879	}
1880
1881	return r;
1882
1883}
1884
1885int rv770_suspend(struct radeon_device *rdev)
1886{
1887	radeon_pm_suspend(rdev);
1888	radeon_audio_fini(rdev);
1889	if (rdev->has_uvd) {
1890		uvd_v1_0_fini(rdev);
1891		radeon_uvd_suspend(rdev);
1892	}
1893	r700_cp_stop(rdev);
1894	r600_dma_stop(rdev);
1895	r600_irq_suspend(rdev);
1896	radeon_wb_disable(rdev);
1897	rv770_pcie_gart_disable(rdev);
1898
1899	return 0;
1900}
1901
1902/* Plan is to move initialization in that function and use
1903 * helper function so that radeon_device_init pretty much
1904 * do nothing more than calling asic specific function. This
1905 * should also allow to remove a bunch of callback function
1906 * like vram_info.
1907 */
1908int rv770_init(struct radeon_device *rdev)
1909{
1910	int r;
1911
1912	/* Read BIOS */
1913	if (!radeon_get_bios(rdev)) {
1914		if (ASIC_IS_AVIVO(rdev))
1915			return -EINVAL;
1916	}
1917	/* Must be an ATOMBIOS */
1918	if (!rdev->is_atom_bios) {
1919		dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
1920		return -EINVAL;
1921	}
1922	r = radeon_atombios_init(rdev);
1923	if (r)
1924		return r;
1925	/* Post card if necessary */
1926	if (!radeon_card_posted(rdev)) {
1927		if (!rdev->bios) {
1928			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
1929			return -EINVAL;
1930		}
1931		DRM_INFO("GPU not posted. posting now...\n");
1932		atom_asic_init(rdev->mode_info.atom_context);
1933	}
1934	/* init golden registers */
1935	rv770_init_golden_registers(rdev);
1936	/* Initialize scratch registers */
1937	r600_scratch_init(rdev);
1938	/* Initialize surface registers */
1939	radeon_surface_init(rdev);
1940	/* Initialize clocks */
1941	radeon_get_clock_info(rdev->ddev);
1942	/* Fence driver */
1943	r = radeon_fence_driver_init(rdev);
1944	if (r)
1945		return r;
1946	/* initialize AGP */
1947	if (rdev->flags & RADEON_IS_AGP) {
1948		r = radeon_agp_init(rdev);
1949		if (r)
1950			radeon_agp_disable(rdev);
1951	}
1952	r = rv770_mc_init(rdev);
1953	if (r)
1954		return r;
1955	/* Memory manager */
1956	r = radeon_bo_init(rdev);
1957	if (r)
1958		return r;
1959
1960	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1961		r = r600_init_microcode(rdev);
1962		if (r) {
1963			DRM_ERROR("Failed to load firmware!\n");
1964			return r;
1965		}
1966	}
1967
1968	/* Initialize power management */
1969	radeon_pm_init(rdev);
1970
1971	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
1972	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1973
1974	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
1975	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
1976
1977	rv770_uvd_init(rdev);
1978
1979	rdev->ih.ring_obj = NULL;
1980	r600_ih_ring_init(rdev, 64 * 1024);
1981
1982	r = r600_pcie_gart_init(rdev);
1983	if (r)
1984		return r;
1985
1986	rdev->accel_working = true;
1987	r = rv770_startup(rdev);
1988	if (r) {
1989		dev_err(rdev->dev, "disabling GPU acceleration\n");
1990		r700_cp_fini(rdev);
1991		r600_dma_fini(rdev);
1992		r600_irq_fini(rdev);
1993		radeon_wb_fini(rdev);
1994		radeon_ib_pool_fini(rdev);
1995		radeon_irq_kms_fini(rdev);
1996		rv770_pcie_gart_fini(rdev);
1997		rdev->accel_working = false;
1998	}
1999
2000	return 0;
2001}
2002
2003void rv770_fini(struct radeon_device *rdev)
2004{
2005	radeon_pm_fini(rdev);
2006	r700_cp_fini(rdev);
2007	r600_dma_fini(rdev);
2008	r600_irq_fini(rdev);
2009	radeon_wb_fini(rdev);
2010	radeon_ib_pool_fini(rdev);
2011	radeon_irq_kms_fini(rdev);
2012	uvd_v1_0_fini(rdev);
2013	radeon_uvd_fini(rdev);
2014	rv770_pcie_gart_fini(rdev);
2015	r600_vram_scratch_fini(rdev);
2016	radeon_gem_fini(rdev);
2017	radeon_fence_driver_fini(rdev);
2018	radeon_agp_fini(rdev);
2019	radeon_bo_fini(rdev);
2020	radeon_atombios_fini(rdev);
2021	kfree(rdev->bios);
2022	rdev->bios = NULL;
2023}
2024
2025static void rv770_pcie_gen2_enable(struct radeon_device *rdev)
2026{
2027	u32 link_width_cntl, lanes, speed_cntl, tmp;
2028	u16 link_cntl2;
2029
2030	if (radeon_pcie_gen2 == 0)
2031		return;
2032
2033	if (rdev->flags & RADEON_IS_IGP)
2034		return;
2035
2036	if (!(rdev->flags & RADEON_IS_PCIE))
2037		return;
2038
2039	/* x2 cards have a special sequence */
2040	if (ASIC_IS_X2(rdev))
2041		return;
2042
2043	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
2044		(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
2045		return;
2046
2047	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
2048
2049	/* advertise upconfig capability */
2050	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2051	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2052	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2053	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2054	if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
2055		lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
2056		link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
2057				     LC_RECONFIG_ARC_MISSING_ESCAPE);
2058		link_width_cntl |= lanes | LC_RECONFIG_NOW |
2059			LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT;
2060		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2061	} else {
2062		link_width_cntl |= LC_UPCONFIGURE_DIS;
2063		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2064	}
2065
2066	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2067	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
2068	    (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
2069
2070		tmp = RREG32(0x541c);
2071		WREG32(0x541c, tmp | 0x8);
2072		WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
2073		link_cntl2 = RREG16(0x4088);
2074		link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
2075		link_cntl2 |= 0x2;
2076		WREG16(0x4088, link_cntl2);
2077		WREG32(MM_CFGREGS_CNTL, 0);
2078
2079		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2080		speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
2081		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2082
2083		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2084		speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
2085		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2086
2087		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2088		speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
2089		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2090
2091		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2092		speed_cntl |= LC_GEN2_EN_STRAP;
2093		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2094
2095	} else {
2096		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2097		/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
2098		if (1)
2099			link_width_cntl |= LC_UPCONFIGURE_DIS;
2100		else
2101			link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2102		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2103	}
2104}