1/*
   2 * Copyright 2008 Advanced Micro Devices, Inc.
   3 * Copyright 2008 Red Hat Inc.
   4 * Copyright 2009 Jerome Glisse.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 *
  24 * Authors: Dave Airlie
  25 *          Alex Deucher
  26 *          Jerome Glisse
  27 */
  28
  29#include <linux/firmware.h>
  30#include <linux/pci.h>
  31#include <linux/slab.h>
  32
  33#include <drm/drm_device.h>
  34#include <drm/radeon_drm.h>
  35#include <drm/drm_fourcc.h>
  36#include <drm/drm_framebuffer.h>
  37
  38#include "atom.h"
  39#include "avivod.h"
  40#include "radeon.h"
  41#include "radeon_asic.h"
  42#include "radeon_audio.h"
 
  43#include "rv770d.h"
  44#include "rv770.h"
 
  45
  46#define R700_PFP_UCODE_SIZE 848
  47#define R700_PM4_UCODE_SIZE 1360
  48
  49static void rv770_gpu_init(struct radeon_device *rdev);
  50void rv770_fini(struct radeon_device *rdev);
  51static void rv770_pcie_gen2_enable(struct radeon_device *rdev);
  52int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk);
  53
  54int rv770_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
  55{
  56	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
  57	int r;
  58
  59	/* RV740 uses evergreen uvd clk programming */
  60	if (rdev->family == CHIP_RV740)
  61		return evergreen_set_uvd_clocks(rdev, vclk, dclk);
  62
  63	/* bypass vclk and dclk with bclk */
  64	WREG32_P(CG_UPLL_FUNC_CNTL_2,
  65		 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
  66		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
  67
  68	if (!vclk || !dclk) {
  69		/* keep the Bypass mode, put PLL to sleep */
  70		WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
  71		return 0;
  72	}
  73
  74	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000,
  75					  43663, 0x03FFFFFE, 1, 30, ~0,
  76					  &fb_div, &vclk_div, &dclk_div);
  77	if (r)
  78		return r;
  79
  80	fb_div |= 1;
  81	vclk_div -= 1;
  82	dclk_div -= 1;
  83
  84	/* set UPLL_FB_DIV to 0x50000 */
  85	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(0x50000), ~UPLL_FB_DIV_MASK);
  86
  87	/* deassert UPLL_RESET and UPLL_SLEEP */
  88	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~(UPLL_RESET_MASK | UPLL_SLEEP_MASK));
  89
  90	/* assert BYPASS EN and FB_DIV[0] <- ??? why? */
  91	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
  92	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(1), ~UPLL_FB_DIV(1));
  93
  94	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
  95	if (r)
  96		return r;
  97
  98	/* assert PLL_RESET */
  99	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
 100
 101	/* set the required FB_DIV, REF_DIV, Post divder values */
 102	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_REF_DIV(1), ~UPLL_REF_DIV_MASK);
 103	WREG32_P(CG_UPLL_FUNC_CNTL_2,
 104		 UPLL_SW_HILEN(vclk_div >> 1) |
 105		 UPLL_SW_LOLEN((vclk_div >> 1) + (vclk_div & 1)) |
 106		 UPLL_SW_HILEN2(dclk_div >> 1) |
 107		 UPLL_SW_LOLEN2((dclk_div >> 1) + (dclk_div & 1)),
 108		 ~UPLL_SW_MASK);
 109
 110	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div),
 111		 ~UPLL_FB_DIV_MASK);
 112
 113	/* give the PLL some time to settle */
 114	mdelay(15);
 115
 116	/* deassert PLL_RESET */
 117	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
 118
 119	mdelay(15);
 120
 121	/* deassert BYPASS EN and FB_DIV[0] <- ??? why? */
 122	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
 123	WREG32_P(CG_UPLL_FUNC_CNTL_3, 0, ~UPLL_FB_DIV(1));
 124
 125	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
 126	if (r)
 127		return r;
 128
 129	/* switch VCLK and DCLK selection */
 130	WREG32_P(CG_UPLL_FUNC_CNTL_2,
 131		 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
 132		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
 133
 134	mdelay(100);
 135
 136	return 0;
 137}
 138
 139static const u32 r7xx_golden_registers[] = {
 
 140	0x8d00, 0xffffffff, 0x0e0e0074,
 141	0x8d04, 0xffffffff, 0x013a2b34,
 142	0x9508, 0xffffffff, 0x00000002,
 143	0x8b20, 0xffffffff, 0,
 144	0x88c4, 0xffffffff, 0x000000c2,
 145	0x28350, 0xffffffff, 0,
 146	0x9058, 0xffffffff, 0x0fffc40f,
 147	0x240c, 0xffffffff, 0x00000380,
 148	0x733c, 0xffffffff, 0x00000002,
 149	0x2650, 0x00040000, 0,
 150	0x20bc, 0x00040000, 0,
 151	0x7300, 0xffffffff, 0x001000f0
 152};
 153
 154static const u32 r7xx_golden_dyn_gpr_registers[] = {
 
 155	0x8db0, 0xffffffff, 0x98989898,
 156	0x8db4, 0xffffffff, 0x98989898,
 157	0x8db8, 0xffffffff, 0x98989898,
 158	0x8dbc, 0xffffffff, 0x98989898,
 159	0x8dc0, 0xffffffff, 0x98989898,
 160	0x8dc4, 0xffffffff, 0x98989898,
 161	0x8dc8, 0xffffffff, 0x98989898,
 162	0x8dcc, 0xffffffff, 0x98989898,
 163	0x88c4, 0xffffffff, 0x00000082
 164};
 165
 166static const u32 rv770_golden_registers[] = {
 
 167	0x562c, 0xffffffff, 0,
 168	0x3f90, 0xffffffff, 0,
 169	0x9148, 0xffffffff, 0,
 170	0x3f94, 0xffffffff, 0,
 171	0x914c, 0xffffffff, 0,
 172	0x9698, 0x18000000, 0x18000000
 173};
 174
 175static const u32 rv770ce_golden_registers[] = {
 
 176	0x562c, 0xffffffff, 0,
 177	0x3f90, 0xffffffff, 0x00cc0000,
 178	0x9148, 0xffffffff, 0x00cc0000,
 179	0x3f94, 0xffffffff, 0x00cc0000,
 180	0x914c, 0xffffffff, 0x00cc0000,
 181	0x9b7c, 0xffffffff, 0x00fa0000,
 182	0x3f8c, 0xffffffff, 0x00fa0000,
 183	0x9698, 0x18000000, 0x18000000
 184};
 185
 186static const u32 rv770_mgcg_init[] = {
 
 187	0x8bcc, 0xffffffff, 0x130300f9,
 188	0x5448, 0xffffffff, 0x100,
 189	0x55e4, 0xffffffff, 0x100,
 190	0x160c, 0xffffffff, 0x100,
 191	0x5644, 0xffffffff, 0x100,
 192	0xc164, 0xffffffff, 0x100,
 193	0x8a18, 0xffffffff, 0x100,
 194	0x897c, 0xffffffff, 0x8000100,
 195	0x8b28, 0xffffffff, 0x3c000100,
 196	0x9144, 0xffffffff, 0x100,
 197	0x9a1c, 0xffffffff, 0x10000,
 198	0x9a50, 0xffffffff, 0x100,
 199	0x9a1c, 0xffffffff, 0x10001,
 200	0x9a50, 0xffffffff, 0x100,
 201	0x9a1c, 0xffffffff, 0x10002,
 202	0x9a50, 0xffffffff, 0x100,
 203	0x9a1c, 0xffffffff, 0x10003,
 204	0x9a50, 0xffffffff, 0x100,
 205	0x9a1c, 0xffffffff, 0x0,
 206	0x9870, 0xffffffff, 0x100,
 207	0x8d58, 0xffffffff, 0x100,
 208	0x9500, 0xffffffff, 0x0,
 209	0x9510, 0xffffffff, 0x100,
 210	0x9500, 0xffffffff, 0x1,
 211	0x9510, 0xffffffff, 0x100,
 212	0x9500, 0xffffffff, 0x2,
 213	0x9510, 0xffffffff, 0x100,
 214	0x9500, 0xffffffff, 0x3,
 215	0x9510, 0xffffffff, 0x100,
 216	0x9500, 0xffffffff, 0x4,
 217	0x9510, 0xffffffff, 0x100,
 218	0x9500, 0xffffffff, 0x5,
 219	0x9510, 0xffffffff, 0x100,
 220	0x9500, 0xffffffff, 0x6,
 221	0x9510, 0xffffffff, 0x100,
 222	0x9500, 0xffffffff, 0x7,
 223	0x9510, 0xffffffff, 0x100,
 224	0x9500, 0xffffffff, 0x8,
 225	0x9510, 0xffffffff, 0x100,
 226	0x9500, 0xffffffff, 0x9,
 227	0x9510, 0xffffffff, 0x100,
 228	0x9500, 0xffffffff, 0x8000,
 229	0x9490, 0xffffffff, 0x0,
 230	0x949c, 0xffffffff, 0x100,
 231	0x9490, 0xffffffff, 0x1,
 232	0x949c, 0xffffffff, 0x100,
 233	0x9490, 0xffffffff, 0x2,
 234	0x949c, 0xffffffff, 0x100,
 235	0x9490, 0xffffffff, 0x3,
 236	0x949c, 0xffffffff, 0x100,
 237	0x9490, 0xffffffff, 0x4,
 238	0x949c, 0xffffffff, 0x100,
 239	0x9490, 0xffffffff, 0x5,
 240	0x949c, 0xffffffff, 0x100,
 241	0x9490, 0xffffffff, 0x6,
 242	0x949c, 0xffffffff, 0x100,
 243	0x9490, 0xffffffff, 0x7,
 244	0x949c, 0xffffffff, 0x100,
 245	0x9490, 0xffffffff, 0x8,
 246	0x949c, 0xffffffff, 0x100,
 247	0x9490, 0xffffffff, 0x9,
 248	0x949c, 0xffffffff, 0x100,
 249	0x9490, 0xffffffff, 0x8000,
 250	0x9604, 0xffffffff, 0x0,
 251	0x9654, 0xffffffff, 0x100,
 252	0x9604, 0xffffffff, 0x1,
 253	0x9654, 0xffffffff, 0x100,
 254	0x9604, 0xffffffff, 0x2,
 255	0x9654, 0xffffffff, 0x100,
 256	0x9604, 0xffffffff, 0x3,
 257	0x9654, 0xffffffff, 0x100,
 258	0x9604, 0xffffffff, 0x4,
 259	0x9654, 0xffffffff, 0x100,
 260	0x9604, 0xffffffff, 0x5,
 261	0x9654, 0xffffffff, 0x100,
 262	0x9604, 0xffffffff, 0x6,
 263	0x9654, 0xffffffff, 0x100,
 264	0x9604, 0xffffffff, 0x7,
 265	0x9654, 0xffffffff, 0x100,
 266	0x9604, 0xffffffff, 0x8,
 267	0x9654, 0xffffffff, 0x100,
 268	0x9604, 0xffffffff, 0x9,
 269	0x9654, 0xffffffff, 0x100,
 270	0x9604, 0xffffffff, 0x80000000,
 271	0x9030, 0xffffffff, 0x100,
 272	0x9034, 0xffffffff, 0x100,
 273	0x9038, 0xffffffff, 0x100,
 274	0x903c, 0xffffffff, 0x100,
 275	0x9040, 0xffffffff, 0x100,
 276	0xa200, 0xffffffff, 0x100,
 277	0xa204, 0xffffffff, 0x100,
 278	0xa208, 0xffffffff, 0x100,
 279	0xa20c, 0xffffffff, 0x100,
 280	0x971c, 0xffffffff, 0x100,
 281	0x915c, 0xffffffff, 0x00020001,
 282	0x9160, 0xffffffff, 0x00040003,
 283	0x916c, 0xffffffff, 0x00060005,
 284	0x9170, 0xffffffff, 0x00080007,
 285	0x9174, 0xffffffff, 0x000a0009,
 286	0x9178, 0xffffffff, 0x000c000b,
 287	0x917c, 0xffffffff, 0x000e000d,
 288	0x9180, 0xffffffff, 0x0010000f,
 289	0x918c, 0xffffffff, 0x00120011,
 290	0x9190, 0xffffffff, 0x00140013,
 291	0x9194, 0xffffffff, 0x00020001,
 292	0x9198, 0xffffffff, 0x00040003,
 293	0x919c, 0xffffffff, 0x00060005,
 294	0x91a8, 0xffffffff, 0x00080007,
 295	0x91ac, 0xffffffff, 0x000a0009,
 296	0x91b0, 0xffffffff, 0x000c000b,
 297	0x91b4, 0xffffffff, 0x000e000d,
 298	0x91b8, 0xffffffff, 0x0010000f,
 299	0x91c4, 0xffffffff, 0x00120011,
 300	0x91c8, 0xffffffff, 0x00140013,
 301	0x91cc, 0xffffffff, 0x00020001,
 302	0x91d0, 0xffffffff, 0x00040003,
 303	0x91d4, 0xffffffff, 0x00060005,
 304	0x91e0, 0xffffffff, 0x00080007,
 305	0x91e4, 0xffffffff, 0x000a0009,
 306	0x91e8, 0xffffffff, 0x000c000b,
 307	0x91ec, 0xffffffff, 0x00020001,
 308	0x91f0, 0xffffffff, 0x00040003,
 309	0x91f4, 0xffffffff, 0x00060005,
 310	0x9200, 0xffffffff, 0x00080007,
 311	0x9204, 0xffffffff, 0x000a0009,
 312	0x9208, 0xffffffff, 0x000c000b,
 313	0x920c, 0xffffffff, 0x000e000d,
 314	0x9210, 0xffffffff, 0x0010000f,
 315	0x921c, 0xffffffff, 0x00120011,
 316	0x9220, 0xffffffff, 0x00140013,
 317	0x9224, 0xffffffff, 0x00020001,
 318	0x9228, 0xffffffff, 0x00040003,
 319	0x922c, 0xffffffff, 0x00060005,
 320	0x9238, 0xffffffff, 0x00080007,
 321	0x923c, 0xffffffff, 0x000a0009,
 322	0x9240, 0xffffffff, 0x000c000b,
 323	0x9244, 0xffffffff, 0x000e000d,
 324	0x9248, 0xffffffff, 0x0010000f,
 325	0x9254, 0xffffffff, 0x00120011,
 326	0x9258, 0xffffffff, 0x00140013,
 327	0x925c, 0xffffffff, 0x00020001,
 328	0x9260, 0xffffffff, 0x00040003,
 329	0x9264, 0xffffffff, 0x00060005,
 330	0x9270, 0xffffffff, 0x00080007,
 331	0x9274, 0xffffffff, 0x000a0009,
 332	0x9278, 0xffffffff, 0x000c000b,
 333	0x927c, 0xffffffff, 0x000e000d,
 334	0x9280, 0xffffffff, 0x0010000f,
 335	0x928c, 0xffffffff, 0x00120011,
 336	0x9290, 0xffffffff, 0x00140013,
 337	0x9294, 0xffffffff, 0x00020001,
 338	0x929c, 0xffffffff, 0x00040003,
 339	0x92a0, 0xffffffff, 0x00060005,
 340	0x92a4, 0xffffffff, 0x00080007
 341};
 342
 343static const u32 rv710_golden_registers[] = {
 
 344	0x3f90, 0x00ff0000, 0x00fc0000,
 345	0x9148, 0x00ff0000, 0x00fc0000,
 346	0x3f94, 0x00ff0000, 0x00fc0000,
 347	0x914c, 0x00ff0000, 0x00fc0000,
 348	0xb4c, 0x00000020, 0x00000020,
 349	0xa180, 0xffffffff, 0x00003f3f
 350};
 351
 352static const u32 rv710_mgcg_init[] = {
 
 353	0x8bcc, 0xffffffff, 0x13030040,
 354	0x5448, 0xffffffff, 0x100,
 355	0x55e4, 0xffffffff, 0x100,
 356	0x160c, 0xffffffff, 0x100,
 357	0x5644, 0xffffffff, 0x100,
 358	0xc164, 0xffffffff, 0x100,
 359	0x8a18, 0xffffffff, 0x100,
 360	0x897c, 0xffffffff, 0x8000100,
 361	0x8b28, 0xffffffff, 0x3c000100,
 362	0x9144, 0xffffffff, 0x100,
 363	0x9a1c, 0xffffffff, 0x10000,
 364	0x9a50, 0xffffffff, 0x100,
 365	0x9a1c, 0xffffffff, 0x0,
 366	0x9870, 0xffffffff, 0x100,
 367	0x8d58, 0xffffffff, 0x100,
 368	0x9500, 0xffffffff, 0x0,
 369	0x9510, 0xffffffff, 0x100,
 370	0x9500, 0xffffffff, 0x1,
 371	0x9510, 0xffffffff, 0x100,
 372	0x9500, 0xffffffff, 0x8000,
 373	0x9490, 0xffffffff, 0x0,
 374	0x949c, 0xffffffff, 0x100,
 375	0x9490, 0xffffffff, 0x1,
 376	0x949c, 0xffffffff, 0x100,
 377	0x9490, 0xffffffff, 0x8000,
 378	0x9604, 0xffffffff, 0x0,
 379	0x9654, 0xffffffff, 0x100,
 380	0x9604, 0xffffffff, 0x1,
 381	0x9654, 0xffffffff, 0x100,
 382	0x9604, 0xffffffff, 0x80000000,
 383	0x9030, 0xffffffff, 0x100,
 384	0x9034, 0xffffffff, 0x100,
 385	0x9038, 0xffffffff, 0x100,
 386	0x903c, 0xffffffff, 0x100,
 387	0x9040, 0xffffffff, 0x100,
 388	0xa200, 0xffffffff, 0x100,
 389	0xa204, 0xffffffff, 0x100,
 390	0xa208, 0xffffffff, 0x100,
 391	0xa20c, 0xffffffff, 0x100,
 392	0x971c, 0xffffffff, 0x100,
 393	0x915c, 0xffffffff, 0x00020001,
 394	0x9174, 0xffffffff, 0x00000003,
 395	0x9178, 0xffffffff, 0x00050001,
 396	0x917c, 0xffffffff, 0x00030002,
 397	0x918c, 0xffffffff, 0x00000004,
 398	0x9190, 0xffffffff, 0x00070006,
 399	0x9194, 0xffffffff, 0x00050001,
 400	0x9198, 0xffffffff, 0x00030002,
 401	0x91a8, 0xffffffff, 0x00000004,
 402	0x91ac, 0xffffffff, 0x00070006,
 403	0x91e8, 0xffffffff, 0x00000001,
 404	0x9294, 0xffffffff, 0x00000001,
 405	0x929c, 0xffffffff, 0x00000002,
 406	0x92a0, 0xffffffff, 0x00040003,
 407	0x9150, 0xffffffff, 0x4d940000
 408};
 409
 410static const u32 rv730_golden_registers[] = {
 
 411	0x3f90, 0x00ff0000, 0x00f00000,
 412	0x9148, 0x00ff0000, 0x00f00000,
 413	0x3f94, 0x00ff0000, 0x00f00000,
 414	0x914c, 0x00ff0000, 0x00f00000,
 415	0x900c, 0xffffffff, 0x003b033f,
 416	0xb4c, 0x00000020, 0x00000020,
 417	0xa180, 0xffffffff, 0x00003f3f
 418};
 419
 420static const u32 rv730_mgcg_init[] = {
 
 421	0x8bcc, 0xffffffff, 0x130300f9,
 422	0x5448, 0xffffffff, 0x100,
 423	0x55e4, 0xffffffff, 0x100,
 424	0x160c, 0xffffffff, 0x100,
 425	0x5644, 0xffffffff, 0x100,
 426	0xc164, 0xffffffff, 0x100,
 427	0x8a18, 0xffffffff, 0x100,
 428	0x897c, 0xffffffff, 0x8000100,
 429	0x8b28, 0xffffffff, 0x3c000100,
 430	0x9144, 0xffffffff, 0x100,
 431	0x9a1c, 0xffffffff, 0x10000,
 432	0x9a50, 0xffffffff, 0x100,
 433	0x9a1c, 0xffffffff, 0x10001,
 434	0x9a50, 0xffffffff, 0x100,
 435	0x9a1c, 0xffffffff, 0x0,
 436	0x9870, 0xffffffff, 0x100,
 437	0x8d58, 0xffffffff, 0x100,
 438	0x9500, 0xffffffff, 0x0,
 439	0x9510, 0xffffffff, 0x100,
 440	0x9500, 0xffffffff, 0x1,
 441	0x9510, 0xffffffff, 0x100,
 442	0x9500, 0xffffffff, 0x2,
 443	0x9510, 0xffffffff, 0x100,
 444	0x9500, 0xffffffff, 0x3,
 445	0x9510, 0xffffffff, 0x100,
 446	0x9500, 0xffffffff, 0x4,
 447	0x9510, 0xffffffff, 0x100,
 448	0x9500, 0xffffffff, 0x5,
 449	0x9510, 0xffffffff, 0x100,
 450	0x9500, 0xffffffff, 0x6,
 451	0x9510, 0xffffffff, 0x100,
 452	0x9500, 0xffffffff, 0x7,
 453	0x9510, 0xffffffff, 0x100,
 454	0x9500, 0xffffffff, 0x8000,
 455	0x9490, 0xffffffff, 0x0,
 456	0x949c, 0xffffffff, 0x100,
 457	0x9490, 0xffffffff, 0x1,
 458	0x949c, 0xffffffff, 0x100,
 459	0x9490, 0xffffffff, 0x2,
 460	0x949c, 0xffffffff, 0x100,
 461	0x9490, 0xffffffff, 0x3,
 462	0x949c, 0xffffffff, 0x100,
 463	0x9490, 0xffffffff, 0x4,
 464	0x949c, 0xffffffff, 0x100,
 465	0x9490, 0xffffffff, 0x5,
 466	0x949c, 0xffffffff, 0x100,
 467	0x9490, 0xffffffff, 0x6,
 468	0x949c, 0xffffffff, 0x100,
 469	0x9490, 0xffffffff, 0x7,
 470	0x949c, 0xffffffff, 0x100,
 471	0x9490, 0xffffffff, 0x8000,
 472	0x9604, 0xffffffff, 0x0,
 473	0x9654, 0xffffffff, 0x100,
 474	0x9604, 0xffffffff, 0x1,
 475	0x9654, 0xffffffff, 0x100,
 476	0x9604, 0xffffffff, 0x2,
 477	0x9654, 0xffffffff, 0x100,
 478	0x9604, 0xffffffff, 0x3,
 479	0x9654, 0xffffffff, 0x100,
 480	0x9604, 0xffffffff, 0x4,
 481	0x9654, 0xffffffff, 0x100,
 482	0x9604, 0xffffffff, 0x5,
 483	0x9654, 0xffffffff, 0x100,
 484	0x9604, 0xffffffff, 0x6,
 485	0x9654, 0xffffffff, 0x100,
 486	0x9604, 0xffffffff, 0x7,
 487	0x9654, 0xffffffff, 0x100,
 488	0x9604, 0xffffffff, 0x80000000,
 489	0x9030, 0xffffffff, 0x100,
 490	0x9034, 0xffffffff, 0x100,
 491	0x9038, 0xffffffff, 0x100,
 492	0x903c, 0xffffffff, 0x100,
 493	0x9040, 0xffffffff, 0x100,
 494	0xa200, 0xffffffff, 0x100,
 495	0xa204, 0xffffffff, 0x100,
 496	0xa208, 0xffffffff, 0x100,
 497	0xa20c, 0xffffffff, 0x100,
 498	0x971c, 0xffffffff, 0x100,
 499	0x915c, 0xffffffff, 0x00020001,
 500	0x916c, 0xffffffff, 0x00040003,
 501	0x9170, 0xffffffff, 0x00000005,
 502	0x9178, 0xffffffff, 0x00050001,
 503	0x917c, 0xffffffff, 0x00030002,
 504	0x918c, 0xffffffff, 0x00000004,
 505	0x9190, 0xffffffff, 0x00070006,
 506	0x9194, 0xffffffff, 0x00050001,
 507	0x9198, 0xffffffff, 0x00030002,
 508	0x91a8, 0xffffffff, 0x00000004,
 509	0x91ac, 0xffffffff, 0x00070006,
 510	0x91b0, 0xffffffff, 0x00050001,
 511	0x91b4, 0xffffffff, 0x00030002,
 512	0x91c4, 0xffffffff, 0x00000004,
 513	0x91c8, 0xffffffff, 0x00070006,
 514	0x91cc, 0xffffffff, 0x00050001,
 515	0x91d0, 0xffffffff, 0x00030002,
 516	0x91e0, 0xffffffff, 0x00000004,
 517	0x91e4, 0xffffffff, 0x00070006,
 518	0x91e8, 0xffffffff, 0x00000001,
 519	0x91ec, 0xffffffff, 0x00050001,
 520	0x91f0, 0xffffffff, 0x00030002,
 521	0x9200, 0xffffffff, 0x00000004,
 522	0x9204, 0xffffffff, 0x00070006,
 523	0x9208, 0xffffffff, 0x00050001,
 524	0x920c, 0xffffffff, 0x00030002,
 525	0x921c, 0xffffffff, 0x00000004,
 526	0x9220, 0xffffffff, 0x00070006,
 527	0x9224, 0xffffffff, 0x00050001,
 528	0x9228, 0xffffffff, 0x00030002,
 529	0x9238, 0xffffffff, 0x00000004,
 530	0x923c, 0xffffffff, 0x00070006,
 531	0x9240, 0xffffffff, 0x00050001,
 532	0x9244, 0xffffffff, 0x00030002,
 533	0x9254, 0xffffffff, 0x00000004,
 534	0x9258, 0xffffffff, 0x00070006,
 535	0x9294, 0xffffffff, 0x00000001,
 536	0x929c, 0xffffffff, 0x00000002,
 537	0x92a0, 0xffffffff, 0x00040003,
 538	0x92a4, 0xffffffff, 0x00000005
 539};
 540
 541static const u32 rv740_golden_registers[] = {
 
 542	0x88c4, 0xffffffff, 0x00000082,
 543	0x28a50, 0xfffffffc, 0x00000004,
 544	0x2650, 0x00040000, 0,
 545	0x20bc, 0x00040000, 0,
 546	0x733c, 0xffffffff, 0x00000002,
 547	0x7300, 0xffffffff, 0x001000f0,
 548	0x3f90, 0x00ff0000, 0,
 549	0x9148, 0x00ff0000, 0,
 550	0x3f94, 0x00ff0000, 0,
 551	0x914c, 0x00ff0000, 0,
 552	0x240c, 0xffffffff, 0x00000380,
 553	0x8a14, 0x00000007, 0x00000007,
 554	0x8b24, 0xffffffff, 0x00ff0fff,
 555	0x28a4c, 0xffffffff, 0x00004000,
 556	0xa180, 0xffffffff, 0x00003f3f,
 557	0x8d00, 0xffffffff, 0x0e0e003a,
 558	0x8d04, 0xffffffff, 0x013a0e2a,
 559	0x8c00, 0xffffffff, 0xe400000f,
 560	0x8db0, 0xffffffff, 0x98989898,
 561	0x8db4, 0xffffffff, 0x98989898,
 562	0x8db8, 0xffffffff, 0x98989898,
 563	0x8dbc, 0xffffffff, 0x98989898,
 564	0x8dc0, 0xffffffff, 0x98989898,
 565	0x8dc4, 0xffffffff, 0x98989898,
 566	0x8dc8, 0xffffffff, 0x98989898,
 567	0x8dcc, 0xffffffff, 0x98989898,
 568	0x9058, 0xffffffff, 0x0fffc40f,
 569	0x900c, 0xffffffff, 0x003b033f,
 570	0x28350, 0xffffffff, 0,
 571	0x8cf0, 0x1fffffff, 0x08e00420,
 572	0x9508, 0xffffffff, 0x00000002,
 573	0x88c4, 0xffffffff, 0x000000c2,
 574	0x9698, 0x18000000, 0x18000000
 575};
 576
 577static const u32 rv740_mgcg_init[] = {
 
 578	0x8bcc, 0xffffffff, 0x13030100,
 579	0x5448, 0xffffffff, 0x100,
 580	0x55e4, 0xffffffff, 0x100,
 581	0x160c, 0xffffffff, 0x100,
 582	0x5644, 0xffffffff, 0x100,
 583	0xc164, 0xffffffff, 0x100,
 584	0x8a18, 0xffffffff, 0x100,
 585	0x897c, 0xffffffff, 0x100,
 586	0x8b28, 0xffffffff, 0x100,
 587	0x9144, 0xffffffff, 0x100,
 588	0x9a1c, 0xffffffff, 0x10000,
 589	0x9a50, 0xffffffff, 0x100,
 590	0x9a1c, 0xffffffff, 0x10001,
 591	0x9a50, 0xffffffff, 0x100,
 592	0x9a1c, 0xffffffff, 0x10002,
 593	0x9a50, 0xffffffff, 0x100,
 594	0x9a1c, 0xffffffff, 0x10003,
 595	0x9a50, 0xffffffff, 0x100,
 596	0x9a1c, 0xffffffff, 0x0,
 597	0x9870, 0xffffffff, 0x100,
 598	0x8d58, 0xffffffff, 0x100,
 599	0x9500, 0xffffffff, 0x0,
 600	0x9510, 0xffffffff, 0x100,
 601	0x9500, 0xffffffff, 0x1,
 602	0x9510, 0xffffffff, 0x100,
 603	0x9500, 0xffffffff, 0x2,
 604	0x9510, 0xffffffff, 0x100,
 605	0x9500, 0xffffffff, 0x3,
 606	0x9510, 0xffffffff, 0x100,
 607	0x9500, 0xffffffff, 0x4,
 608	0x9510, 0xffffffff, 0x100,
 609	0x9500, 0xffffffff, 0x5,
 610	0x9510, 0xffffffff, 0x100,
 611	0x9500, 0xffffffff, 0x6,
 612	0x9510, 0xffffffff, 0x100,
 613	0x9500, 0xffffffff, 0x7,
 614	0x9510, 0xffffffff, 0x100,
 615	0x9500, 0xffffffff, 0x8000,
 616	0x9490, 0xffffffff, 0x0,
 617	0x949c, 0xffffffff, 0x100,
 618	0x9490, 0xffffffff, 0x1,
 619	0x949c, 0xffffffff, 0x100,
 620	0x9490, 0xffffffff, 0x2,
 621	0x949c, 0xffffffff, 0x100,
 622	0x9490, 0xffffffff, 0x3,
 623	0x949c, 0xffffffff, 0x100,
 624	0x9490, 0xffffffff, 0x4,
 625	0x949c, 0xffffffff, 0x100,
 626	0x9490, 0xffffffff, 0x5,
 627	0x949c, 0xffffffff, 0x100,
 628	0x9490, 0xffffffff, 0x6,
 629	0x949c, 0xffffffff, 0x100,
 630	0x9490, 0xffffffff, 0x7,
 631	0x949c, 0xffffffff, 0x100,
 632	0x9490, 0xffffffff, 0x8000,
 633	0x9604, 0xffffffff, 0x0,
 634	0x9654, 0xffffffff, 0x100,
 635	0x9604, 0xffffffff, 0x1,
 636	0x9654, 0xffffffff, 0x100,
 637	0x9604, 0xffffffff, 0x2,
 638	0x9654, 0xffffffff, 0x100,
 639	0x9604, 0xffffffff, 0x3,
 640	0x9654, 0xffffffff, 0x100,
 641	0x9604, 0xffffffff, 0x4,
 642	0x9654, 0xffffffff, 0x100,
 643	0x9604, 0xffffffff, 0x5,
 644	0x9654, 0xffffffff, 0x100,
 645	0x9604, 0xffffffff, 0x6,
 646	0x9654, 0xffffffff, 0x100,
 647	0x9604, 0xffffffff, 0x7,
 648	0x9654, 0xffffffff, 0x100,
 649	0x9604, 0xffffffff, 0x80000000,
 650	0x9030, 0xffffffff, 0x100,
 651	0x9034, 0xffffffff, 0x100,
 652	0x9038, 0xffffffff, 0x100,
 653	0x903c, 0xffffffff, 0x100,
 654	0x9040, 0xffffffff, 0x100,
 655	0xa200, 0xffffffff, 0x100,
 656	0xa204, 0xffffffff, 0x100,
 657	0xa208, 0xffffffff, 0x100,
 658	0xa20c, 0xffffffff, 0x100,
 659	0x971c, 0xffffffff, 0x100,
 660	0x915c, 0xffffffff, 0x00020001,
 661	0x9160, 0xffffffff, 0x00040003,
 662	0x916c, 0xffffffff, 0x00060005,
 663	0x9170, 0xffffffff, 0x00080007,
 664	0x9174, 0xffffffff, 0x000a0009,
 665	0x9178, 0xffffffff, 0x000c000b,
 666	0x917c, 0xffffffff, 0x000e000d,
 667	0x9180, 0xffffffff, 0x0010000f,
 668	0x918c, 0xffffffff, 0x00120011,
 669	0x9190, 0xffffffff, 0x00140013,
 670	0x9194, 0xffffffff, 0x00020001,
 671	0x9198, 0xffffffff, 0x00040003,
 672	0x919c, 0xffffffff, 0x00060005,
 673	0x91a8, 0xffffffff, 0x00080007,
 674	0x91ac, 0xffffffff, 0x000a0009,
 675	0x91b0, 0xffffffff, 0x000c000b,
 676	0x91b4, 0xffffffff, 0x000e000d,
 677	0x91b8, 0xffffffff, 0x0010000f,
 678	0x91c4, 0xffffffff, 0x00120011,
 679	0x91c8, 0xffffffff, 0x00140013,
 680	0x91cc, 0xffffffff, 0x00020001,
 681	0x91d0, 0xffffffff, 0x00040003,
 682	0x91d4, 0xffffffff, 0x00060005,
 683	0x91e0, 0xffffffff, 0x00080007,
 684	0x91e4, 0xffffffff, 0x000a0009,
 685	0x91e8, 0xffffffff, 0x000c000b,
 686	0x91ec, 0xffffffff, 0x00020001,
 687	0x91f0, 0xffffffff, 0x00040003,
 688	0x91f4, 0xffffffff, 0x00060005,
 689	0x9200, 0xffffffff, 0x00080007,
 690	0x9204, 0xffffffff, 0x000a0009,
 691	0x9208, 0xffffffff, 0x000c000b,
 692	0x920c, 0xffffffff, 0x000e000d,
 693	0x9210, 0xffffffff, 0x0010000f,
 694	0x921c, 0xffffffff, 0x00120011,
 695	0x9220, 0xffffffff, 0x00140013,
 696	0x9224, 0xffffffff, 0x00020001,
 697	0x9228, 0xffffffff, 0x00040003,
 698	0x922c, 0xffffffff, 0x00060005,
 699	0x9238, 0xffffffff, 0x00080007,
 700	0x923c, 0xffffffff, 0x000a0009,
 701	0x9240, 0xffffffff, 0x000c000b,
 702	0x9244, 0xffffffff, 0x000e000d,
 703	0x9248, 0xffffffff, 0x0010000f,
 704	0x9254, 0xffffffff, 0x00120011,
 705	0x9258, 0xffffffff, 0x00140013,
 706	0x9294, 0xffffffff, 0x00020001,
 707	0x929c, 0xffffffff, 0x00040003,
 708	0x92a0, 0xffffffff, 0x00060005,
 709	0x92a4, 0xffffffff, 0x00080007
 710};
 711
 712static void rv770_init_golden_registers(struct radeon_device *rdev)
 713{
 714	switch (rdev->family) {
 715	case CHIP_RV770:
 716		radeon_program_register_sequence(rdev,
 717						 r7xx_golden_registers,
 718						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
 719		radeon_program_register_sequence(rdev,
 720						 r7xx_golden_dyn_gpr_registers,
 721						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 722		if (rdev->pdev->device == 0x994e)
 723			radeon_program_register_sequence(rdev,
 724							 rv770ce_golden_registers,
 725							 (const u32)ARRAY_SIZE(rv770ce_golden_registers));
 726		else
 727			radeon_program_register_sequence(rdev,
 728							 rv770_golden_registers,
 729							 (const u32)ARRAY_SIZE(rv770_golden_registers));
 730		radeon_program_register_sequence(rdev,
 731						 rv770_mgcg_init,
 732						 (const u32)ARRAY_SIZE(rv770_mgcg_init));
 733		break;
 734	case CHIP_RV730:
 735		radeon_program_register_sequence(rdev,
 736						 r7xx_golden_registers,
 737						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
 738		radeon_program_register_sequence(rdev,
 739						 r7xx_golden_dyn_gpr_registers,
 740						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 741		radeon_program_register_sequence(rdev,
 742						 rv730_golden_registers,
 743						 (const u32)ARRAY_SIZE(rv730_golden_registers));
 744		radeon_program_register_sequence(rdev,
 745						 rv730_mgcg_init,
 746						 (const u32)ARRAY_SIZE(rv730_mgcg_init));
 747		break;
 748	case CHIP_RV710:
 749		radeon_program_register_sequence(rdev,
 750						 r7xx_golden_registers,
 751						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
 752		radeon_program_register_sequence(rdev,
 753						 r7xx_golden_dyn_gpr_registers,
 754						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 755		radeon_program_register_sequence(rdev,
 756						 rv710_golden_registers,
 757						 (const u32)ARRAY_SIZE(rv710_golden_registers));
 758		radeon_program_register_sequence(rdev,
 759						 rv710_mgcg_init,
 760						 (const u32)ARRAY_SIZE(rv710_mgcg_init));
 761		break;
 762	case CHIP_RV740:
 763		radeon_program_register_sequence(rdev,
 764						 rv740_golden_registers,
 765						 (const u32)ARRAY_SIZE(rv740_golden_registers));
 766		radeon_program_register_sequence(rdev,
 767						 rv740_mgcg_init,
 768						 (const u32)ARRAY_SIZE(rv740_mgcg_init));
 769		break;
 770	default:
 771		break;
 772	}
 773}
 774
 775#define PCIE_BUS_CLK                10000
 776#define TCLK                        (PCIE_BUS_CLK / 10)
 777
 778/**
 779 * rv770_get_xclk - get the xclk
 780 *
 781 * @rdev: radeon_device pointer
 782 *
 783 * Returns the reference clock used by the gfx engine
 784 * (r7xx-cayman).
 785 */
 786u32 rv770_get_xclk(struct radeon_device *rdev)
 787{
 788	u32 reference_clock = rdev->clock.spll.reference_freq;
 789	u32 tmp = RREG32(CG_CLKPIN_CNTL);
 790
 791	if (tmp & MUX_TCLK_TO_XCLK)
 792		return TCLK;
 793
 794	if (tmp & XTALIN_DIVIDE)
 795		return reference_clock / 4;
 796
 797	return reference_clock;
 798}
 799
 800void rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, bool async)
 801{
 802	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 803	struct drm_framebuffer *fb = radeon_crtc->base.primary->fb;
 804	u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
 805	int i;
 806
 807	/* Lock the graphics update lock */
 808	tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
 809	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 810
 811	/* flip at hsync for async, default is vsync */
 812	WREG32(AVIVO_D1GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
 813	       async ? AVIVO_D1GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
 814	/* update pitch */
 815	WREG32(AVIVO_D1GRPH_PITCH + radeon_crtc->crtc_offset,
 816	       fb->pitches[0] / fb->format->cpp[0]);
 817	/* update the scanout addresses */
 818	if (radeon_crtc->crtc_id) {
 819		WREG32(D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 820		WREG32(D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 821	} else {
 822		WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 823		WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 824	}
 825	WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 826	       (u32)crtc_base);
 827	WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 828	       (u32)crtc_base);
 829
 830	/* Wait for update_pending to go high. */
 831	for (i = 0; i < rdev->usec_timeout; i++) {
 832		if (RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING)
 833			break;
 834		udelay(1);
 835	}
 836	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
 837
 838	/* Unlock the lock, so double-buffering can take place inside vblank */
 839	tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
 840	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 841}
 842
 843bool rv770_page_flip_pending(struct radeon_device *rdev, int crtc_id)
 844{
 845	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 846
 847	/* Return current update_pending status: */
 848	return !!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) &
 849		AVIVO_D1GRPH_SURFACE_UPDATE_PENDING);
 850}
 851
 852/* get temperature in millidegrees */
 853int rv770_get_temp(struct radeon_device *rdev)
 854{
 855	u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
 856		ASIC_T_SHIFT;
 857	int actual_temp;
 858
 859	if (temp & 0x400)
 860		actual_temp = -256;
 861	else if (temp & 0x200)
 862		actual_temp = 255;
 863	else if (temp & 0x100) {
 864		actual_temp = temp & 0x1ff;
 865		actual_temp |= ~0x1ff;
 866	} else
 867		actual_temp = temp & 0xff;
 868
 869	return (actual_temp * 1000) / 2;
 870}
 871
 872void rv770_pm_misc(struct radeon_device *rdev)
 873{
 874	int req_ps_idx = rdev->pm.requested_power_state_index;
 875	int req_cm_idx = rdev->pm.requested_clock_mode_index;
 876	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
 877	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 878
 879	if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
 880		/* 0xff01 is a flag rather then an actual voltage */
 881		if (voltage->voltage == 0xff01)
 882			return;
 883		if (voltage->voltage != rdev->pm.current_vddc) {
 884			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
 885			rdev->pm.current_vddc = voltage->voltage;
 886			DRM_DEBUG("Setting: v: %d\n", voltage->voltage);
 887		}
 888	}
 889}
 890
 891/*
 892 * GART
 893 */
 894static int rv770_pcie_gart_enable(struct radeon_device *rdev)
 895{
 896	u32 tmp;
 897	int r, i;
 898
 899	if (rdev->gart.robj == NULL) {
 900		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 901		return -EINVAL;
 902	}
 903	r = radeon_gart_table_vram_pin(rdev);
 904	if (r)
 905		return r;
 906	/* Setup L2 cache */
 907	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 908				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 909				EFFECTIVE_L2_QUEUE_SIZE(7));
 910	WREG32(VM_L2_CNTL2, 0);
 911	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 912	/* Setup TLB control */
 913	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 914		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 915		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 916		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 917	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 918	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 919	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 920	if (rdev->family == CHIP_RV740)
 921		WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
 922	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 923	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 924	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 925	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 926	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 927	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 928	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 929	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 930				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 931	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 932			(u32)(rdev->dummy_page.addr >> 12));
 933	for (i = 1; i < 7; i++)
 934		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 935
 936	r600_pcie_gart_tlb_flush(rdev);
 937	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
 938		 (unsigned)(rdev->mc.gtt_size >> 20),
 939		 (unsigned long long)rdev->gart.table_addr);
 940	rdev->gart.ready = true;
 941	return 0;
 942}
 943
 944static void rv770_pcie_gart_disable(struct radeon_device *rdev)
 945{
 946	u32 tmp;
 947	int i;
 948
 949	/* Disable all tables */
 950	for (i = 0; i < 7; i++)
 951		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 952
 953	/* Setup L2 cache */
 954	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 955				EFFECTIVE_L2_QUEUE_SIZE(7));
 956	WREG32(VM_L2_CNTL2, 0);
 957	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 958	/* Setup TLB control */
 959	tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 960	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 961	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 962	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 963	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 964	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 965	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 966	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 967	radeon_gart_table_vram_unpin(rdev);
 968}
 969
 970static void rv770_pcie_gart_fini(struct radeon_device *rdev)
 971{
 972	radeon_gart_fini(rdev);
 973	rv770_pcie_gart_disable(rdev);
 974	radeon_gart_table_vram_free(rdev);
 975}
 976
 977
 978static void rv770_agp_enable(struct radeon_device *rdev)
 979{
 980	u32 tmp;
 981	int i;
 982
 983	/* Setup L2 cache */
 984	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 985				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 986				EFFECTIVE_L2_QUEUE_SIZE(7));
 987	WREG32(VM_L2_CNTL2, 0);
 988	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 989	/* Setup TLB control */
 990	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 991		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 992		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 993		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 994	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 995	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 996	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 997	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 998	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 999	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1000	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1001	for (i = 0; i < 7; i++)
1002		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1003}
1004
1005static void rv770_mc_program(struct radeon_device *rdev)
1006{
1007	struct rv515_mc_save save;
1008	u32 tmp;
1009	int i, j;
1010
1011	/* Initialize HDP */
1012	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1013		WREG32((0x2c14 + j), 0x00000000);
1014		WREG32((0x2c18 + j), 0x00000000);
1015		WREG32((0x2c1c + j), 0x00000000);
1016		WREG32((0x2c20 + j), 0x00000000);
1017		WREG32((0x2c24 + j), 0x00000000);
1018	}
1019	/* r7xx hw bug.  Read from HDP_DEBUG1 rather
1020	 * than writing to HDP_REG_COHERENCY_FLUSH_CNTL
1021	 */
1022	tmp = RREG32(HDP_DEBUG1);
1023
1024	rv515_mc_stop(rdev, &save);
1025	if (r600_mc_wait_for_idle(rdev)) {
1026		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1027	}
1028	/* Lockout access through VGA aperture*/
1029	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1030	/* Update configuration */
1031	if (rdev->flags & RADEON_IS_AGP) {
1032		if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1033			/* VRAM before AGP */
1034			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1035				rdev->mc.vram_start >> 12);
1036			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1037				rdev->mc.gtt_end >> 12);
1038		} else {
1039			/* VRAM after AGP */
1040			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1041				rdev->mc.gtt_start >> 12);
1042			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1043				rdev->mc.vram_end >> 12);
1044		}
1045	} else {
1046		WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1047			rdev->mc.vram_start >> 12);
1048		WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1049			rdev->mc.vram_end >> 12);
1050	}
1051	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1052	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1053	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1054	WREG32(MC_VM_FB_LOCATION, tmp);
1055	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1056	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1057	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1058	if (rdev->flags & RADEON_IS_AGP) {
1059		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1060		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1061		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1062	} else {
1063		WREG32(MC_VM_AGP_BASE, 0);
1064		WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1065		WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1066	}
1067	if (r600_mc_wait_for_idle(rdev)) {
1068		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1069	}
1070	rv515_mc_resume(rdev, &save);
1071	/* we need to own VRAM, so turn off the VGA renderer here
1072	 * to stop it overwriting our objects */
1073	rv515_vga_render_disable(rdev);
1074}
1075
1076
1077/*
1078 * CP.
1079 */
1080void r700_cp_stop(struct radeon_device *rdev)
1081{
1082	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1083		radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1084	WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1085	WREG32(SCRATCH_UMSK, 0);
1086	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1087}
1088
1089static int rv770_cp_load_microcode(struct radeon_device *rdev)
1090{
1091	const __be32 *fw_data;
1092	int i;
1093
1094	if (!rdev->me_fw || !rdev->pfp_fw)
1095		return -EINVAL;
1096
1097	r700_cp_stop(rdev);
1098	WREG32(CP_RB_CNTL,
1099#ifdef __BIG_ENDIAN
1100	       BUF_SWAP_32BIT |
1101#endif
1102	       RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1103
1104	/* Reset cp */
1105	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1106	RREG32(GRBM_SOFT_RESET);
1107	mdelay(15);
1108	WREG32(GRBM_SOFT_RESET, 0);
1109
1110	fw_data = (const __be32 *)rdev->pfp_fw->data;
1111	WREG32(CP_PFP_UCODE_ADDR, 0);
1112	for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
1113		WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1114	WREG32(CP_PFP_UCODE_ADDR, 0);
1115
1116	fw_data = (const __be32 *)rdev->me_fw->data;
1117	WREG32(CP_ME_RAM_WADDR, 0);
1118	for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
1119		WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1120
1121	WREG32(CP_PFP_UCODE_ADDR, 0);
1122	WREG32(CP_ME_RAM_WADDR, 0);
1123	WREG32(CP_ME_RAM_RADDR, 0);
1124	return 0;
1125}
1126
1127void r700_cp_fini(struct radeon_device *rdev)
1128{
1129	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1130	r700_cp_stop(rdev);
1131	radeon_ring_fini(rdev, ring);
1132	radeon_scratch_free(rdev, ring->rptr_save_reg);
1133}
1134
1135void rv770_set_clk_bypass_mode(struct radeon_device *rdev)
1136{
1137	u32 tmp, i;
1138
1139	if (rdev->flags & RADEON_IS_IGP)
1140		return;
1141
1142	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
1143	tmp &= SCLK_MUX_SEL_MASK;
1144	tmp |= SCLK_MUX_SEL(1) | SCLK_MUX_UPDATE;
1145	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1146
1147	for (i = 0; i < rdev->usec_timeout; i++) {
1148		if (RREG32(CG_SPLL_STATUS) & SPLL_CHG_STATUS)
1149			break;
1150		udelay(1);
1151	}
1152
1153	tmp &= ~SCLK_MUX_UPDATE;
1154	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1155
1156	tmp = RREG32(MPLL_CNTL_MODE);
1157	if ((rdev->family == CHIP_RV710) || (rdev->family == CHIP_RV730))
1158		tmp &= ~RV730_MPLL_MCLK_SEL;
1159	else
1160		tmp &= ~MPLL_MCLK_SEL;
1161	WREG32(MPLL_CNTL_MODE, tmp);
1162}
1163
1164/*
1165 * Core functions
1166 */
1167static void rv770_gpu_init(struct radeon_device *rdev)
1168{
1169	int i, j, num_qd_pipes;
1170	u32 ta_aux_cntl;
1171	u32 sx_debug_1;
1172	u32 smx_dc_ctl0;
1173	u32 db_debug3;
1174	u32 num_gs_verts_per_thread;
1175	u32 vgt_gs_per_es;
1176	u32 gs_prim_buffer_depth = 0;
1177	u32 sq_ms_fifo_sizes;
1178	u32 sq_config;
1179	u32 sq_thread_resource_mgmt;
1180	u32 hdp_host_path_cntl;
1181	u32 sq_dyn_gpr_size_simd_ab_0;
1182	u32 gb_tiling_config = 0;
1183	u32 cc_gc_shader_pipe_config = 0;
1184	u32 mc_arb_ramcfg;
1185	u32 db_debug4, tmp;
1186	u32 inactive_pipes, shader_pipe_config;
1187	u32 disabled_rb_mask;
1188	unsigned active_number;
1189
1190	/* setup chip specs */
1191	rdev->config.rv770.tiling_group_size = 256;
1192	switch (rdev->family) {
1193	case CHIP_RV770:
1194		rdev->config.rv770.max_pipes = 4;
1195		rdev->config.rv770.max_tile_pipes = 8;
1196		rdev->config.rv770.max_simds = 10;
1197		rdev->config.rv770.max_backends = 4;
1198		rdev->config.rv770.max_gprs = 256;
1199		rdev->config.rv770.max_threads = 248;
1200		rdev->config.rv770.max_stack_entries = 512;
1201		rdev->config.rv770.max_hw_contexts = 8;
1202		rdev->config.rv770.max_gs_threads = 16 * 2;
1203		rdev->config.rv770.sx_max_export_size = 128;
1204		rdev->config.rv770.sx_max_export_pos_size = 16;
1205		rdev->config.rv770.sx_max_export_smx_size = 112;
1206		rdev->config.rv770.sq_num_cf_insts = 2;
1207
1208		rdev->config.rv770.sx_num_of_sets = 7;
1209		rdev->config.rv770.sc_prim_fifo_size = 0xF9;
1210		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1211		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1212		break;
1213	case CHIP_RV730:
1214		rdev->config.rv770.max_pipes = 2;
1215		rdev->config.rv770.max_tile_pipes = 4;
1216		rdev->config.rv770.max_simds = 8;
1217		rdev->config.rv770.max_backends = 2;
1218		rdev->config.rv770.max_gprs = 128;
1219		rdev->config.rv770.max_threads = 248;
1220		rdev->config.rv770.max_stack_entries = 256;
1221		rdev->config.rv770.max_hw_contexts = 8;
1222		rdev->config.rv770.max_gs_threads = 16 * 2;
1223		rdev->config.rv770.sx_max_export_size = 256;
1224		rdev->config.rv770.sx_max_export_pos_size = 32;
1225		rdev->config.rv770.sx_max_export_smx_size = 224;
1226		rdev->config.rv770.sq_num_cf_insts = 2;
1227
1228		rdev->config.rv770.sx_num_of_sets = 7;
1229		rdev->config.rv770.sc_prim_fifo_size = 0xf9;
1230		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1231		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1232		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1233			rdev->config.rv770.sx_max_export_pos_size -= 16;
1234			rdev->config.rv770.sx_max_export_smx_size += 16;
1235		}
1236		break;
1237	case CHIP_RV710:
1238		rdev->config.rv770.max_pipes = 2;
1239		rdev->config.rv770.max_tile_pipes = 2;
1240		rdev->config.rv770.max_simds = 2;
1241		rdev->config.rv770.max_backends = 1;
1242		rdev->config.rv770.max_gprs = 256;
1243		rdev->config.rv770.max_threads = 192;
1244		rdev->config.rv770.max_stack_entries = 256;
1245		rdev->config.rv770.max_hw_contexts = 4;
1246		rdev->config.rv770.max_gs_threads = 8 * 2;
1247		rdev->config.rv770.sx_max_export_size = 128;
1248		rdev->config.rv770.sx_max_export_pos_size = 16;
1249		rdev->config.rv770.sx_max_export_smx_size = 112;
1250		rdev->config.rv770.sq_num_cf_insts = 1;
1251
1252		rdev->config.rv770.sx_num_of_sets = 7;
1253		rdev->config.rv770.sc_prim_fifo_size = 0x40;
1254		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1255		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1256		break;
1257	case CHIP_RV740:
1258		rdev->config.rv770.max_pipes = 4;
1259		rdev->config.rv770.max_tile_pipes = 4;
1260		rdev->config.rv770.max_simds = 8;
1261		rdev->config.rv770.max_backends = 4;
1262		rdev->config.rv770.max_gprs = 256;
1263		rdev->config.rv770.max_threads = 248;
1264		rdev->config.rv770.max_stack_entries = 512;
1265		rdev->config.rv770.max_hw_contexts = 8;
1266		rdev->config.rv770.max_gs_threads = 16 * 2;
1267		rdev->config.rv770.sx_max_export_size = 256;
1268		rdev->config.rv770.sx_max_export_pos_size = 32;
1269		rdev->config.rv770.sx_max_export_smx_size = 224;
1270		rdev->config.rv770.sq_num_cf_insts = 2;
1271
1272		rdev->config.rv770.sx_num_of_sets = 7;
1273		rdev->config.rv770.sc_prim_fifo_size = 0x100;
1274		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1275		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1276
1277		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1278			rdev->config.rv770.sx_max_export_pos_size -= 16;
1279			rdev->config.rv770.sx_max_export_smx_size += 16;
1280		}
1281		break;
1282	default:
1283		break;
1284	}
1285
1286	/* Initialize HDP */
1287	j = 0;
1288	for (i = 0; i < 32; i++) {
1289		WREG32((0x2c14 + j), 0x00000000);
1290		WREG32((0x2c18 + j), 0x00000000);
1291		WREG32((0x2c1c + j), 0x00000000);
1292		WREG32((0x2c20 + j), 0x00000000);
1293		WREG32((0x2c24 + j), 0x00000000);
1294		j += 0x18;
1295	}
1296
1297	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1298
1299	/* setup tiling, simd, pipe config */
1300	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1301
1302	shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG);
1303	inactive_pipes = (shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> INACTIVE_QD_PIPES_SHIFT;
1304	for (i = 0, tmp = 1, active_number = 0; i < R7XX_MAX_PIPES; i++) {
1305		if (!(inactive_pipes & tmp)) {
1306			active_number++;
1307		}
1308		tmp <<= 1;
1309	}
1310	if (active_number == 1) {
1311		WREG32(SPI_CONFIG_CNTL, DISABLE_INTERP_1);
1312	} else {
1313		WREG32(SPI_CONFIG_CNTL, 0);
1314	}
1315
1316	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1317	tmp = rdev->config.rv770.max_simds -
1318		r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R7XX_MAX_SIMDS_MASK);
1319	rdev->config.rv770.active_simds = tmp;
1320
1321	switch (rdev->config.rv770.max_tile_pipes) {
1322	case 1:
1323	default:
1324		gb_tiling_config = PIPE_TILING(0);
1325		break;
1326	case 2:
1327		gb_tiling_config = PIPE_TILING(1);
1328		break;
1329	case 4:
1330		gb_tiling_config = PIPE_TILING(2);
1331		break;
1332	case 8:
1333		gb_tiling_config = PIPE_TILING(3);
1334		break;
1335	}
1336	rdev->config.rv770.tiling_npipes = rdev->config.rv770.max_tile_pipes;
1337
1338	disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R7XX_MAX_BACKENDS_MASK;
1339	tmp = 0;
1340	for (i = 0; i < rdev->config.rv770.max_backends; i++)
1341		tmp |= (1 << i);
1342	/* if all the backends are disabled, fix it up here */
1343	if ((disabled_rb_mask & tmp) == tmp) {
1344		for (i = 0; i < rdev->config.rv770.max_backends; i++)
1345			disabled_rb_mask &= ~(1 << i);
1346	}
1347	tmp = (gb_tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1348	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.rv770.max_backends,
1349					R7XX_MAX_BACKENDS, disabled_rb_mask);
1350	gb_tiling_config |= tmp << 16;
1351	rdev->config.rv770.backend_map = tmp;
1352
1353	if (rdev->family == CHIP_RV770)
1354		gb_tiling_config |= BANK_TILING(1);
1355	else {
1356		if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
1357			gb_tiling_config |= BANK_TILING(1);
1358		else
1359			gb_tiling_config |= BANK_TILING(0);
1360	}
1361	rdev->config.rv770.tiling_nbanks = 4 << ((gb_tiling_config >> 4) & 0x3);
1362	gb_tiling_config |= GROUP_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1363	if (((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT) > 3) {
1364		gb_tiling_config |= ROW_TILING(3);
1365		gb_tiling_config |= SAMPLE_SPLIT(3);
1366	} else {
1367		gb_tiling_config |=
1368			ROW_TILING(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1369		gb_tiling_config |=
1370			SAMPLE_SPLIT(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1371	}
1372
1373	gb_tiling_config |= BANK_SWAPS(1);
1374	rdev->config.rv770.tile_config = gb_tiling_config;
1375
1376	WREG32(GB_TILING_CONFIG, gb_tiling_config);
1377	WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1378	WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1379	WREG32(DMA_TILING_CONFIG, (gb_tiling_config & 0xffff));
1380	WREG32(DMA_TILING_CONFIG2, (gb_tiling_config & 0xffff));
1381	if (rdev->family == CHIP_RV730) {
1382		WREG32(UVD_UDEC_DB_TILING_CONFIG, (gb_tiling_config & 0xffff));
1383		WREG32(UVD_UDEC_DBW_TILING_CONFIG, (gb_tiling_config & 0xffff));
1384		WREG32(UVD_UDEC_TILING_CONFIG, (gb_tiling_config & 0xffff));
1385	}
1386
1387	WREG32(CGTS_SYS_TCC_DISABLE, 0);
1388	WREG32(CGTS_TCC_DISABLE, 0);
1389	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
1390	WREG32(CGTS_USER_TCC_DISABLE, 0);
1391
1392
1393	num_qd_pipes = R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1394	WREG32(VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & DEALLOC_DIST_MASK);
1395	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1396
1397	/* set HW defaults for 3D engine */
1398	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
1399				     ROQ_IB2_START(0x2b)));
1400
1401	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
1402
1403	ta_aux_cntl = RREG32(TA_CNTL_AUX);
1404	WREG32(TA_CNTL_AUX, ta_aux_cntl | DISABLE_CUBE_ANISO);
1405
1406	sx_debug_1 = RREG32(SX_DEBUG_1);
1407	sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1408	WREG32(SX_DEBUG_1, sx_debug_1);
1409
1410	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1411	smx_dc_ctl0 &= ~CACHE_DEPTH(0x1ff);
1412	smx_dc_ctl0 |= CACHE_DEPTH((rdev->config.rv770.sx_num_of_sets * 64) - 1);
1413	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1414
1415	if (rdev->family != CHIP_RV740)
1416		WREG32(SMX_EVENT_CTL, (ES_FLUSH_CTL(4) |
1417				       GS_FLUSH_CTL(4) |
1418				       ACK_FLUSH_CTL(3) |
1419				       SYNC_FLUSH_CTL));
1420
1421	if (rdev->family != CHIP_RV770)
1422		WREG32(SMX_SAR_CTL0, 0x00003f3f);
1423
1424	db_debug3 = RREG32(DB_DEBUG3);
1425	db_debug3 &= ~DB_CLK_OFF_DELAY(0x1f);
1426	switch (rdev->family) {
1427	case CHIP_RV770:
1428	case CHIP_RV740:
1429		db_debug3 |= DB_CLK_OFF_DELAY(0x1f);
1430		break;
1431	case CHIP_RV710:
1432	case CHIP_RV730:
1433	default:
1434		db_debug3 |= DB_CLK_OFF_DELAY(2);
1435		break;
1436	}
1437	WREG32(DB_DEBUG3, db_debug3);
1438
1439	if (rdev->family != CHIP_RV770) {
1440		db_debug4 = RREG32(DB_DEBUG4);
1441		db_debug4 |= DISABLE_TILE_COVERED_FOR_PS_ITER;
1442		WREG32(DB_DEBUG4, db_debug4);
1443	}
1444
1445	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.rv770.sx_max_export_size / 4) - 1) |
1446					POSITION_BUFFER_SIZE((rdev->config.rv770.sx_max_export_pos_size / 4) - 1) |
1447					SMX_BUFFER_SIZE((rdev->config.rv770.sx_max_export_smx_size / 4) - 1)));
1448
1449	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.rv770.sc_prim_fifo_size) |
1450				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_hiz_tile_fifo_size) |
1451				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_earlyz_tile_fifo_fize)));
1452
1453	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1454
1455	WREG32(VGT_NUM_INSTANCES, 1);
1456
1457	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
1458
1459	WREG32(CP_PERFMON_CNTL, 0);
1460
1461	sq_ms_fifo_sizes = (CACHE_FIFO_SIZE(16 * rdev->config.rv770.sq_num_cf_insts) |
1462			    DONE_FIFO_HIWATER(0xe0) |
1463			    ALU_UPDATE_FIFO_HIWATER(0x8));
1464	switch (rdev->family) {
1465	case CHIP_RV770:
1466	case CHIP_RV730:
1467	case CHIP_RV710:
1468		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x1);
1469		break;
1470	case CHIP_RV740:
1471	default:
1472		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x4);
1473		break;
1474	}
1475	WREG32(SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
1476
1477	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1478	 * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1479	 */
1480	sq_config = RREG32(SQ_CONFIG);
1481	sq_config &= ~(PS_PRIO(3) |
1482		       VS_PRIO(3) |
1483		       GS_PRIO(3) |
1484		       ES_PRIO(3));
1485	sq_config |= (DX9_CONSTS |
1486		      VC_ENABLE |
1487		      EXPORT_SRC_C |
1488		      PS_PRIO(0) |
1489		      VS_PRIO(1) |
1490		      GS_PRIO(2) |
1491		      ES_PRIO(3));
1492	if (rdev->family == CHIP_RV710)
1493		/* no vertex cache */
1494		sq_config &= ~VC_ENABLE;
1495
1496	WREG32(SQ_CONFIG, sq_config);
1497
1498	WREG32(SQ_GPR_RESOURCE_MGMT_1,  (NUM_PS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1499					 NUM_VS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1500					 NUM_CLAUSE_TEMP_GPRS(((rdev->config.rv770.max_gprs * 24)/64)/2)));
1501
1502	WREG32(SQ_GPR_RESOURCE_MGMT_2,  (NUM_GS_GPRS((rdev->config.rv770.max_gprs * 7)/64) |
1503					 NUM_ES_GPRS((rdev->config.rv770.max_gprs * 7)/64)));
1504
1505	sq_thread_resource_mgmt = (NUM_PS_THREADS((rdev->config.rv770.max_threads * 4)/8) |
1506				   NUM_VS_THREADS((rdev->config.rv770.max_threads * 2)/8) |
1507				   NUM_ES_THREADS((rdev->config.rv770.max_threads * 1)/8));
1508	if (((rdev->config.rv770.max_threads * 1) / 8) > rdev->config.rv770.max_gs_threads)
1509		sq_thread_resource_mgmt |= NUM_GS_THREADS(rdev->config.rv770.max_gs_threads);
1510	else
1511		sq_thread_resource_mgmt |= NUM_GS_THREADS((rdev->config.rv770.max_gs_threads * 1)/8);
1512	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1513
1514	WREG32(SQ_STACK_RESOURCE_MGMT_1, (NUM_PS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1515						     NUM_VS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1516
1517	WREG32(SQ_STACK_RESOURCE_MGMT_2, (NUM_GS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1518						     NUM_ES_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1519
1520	sq_dyn_gpr_size_simd_ab_0 = (SIMDA_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1521				     SIMDA_RING1((rdev->config.rv770.max_gprs * 38)/64) |
1522				     SIMDB_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1523				     SIMDB_RING1((rdev->config.rv770.max_gprs * 38)/64));
1524
1525	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
1526	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
1527	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
1528	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
1529	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
1530	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
1531	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
1532	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
1533
1534	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1535					  FORCE_EOV_MAX_REZ_CNT(255)));
1536
1537	if (rdev->family == CHIP_RV710)
1538		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(TC_ONLY) |
1539						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1540	else
1541		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(VC_AND_TC) |
1542						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1543
1544	switch (rdev->family) {
1545	case CHIP_RV770:
1546	case CHIP_RV730:
1547	case CHIP_RV740:
1548		gs_prim_buffer_depth = 384;
1549		break;
1550	case CHIP_RV710:
1551		gs_prim_buffer_depth = 128;
1552		break;
1553	default:
1554		break;
1555	}
1556
1557	num_gs_verts_per_thread = rdev->config.rv770.max_pipes * 16;
1558	vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1559	/* Max value for this is 256 */
1560	if (vgt_gs_per_es > 256)
1561		vgt_gs_per_es = 256;
1562
1563	WREG32(VGT_ES_PER_GS, 128);
1564	WREG32(VGT_GS_PER_ES, vgt_gs_per_es);
1565	WREG32(VGT_GS_PER_VS, 2);
1566
1567	/* more default values. 2D/3D driver should adjust as needed */
1568	WREG32(VGT_GS_VERTEX_REUSE, 16);
1569	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1570	WREG32(VGT_STRMOUT_EN, 0);
1571	WREG32(SX_MISC, 0);
1572	WREG32(PA_SC_MODE_CNTL, 0);
1573	WREG32(PA_SC_EDGERULE, 0xaaaaaaaa);
1574	WREG32(PA_SC_AA_CONFIG, 0);
1575	WREG32(PA_SC_CLIPRECT_RULE, 0xffff);
1576	WREG32(PA_SC_LINE_STIPPLE, 0);
1577	WREG32(SPI_INPUT_Z, 0);
1578	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1579	WREG32(CB_COLOR7_FRAG, 0);
1580
1581	/* clear render buffer base addresses */
1582	WREG32(CB_COLOR0_BASE, 0);
1583	WREG32(CB_COLOR1_BASE, 0);
1584	WREG32(CB_COLOR2_BASE, 0);
1585	WREG32(CB_COLOR3_BASE, 0);
1586	WREG32(CB_COLOR4_BASE, 0);
1587	WREG32(CB_COLOR5_BASE, 0);
1588	WREG32(CB_COLOR6_BASE, 0);
1589	WREG32(CB_COLOR7_BASE, 0);
1590
1591	WREG32(TCP_CNTL, 0);
1592
1593	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1594	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1595
1596	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1597
1598	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1599					  NUM_CLIP_SEQ(3)));
1600	WREG32(VC_ENHANCE, 0);
1601}
1602
1603void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1604{
1605	u64 size_bf, size_af;
1606
1607	if (mc->mc_vram_size > 0xE0000000) {
1608		/* leave room for at least 512M GTT */
1609		dev_warn(rdev->dev, "limiting VRAM\n");
1610		mc->real_vram_size = 0xE0000000;
1611		mc->mc_vram_size = 0xE0000000;
1612	}
1613	if (rdev->flags & RADEON_IS_AGP) {
1614		size_bf = mc->gtt_start;
1615		size_af = mc->mc_mask - mc->gtt_end;
1616		if (size_bf > size_af) {
1617			if (mc->mc_vram_size > size_bf) {
1618				dev_warn(rdev->dev, "limiting VRAM\n");
1619				mc->real_vram_size = size_bf;
1620				mc->mc_vram_size = size_bf;
1621			}
1622			mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1623		} else {
1624			if (mc->mc_vram_size > size_af) {
1625				dev_warn(rdev->dev, "limiting VRAM\n");
1626				mc->real_vram_size = size_af;
1627				mc->mc_vram_size = size_af;
1628			}
1629			mc->vram_start = mc->gtt_end + 1;
1630		}
1631		mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1632		dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1633				mc->mc_vram_size >> 20, mc->vram_start,
1634				mc->vram_end, mc->real_vram_size >> 20);
1635	} else {
1636		radeon_vram_location(rdev, &rdev->mc, 0);
1637		rdev->mc.gtt_base_align = 0;
1638		radeon_gtt_location(rdev, mc);
1639	}
1640}
1641
1642static int rv770_mc_init(struct radeon_device *rdev)
1643{
1644	u32 tmp;
1645	int chansize, numchan;
1646
1647	/* Get VRAM informations */
1648	rdev->mc.vram_is_ddr = true;
1649	tmp = RREG32(MC_ARB_RAMCFG);
1650	if (tmp & CHANSIZE_OVERRIDE) {
1651		chansize = 16;
1652	} else if (tmp & CHANSIZE_MASK) {
1653		chansize = 64;
1654	} else {
1655		chansize = 32;
1656	}
1657	tmp = RREG32(MC_SHARED_CHMAP);
1658	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1659	case 0:
1660	default:
1661		numchan = 1;
1662		break;
1663	case 1:
1664		numchan = 2;
1665		break;
1666	case 2:
1667		numchan = 4;
1668		break;
1669	case 3:
1670		numchan = 8;
1671		break;
1672	}
1673	rdev->mc.vram_width = numchan * chansize;
1674	/* Could aper size report 0 ? */
1675	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1676	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1677	/* Setup GPU memory space */
1678	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1679	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1680	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1681	r700_vram_gtt_location(rdev, &rdev->mc);
1682	radeon_update_bandwidth_info(rdev);
1683
1684	return 0;
1685}
1686
1687static void rv770_uvd_init(struct radeon_device *rdev)
1688{
1689	int r;
1690
1691	if (!rdev->has_uvd)
1692		return;
1693
1694	r = radeon_uvd_init(rdev);
1695	if (r) {
1696		dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
1697		/*
1698		 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
1699		 * to early fails uvd_v2_2_resume() and thus nothing happens
1700		 * there. So it is pointless to try to go through that code
1701		 * hence why we disable uvd here.
1702		 */
1703		rdev->has_uvd = false;
1704		return;
1705	}
1706	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
1707	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
1708}
1709
1710static void rv770_uvd_start(struct radeon_device *rdev)
1711{
1712	int r;
1713
1714	if (!rdev->has_uvd)
1715		return;
1716
1717	r = uvd_v2_2_resume(rdev);
1718	if (r) {
1719		dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
1720		goto error;
1721	}
1722	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
1723	if (r) {
1724		dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
1725		goto error;
1726	}
1727	return;
1728
1729error:
1730	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
1731}
1732
1733static void rv770_uvd_resume(struct radeon_device *rdev)
1734{
1735	struct radeon_ring *ring;
1736	int r;
1737
1738	if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
1739		return;
1740
1741	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
1742	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
1743	if (r) {
1744		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
1745		return;
1746	}
1747	r = uvd_v1_0_init(rdev);
1748	if (r) {
1749		dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
1750		return;
1751	}
1752}
1753
1754static int rv770_startup(struct radeon_device *rdev)
1755{
1756	struct radeon_ring *ring;
1757	int r;
1758
1759	/* enable pcie gen2 link */
1760	rv770_pcie_gen2_enable(rdev);
1761
1762	/* scratch needs to be initialized before MC */
1763	r = r600_vram_scratch_init(rdev);
1764	if (r)
1765		return r;
1766
1767	rv770_mc_program(rdev);
1768
1769	if (rdev->flags & RADEON_IS_AGP) {
1770		rv770_agp_enable(rdev);
1771	} else {
1772		r = rv770_pcie_gart_enable(rdev);
1773		if (r)
1774			return r;
1775	}
1776
1777	rv770_gpu_init(rdev);
1778
1779	/* allocate wb buffer */
1780	r = radeon_wb_init(rdev);
1781	if (r)
1782		return r;
1783
1784	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1785	if (r) {
1786		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1787		return r;
1788	}
1789
1790	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
1791	if (r) {
1792		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1793		return r;
1794	}
1795
1796	rv770_uvd_start(rdev);
1797
1798	/* Enable IRQ */
1799	if (!rdev->irq.installed) {
1800		r = radeon_irq_kms_init(rdev);
1801		if (r)
1802			return r;
1803	}
1804
1805	r = r600_irq_init(rdev);
1806	if (r) {
1807		DRM_ERROR("radeon: IH init failed (%d).\n", r);
1808		radeon_irq_kms_fini(rdev);
1809		return r;
1810	}
1811	r600_irq_set(rdev);
1812
1813	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1814	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1815			     RADEON_CP_PACKET2);
1816	if (r)
1817		return r;
1818
1819	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1820	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
1821			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1822	if (r)
1823		return r;
1824
1825	r = rv770_cp_load_microcode(rdev);
1826	if (r)
1827		return r;
1828	r = r600_cp_resume(rdev);
1829	if (r)
1830		return r;
1831
1832	r = r600_dma_resume(rdev);
1833	if (r)
1834		return r;
1835
1836	rv770_uvd_resume(rdev);
1837
1838	r = radeon_ib_pool_init(rdev);
1839	if (r) {
1840		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1841		return r;
1842	}
1843
1844	r = radeon_audio_init(rdev);
1845	if (r) {
1846		DRM_ERROR("radeon: audio init failed\n");
1847		return r;
1848	}
1849
1850	return 0;
1851}
1852
1853int rv770_resume(struct radeon_device *rdev)
1854{
1855	int r;
1856
1857	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
1858	 * posting will perform necessary task to bring back GPU into good
1859	 * shape.
1860	 */
1861	/* post card */
1862	atom_asic_init(rdev->mode_info.atom_context);
1863
1864	/* init golden registers */
1865	rv770_init_golden_registers(rdev);
1866
1867	if (rdev->pm.pm_method == PM_METHOD_DPM)
1868		radeon_pm_resume(rdev);
1869
1870	rdev->accel_working = true;
1871	r = rv770_startup(rdev);
1872	if (r) {
1873		DRM_ERROR("r600 startup failed on resume\n");
1874		rdev->accel_working = false;
1875		return r;
1876	}
1877
1878	return r;
1879
1880}
1881
1882int rv770_suspend(struct radeon_device *rdev)
1883{
1884	radeon_pm_suspend(rdev);
1885	radeon_audio_fini(rdev);
1886	if (rdev->has_uvd) {
1887		radeon_uvd_suspend(rdev);
1888		uvd_v1_0_fini(rdev);
 
1889	}
1890	r700_cp_stop(rdev);
1891	r600_dma_stop(rdev);
1892	r600_irq_suspend(rdev);
1893	radeon_wb_disable(rdev);
1894	rv770_pcie_gart_disable(rdev);
1895
1896	return 0;
1897}
1898
1899/* Plan is to move initialization in that function and use
1900 * helper function so that radeon_device_init pretty much
1901 * do nothing more than calling asic specific function. This
1902 * should also allow to remove a bunch of callback function
1903 * like vram_info.
1904 */
1905int rv770_init(struct radeon_device *rdev)
1906{
1907	int r;
1908
1909	/* Read BIOS */
1910	if (!radeon_get_bios(rdev)) {
1911		if (ASIC_IS_AVIVO(rdev))
1912			return -EINVAL;
1913	}
1914	/* Must be an ATOMBIOS */
1915	if (!rdev->is_atom_bios) {
1916		dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
1917		return -EINVAL;
1918	}
1919	r = radeon_atombios_init(rdev);
1920	if (r)
1921		return r;
1922	/* Post card if necessary */
1923	if (!radeon_card_posted(rdev)) {
1924		if (!rdev->bios) {
1925			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
1926			return -EINVAL;
1927		}
1928		DRM_INFO("GPU not posted. posting now...\n");
1929		atom_asic_init(rdev->mode_info.atom_context);
1930	}
1931	/* init golden registers */
1932	rv770_init_golden_registers(rdev);
1933	/* Initialize scratch registers */
1934	r600_scratch_init(rdev);
1935	/* Initialize surface registers */
1936	radeon_surface_init(rdev);
1937	/* Initialize clocks */
1938	radeon_get_clock_info(rdev_to_drm(rdev));
1939	/* Fence driver */
1940	radeon_fence_driver_init(rdev);
 
 
1941	/* initialize AGP */
1942	if (rdev->flags & RADEON_IS_AGP) {
1943		r = radeon_agp_init(rdev);
1944		if (r)
1945			radeon_agp_disable(rdev);
1946	}
1947	r = rv770_mc_init(rdev);
1948	if (r)
1949		return r;
1950	/* Memory manager */
1951	r = radeon_bo_init(rdev);
1952	if (r)
1953		return r;
1954
1955	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1956		r = r600_init_microcode(rdev);
1957		if (r) {
1958			DRM_ERROR("Failed to load firmware!\n");
1959			return r;
1960		}
1961	}
1962
1963	/* Initialize power management */
1964	radeon_pm_init(rdev);
1965
1966	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
1967	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1968
1969	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
1970	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
1971
1972	rv770_uvd_init(rdev);
1973
1974	rdev->ih.ring_obj = NULL;
1975	r600_ih_ring_init(rdev, 64 * 1024);
1976
1977	r = r600_pcie_gart_init(rdev);
1978	if (r)
1979		return r;
1980
1981	rdev->accel_working = true;
1982	r = rv770_startup(rdev);
1983	if (r) {
1984		dev_err(rdev->dev, "disabling GPU acceleration\n");
1985		r700_cp_fini(rdev);
1986		r600_dma_fini(rdev);
1987		r600_irq_fini(rdev);
1988		radeon_wb_fini(rdev);
1989		radeon_ib_pool_fini(rdev);
1990		radeon_irq_kms_fini(rdev);
1991		rv770_pcie_gart_fini(rdev);
1992		rdev->accel_working = false;
1993	}
1994
1995	return 0;
1996}
1997
1998void rv770_fini(struct radeon_device *rdev)
1999{
2000	radeon_pm_fini(rdev);
2001	r700_cp_fini(rdev);
2002	r600_dma_fini(rdev);
2003	r600_irq_fini(rdev);
2004	radeon_wb_fini(rdev);
2005	radeon_ib_pool_fini(rdev);
2006	radeon_irq_kms_fini(rdev);
2007	uvd_v1_0_fini(rdev);
2008	radeon_uvd_fini(rdev);
2009	rv770_pcie_gart_fini(rdev);
2010	r600_vram_scratch_fini(rdev);
2011	radeon_gem_fini(rdev);
2012	radeon_fence_driver_fini(rdev);
2013	radeon_agp_fini(rdev);
2014	radeon_bo_fini(rdev);
2015	radeon_atombios_fini(rdev);
2016	kfree(rdev->bios);
2017	rdev->bios = NULL;
2018}
2019
2020static void rv770_pcie_gen2_enable(struct radeon_device *rdev)
2021{
2022	u32 link_width_cntl, lanes, speed_cntl, tmp;
2023	u16 link_cntl2;
2024
2025	if (radeon_pcie_gen2 == 0)
2026		return;
2027
2028	if (rdev->flags & RADEON_IS_IGP)
2029		return;
2030
2031	if (!(rdev->flags & RADEON_IS_PCIE))
2032		return;
2033
2034	/* x2 cards have a special sequence */
2035	if (ASIC_IS_X2(rdev))
2036		return;
2037
2038	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
2039		(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
2040		return;
2041
2042	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
2043
2044	/* advertise upconfig capability */
2045	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2046	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2047	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2048	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2049	if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
2050		lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
2051		link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
2052				     LC_RECONFIG_ARC_MISSING_ESCAPE);
2053		link_width_cntl |= lanes | LC_RECONFIG_NOW |
2054			LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT;
2055		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2056	} else {
2057		link_width_cntl |= LC_UPCONFIGURE_DIS;
2058		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2059	}
2060
2061	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2062	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
2063	    (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
2064
2065		tmp = RREG32(0x541c);
2066		WREG32(0x541c, tmp | 0x8);
2067		WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
2068		link_cntl2 = RREG16(0x4088);
2069		link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
2070		link_cntl2 |= 0x2;
2071		WREG16(0x4088, link_cntl2);
2072		WREG32(MM_CFGREGS_CNTL, 0);
2073
2074		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2075		speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
2076		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2077
2078		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2079		speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
2080		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2081
2082		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2083		speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
2084		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2085
2086		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2087		speed_cntl |= LC_GEN2_EN_STRAP;
2088		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2089
2090	} else {
2091		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2092		/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
2093		if (1)
2094			link_width_cntl |= LC_UPCONFIGURE_DIS;
2095		else
2096			link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2097		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2098	}
2099}