1/*
   2 * Copyright 2008 Advanced Micro Devices, Inc.
   3 * Copyright 2008 Red Hat Inc.
   4 * Copyright 2009 Jerome Glisse.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 *
  24 * Authors: Dave Airlie
  25 *          Alex Deucher
  26 *          Jerome Glisse
  27 */
  28
  29#include <linux/firmware.h>
  30#include <linux/pci.h>
  31#include <linux/slab.h>
  32
  33#include <drm/drm_device.h>
  34#include <drm/radeon_drm.h>
  35#include <drm/drm_fourcc.h>
  36#include <drm/drm_framebuffer.h>
  37
  38#include "atom.h"
  39#include "avivod.h"
  40#include "radeon.h"
  41#include "radeon_asic.h"
  42#include "radeon_audio.h"
  43#include "rv770d.h"
  44#include "rv770.h"
  45
  46#define R700_PFP_UCODE_SIZE 848
  47#define R700_PM4_UCODE_SIZE 1360
  48
  49static void rv770_gpu_init(struct radeon_device *rdev);
  50void rv770_fini(struct radeon_device *rdev);
  51static void rv770_pcie_gen2_enable(struct radeon_device *rdev);
  52int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk);
  53
  54int rv770_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
  55{
  56	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
  57	int r;
  58
  59	/* RV740 uses evergreen uvd clk programming */
  60	if (rdev->family == CHIP_RV740)
  61		return evergreen_set_uvd_clocks(rdev, vclk, dclk);
  62
  63	/* bypass vclk and dclk with bclk */
  64	WREG32_P(CG_UPLL_FUNC_CNTL_2,
  65		 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
  66		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
  67
  68	if (!vclk || !dclk) {
  69		/* keep the Bypass mode, put PLL to sleep */
  70		WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
  71		return 0;
  72	}
  73
  74	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000,
  75					  43663, 0x03FFFFFE, 1, 30, ~0,
  76					  &fb_div, &vclk_div, &dclk_div);
  77	if (r)
  78		return r;
  79
  80	fb_div |= 1;
  81	vclk_div -= 1;
  82	dclk_div -= 1;
  83
  84	/* set UPLL_FB_DIV to 0x50000 */
  85	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(0x50000), ~UPLL_FB_DIV_MASK);
  86
  87	/* deassert UPLL_RESET and UPLL_SLEEP */
  88	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~(UPLL_RESET_MASK | UPLL_SLEEP_MASK));
  89
  90	/* assert BYPASS EN and FB_DIV[0] <- ??? why? */
  91	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
  92	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(1), ~UPLL_FB_DIV(1));
  93
  94	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
  95	if (r)
  96		return r;
  97
  98	/* assert PLL_RESET */
  99	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
 100
 101	/* set the required FB_DIV, REF_DIV, Post divder values */
 102	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_REF_DIV(1), ~UPLL_REF_DIV_MASK);
 103	WREG32_P(CG_UPLL_FUNC_CNTL_2,
 104		 UPLL_SW_HILEN(vclk_div >> 1) |
 105		 UPLL_SW_LOLEN((vclk_div >> 1) + (vclk_div & 1)) |
 106		 UPLL_SW_HILEN2(dclk_div >> 1) |
 107		 UPLL_SW_LOLEN2((dclk_div >> 1) + (dclk_div & 1)),
 108		 ~UPLL_SW_MASK);
 109
 110	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div),
 111		 ~UPLL_FB_DIV_MASK);
 112
 113	/* give the PLL some time to settle */
 114	mdelay(15);
 115
 116	/* deassert PLL_RESET */
 117	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
 118
 119	mdelay(15);
 120
 121	/* deassert BYPASS EN and FB_DIV[0] <- ??? why? */
 122	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
 123	WREG32_P(CG_UPLL_FUNC_CNTL_3, 0, ~UPLL_FB_DIV(1));
 124
 125	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
 126	if (r)
 127		return r;
 128
 129	/* switch VCLK and DCLK selection */
 130	WREG32_P(CG_UPLL_FUNC_CNTL_2,
 131		 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
 132		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
 133
 134	mdelay(100);
 135
 136	return 0;
 137}
 138
 139static const u32 r7xx_golden_registers[] =
 140{
 141	0x8d00, 0xffffffff, 0x0e0e0074,
 142	0x8d04, 0xffffffff, 0x013a2b34,
 143	0x9508, 0xffffffff, 0x00000002,
 144	0x8b20, 0xffffffff, 0,
 145	0x88c4, 0xffffffff, 0x000000c2,
 146	0x28350, 0xffffffff, 0,
 147	0x9058, 0xffffffff, 0x0fffc40f,
 148	0x240c, 0xffffffff, 0x00000380,
 149	0x733c, 0xffffffff, 0x00000002,
 150	0x2650, 0x00040000, 0,
 151	0x20bc, 0x00040000, 0,
 152	0x7300, 0xffffffff, 0x001000f0
 153};
 154
 155static const u32 r7xx_golden_dyn_gpr_registers[] =
 156{
 157	0x8db0, 0xffffffff, 0x98989898,
 158	0x8db4, 0xffffffff, 0x98989898,
 159	0x8db8, 0xffffffff, 0x98989898,
 160	0x8dbc, 0xffffffff, 0x98989898,
 161	0x8dc0, 0xffffffff, 0x98989898,
 162	0x8dc4, 0xffffffff, 0x98989898,
 163	0x8dc8, 0xffffffff, 0x98989898,
 164	0x8dcc, 0xffffffff, 0x98989898,
 165	0x88c4, 0xffffffff, 0x00000082
 166};
 167
 168static const u32 rv770_golden_registers[] =
 169{
 170	0x562c, 0xffffffff, 0,
 171	0x3f90, 0xffffffff, 0,
 172	0x9148, 0xffffffff, 0,
 173	0x3f94, 0xffffffff, 0,
 174	0x914c, 0xffffffff, 0,
 175	0x9698, 0x18000000, 0x18000000
 176};
 177
 178static const u32 rv770ce_golden_registers[] =
 179{
 180	0x562c, 0xffffffff, 0,
 181	0x3f90, 0xffffffff, 0x00cc0000,
 182	0x9148, 0xffffffff, 0x00cc0000,
 183	0x3f94, 0xffffffff, 0x00cc0000,
 184	0x914c, 0xffffffff, 0x00cc0000,
 185	0x9b7c, 0xffffffff, 0x00fa0000,
 186	0x3f8c, 0xffffffff, 0x00fa0000,
 187	0x9698, 0x18000000, 0x18000000
 188};
 189
 190static const u32 rv770_mgcg_init[] =
 191{
 192	0x8bcc, 0xffffffff, 0x130300f9,
 193	0x5448, 0xffffffff, 0x100,
 194	0x55e4, 0xffffffff, 0x100,
 195	0x160c, 0xffffffff, 0x100,
 196	0x5644, 0xffffffff, 0x100,
 197	0xc164, 0xffffffff, 0x100,
 198	0x8a18, 0xffffffff, 0x100,
 199	0x897c, 0xffffffff, 0x8000100,
 200	0x8b28, 0xffffffff, 0x3c000100,
 201	0x9144, 0xffffffff, 0x100,
 202	0x9a1c, 0xffffffff, 0x10000,
 203	0x9a50, 0xffffffff, 0x100,
 204	0x9a1c, 0xffffffff, 0x10001,
 205	0x9a50, 0xffffffff, 0x100,
 206	0x9a1c, 0xffffffff, 0x10002,
 207	0x9a50, 0xffffffff, 0x100,
 208	0x9a1c, 0xffffffff, 0x10003,
 209	0x9a50, 0xffffffff, 0x100,
 210	0x9a1c, 0xffffffff, 0x0,
 211	0x9870, 0xffffffff, 0x100,
 212	0x8d58, 0xffffffff, 0x100,
 213	0x9500, 0xffffffff, 0x0,
 214	0x9510, 0xffffffff, 0x100,
 215	0x9500, 0xffffffff, 0x1,
 216	0x9510, 0xffffffff, 0x100,
 217	0x9500, 0xffffffff, 0x2,
 218	0x9510, 0xffffffff, 0x100,
 219	0x9500, 0xffffffff, 0x3,
 220	0x9510, 0xffffffff, 0x100,
 221	0x9500, 0xffffffff, 0x4,
 222	0x9510, 0xffffffff, 0x100,
 223	0x9500, 0xffffffff, 0x5,
 224	0x9510, 0xffffffff, 0x100,
 225	0x9500, 0xffffffff, 0x6,
 226	0x9510, 0xffffffff, 0x100,
 227	0x9500, 0xffffffff, 0x7,
 228	0x9510, 0xffffffff, 0x100,
 229	0x9500, 0xffffffff, 0x8,
 230	0x9510, 0xffffffff, 0x100,
 231	0x9500, 0xffffffff, 0x9,
 232	0x9510, 0xffffffff, 0x100,
 233	0x9500, 0xffffffff, 0x8000,
 234	0x9490, 0xffffffff, 0x0,
 235	0x949c, 0xffffffff, 0x100,
 236	0x9490, 0xffffffff, 0x1,
 237	0x949c, 0xffffffff, 0x100,
 238	0x9490, 0xffffffff, 0x2,
 239	0x949c, 0xffffffff, 0x100,
 240	0x9490, 0xffffffff, 0x3,
 241	0x949c, 0xffffffff, 0x100,
 242	0x9490, 0xffffffff, 0x4,
 243	0x949c, 0xffffffff, 0x100,
 244	0x9490, 0xffffffff, 0x5,
 245	0x949c, 0xffffffff, 0x100,
 246	0x9490, 0xffffffff, 0x6,
 247	0x949c, 0xffffffff, 0x100,
 248	0x9490, 0xffffffff, 0x7,
 249	0x949c, 0xffffffff, 0x100,
 250	0x9490, 0xffffffff, 0x8,
 251	0x949c, 0xffffffff, 0x100,
 252	0x9490, 0xffffffff, 0x9,
 253	0x949c, 0xffffffff, 0x100,
 254	0x9490, 0xffffffff, 0x8000,
 255	0x9604, 0xffffffff, 0x0,
 256	0x9654, 0xffffffff, 0x100,
 257	0x9604, 0xffffffff, 0x1,
 258	0x9654, 0xffffffff, 0x100,
 259	0x9604, 0xffffffff, 0x2,
 260	0x9654, 0xffffffff, 0x100,
 261	0x9604, 0xffffffff, 0x3,
 262	0x9654, 0xffffffff, 0x100,
 263	0x9604, 0xffffffff, 0x4,
 264	0x9654, 0xffffffff, 0x100,
 265	0x9604, 0xffffffff, 0x5,
 266	0x9654, 0xffffffff, 0x100,
 267	0x9604, 0xffffffff, 0x6,
 268	0x9654, 0xffffffff, 0x100,
 269	0x9604, 0xffffffff, 0x7,
 270	0x9654, 0xffffffff, 0x100,
 271	0x9604, 0xffffffff, 0x8,
 272	0x9654, 0xffffffff, 0x100,
 273	0x9604, 0xffffffff, 0x9,
 274	0x9654, 0xffffffff, 0x100,
 275	0x9604, 0xffffffff, 0x80000000,
 276	0x9030, 0xffffffff, 0x100,
 277	0x9034, 0xffffffff, 0x100,
 278	0x9038, 0xffffffff, 0x100,
 279	0x903c, 0xffffffff, 0x100,
 280	0x9040, 0xffffffff, 0x100,
 281	0xa200, 0xffffffff, 0x100,
 282	0xa204, 0xffffffff, 0x100,
 283	0xa208, 0xffffffff, 0x100,
 284	0xa20c, 0xffffffff, 0x100,
 285	0x971c, 0xffffffff, 0x100,
 286	0x915c, 0xffffffff, 0x00020001,
 287	0x9160, 0xffffffff, 0x00040003,
 288	0x916c, 0xffffffff, 0x00060005,
 289	0x9170, 0xffffffff, 0x00080007,
 290	0x9174, 0xffffffff, 0x000a0009,
 291	0x9178, 0xffffffff, 0x000c000b,
 292	0x917c, 0xffffffff, 0x000e000d,
 293	0x9180, 0xffffffff, 0x0010000f,
 294	0x918c, 0xffffffff, 0x00120011,
 295	0x9190, 0xffffffff, 0x00140013,
 296	0x9194, 0xffffffff, 0x00020001,
 297	0x9198, 0xffffffff, 0x00040003,
 298	0x919c, 0xffffffff, 0x00060005,
 299	0x91a8, 0xffffffff, 0x00080007,
 300	0x91ac, 0xffffffff, 0x000a0009,
 301	0x91b0, 0xffffffff, 0x000c000b,
 302	0x91b4, 0xffffffff, 0x000e000d,
 303	0x91b8, 0xffffffff, 0x0010000f,
 304	0x91c4, 0xffffffff, 0x00120011,
 305	0x91c8, 0xffffffff, 0x00140013,
 306	0x91cc, 0xffffffff, 0x00020001,
 307	0x91d0, 0xffffffff, 0x00040003,
 308	0x91d4, 0xffffffff, 0x00060005,
 309	0x91e0, 0xffffffff, 0x00080007,
 310	0x91e4, 0xffffffff, 0x000a0009,
 311	0x91e8, 0xffffffff, 0x000c000b,
 312	0x91ec, 0xffffffff, 0x00020001,
 313	0x91f0, 0xffffffff, 0x00040003,
 314	0x91f4, 0xffffffff, 0x00060005,
 315	0x9200, 0xffffffff, 0x00080007,
 316	0x9204, 0xffffffff, 0x000a0009,
 317	0x9208, 0xffffffff, 0x000c000b,
 318	0x920c, 0xffffffff, 0x000e000d,
 319	0x9210, 0xffffffff, 0x0010000f,
 320	0x921c, 0xffffffff, 0x00120011,
 321	0x9220, 0xffffffff, 0x00140013,
 322	0x9224, 0xffffffff, 0x00020001,
 323	0x9228, 0xffffffff, 0x00040003,
 324	0x922c, 0xffffffff, 0x00060005,
 325	0x9238, 0xffffffff, 0x00080007,
 326	0x923c, 0xffffffff, 0x000a0009,
 327	0x9240, 0xffffffff, 0x000c000b,
 328	0x9244, 0xffffffff, 0x000e000d,
 329	0x9248, 0xffffffff, 0x0010000f,
 330	0x9254, 0xffffffff, 0x00120011,
 331	0x9258, 0xffffffff, 0x00140013,
 332	0x925c, 0xffffffff, 0x00020001,
 333	0x9260, 0xffffffff, 0x00040003,
 334	0x9264, 0xffffffff, 0x00060005,
 335	0x9270, 0xffffffff, 0x00080007,
 336	0x9274, 0xffffffff, 0x000a0009,
 337	0x9278, 0xffffffff, 0x000c000b,
 338	0x927c, 0xffffffff, 0x000e000d,
 339	0x9280, 0xffffffff, 0x0010000f,
 340	0x928c, 0xffffffff, 0x00120011,
 341	0x9290, 0xffffffff, 0x00140013,
 342	0x9294, 0xffffffff, 0x00020001,
 343	0x929c, 0xffffffff, 0x00040003,
 344	0x92a0, 0xffffffff, 0x00060005,
 345	0x92a4, 0xffffffff, 0x00080007
 346};
 347
 348static const u32 rv710_golden_registers[] =
 349{
 350	0x3f90, 0x00ff0000, 0x00fc0000,
 351	0x9148, 0x00ff0000, 0x00fc0000,
 352	0x3f94, 0x00ff0000, 0x00fc0000,
 353	0x914c, 0x00ff0000, 0x00fc0000,
 354	0xb4c, 0x00000020, 0x00000020,
 355	0xa180, 0xffffffff, 0x00003f3f
 356};
 357
 358static const u32 rv710_mgcg_init[] =
 359{
 360	0x8bcc, 0xffffffff, 0x13030040,
 361	0x5448, 0xffffffff, 0x100,
 362	0x55e4, 0xffffffff, 0x100,
 363	0x160c, 0xffffffff, 0x100,
 364	0x5644, 0xffffffff, 0x100,
 365	0xc164, 0xffffffff, 0x100,
 366	0x8a18, 0xffffffff, 0x100,
 367	0x897c, 0xffffffff, 0x8000100,
 368	0x8b28, 0xffffffff, 0x3c000100,
 369	0x9144, 0xffffffff, 0x100,
 370	0x9a1c, 0xffffffff, 0x10000,
 371	0x9a50, 0xffffffff, 0x100,
 372	0x9a1c, 0xffffffff, 0x0,
 373	0x9870, 0xffffffff, 0x100,
 374	0x8d58, 0xffffffff, 0x100,
 375	0x9500, 0xffffffff, 0x0,
 376	0x9510, 0xffffffff, 0x100,
 377	0x9500, 0xffffffff, 0x1,
 378	0x9510, 0xffffffff, 0x100,
 379	0x9500, 0xffffffff, 0x8000,
 380	0x9490, 0xffffffff, 0x0,
 381	0x949c, 0xffffffff, 0x100,
 382	0x9490, 0xffffffff, 0x1,
 383	0x949c, 0xffffffff, 0x100,
 384	0x9490, 0xffffffff, 0x8000,
 385	0x9604, 0xffffffff, 0x0,
 386	0x9654, 0xffffffff, 0x100,
 387	0x9604, 0xffffffff, 0x1,
 388	0x9654, 0xffffffff, 0x100,
 389	0x9604, 0xffffffff, 0x80000000,
 390	0x9030, 0xffffffff, 0x100,
 391	0x9034, 0xffffffff, 0x100,
 392	0x9038, 0xffffffff, 0x100,
 393	0x903c, 0xffffffff, 0x100,
 394	0x9040, 0xffffffff, 0x100,
 395	0xa200, 0xffffffff, 0x100,
 396	0xa204, 0xffffffff, 0x100,
 397	0xa208, 0xffffffff, 0x100,
 398	0xa20c, 0xffffffff, 0x100,
 399	0x971c, 0xffffffff, 0x100,
 400	0x915c, 0xffffffff, 0x00020001,
 401	0x9174, 0xffffffff, 0x00000003,
 402	0x9178, 0xffffffff, 0x00050001,
 403	0x917c, 0xffffffff, 0x00030002,
 404	0x918c, 0xffffffff, 0x00000004,
 405	0x9190, 0xffffffff, 0x00070006,
 406	0x9194, 0xffffffff, 0x00050001,
 407	0x9198, 0xffffffff, 0x00030002,
 408	0x91a8, 0xffffffff, 0x00000004,
 409	0x91ac, 0xffffffff, 0x00070006,
 410	0x91e8, 0xffffffff, 0x00000001,
 411	0x9294, 0xffffffff, 0x00000001,
 412	0x929c, 0xffffffff, 0x00000002,
 413	0x92a0, 0xffffffff, 0x00040003,
 414	0x9150, 0xffffffff, 0x4d940000
 415};
 416
 417static const u32 rv730_golden_registers[] =
 418{
 419	0x3f90, 0x00ff0000, 0x00f00000,
 420	0x9148, 0x00ff0000, 0x00f00000,
 421	0x3f94, 0x00ff0000, 0x00f00000,
 422	0x914c, 0x00ff0000, 0x00f00000,
 423	0x900c, 0xffffffff, 0x003b033f,
 424	0xb4c, 0x00000020, 0x00000020,
 425	0xa180, 0xffffffff, 0x00003f3f
 426};
 427
 428static const u32 rv730_mgcg_init[] =
 429{
 430	0x8bcc, 0xffffffff, 0x130300f9,
 431	0x5448, 0xffffffff, 0x100,
 432	0x55e4, 0xffffffff, 0x100,
 433	0x160c, 0xffffffff, 0x100,
 434	0x5644, 0xffffffff, 0x100,
 435	0xc164, 0xffffffff, 0x100,
 436	0x8a18, 0xffffffff, 0x100,
 437	0x897c, 0xffffffff, 0x8000100,
 438	0x8b28, 0xffffffff, 0x3c000100,
 439	0x9144, 0xffffffff, 0x100,
 440	0x9a1c, 0xffffffff, 0x10000,
 441	0x9a50, 0xffffffff, 0x100,
 442	0x9a1c, 0xffffffff, 0x10001,
 443	0x9a50, 0xffffffff, 0x100,
 444	0x9a1c, 0xffffffff, 0x0,
 445	0x9870, 0xffffffff, 0x100,
 446	0x8d58, 0xffffffff, 0x100,
 447	0x9500, 0xffffffff, 0x0,
 448	0x9510, 0xffffffff, 0x100,
 449	0x9500, 0xffffffff, 0x1,
 450	0x9510, 0xffffffff, 0x100,
 451	0x9500, 0xffffffff, 0x2,
 452	0x9510, 0xffffffff, 0x100,
 453	0x9500, 0xffffffff, 0x3,
 454	0x9510, 0xffffffff, 0x100,
 455	0x9500, 0xffffffff, 0x4,
 456	0x9510, 0xffffffff, 0x100,
 457	0x9500, 0xffffffff, 0x5,
 458	0x9510, 0xffffffff, 0x100,
 459	0x9500, 0xffffffff, 0x6,
 460	0x9510, 0xffffffff, 0x100,
 461	0x9500, 0xffffffff, 0x7,
 462	0x9510, 0xffffffff, 0x100,
 463	0x9500, 0xffffffff, 0x8000,
 464	0x9490, 0xffffffff, 0x0,
 465	0x949c, 0xffffffff, 0x100,
 466	0x9490, 0xffffffff, 0x1,
 467	0x949c, 0xffffffff, 0x100,
 468	0x9490, 0xffffffff, 0x2,
 469	0x949c, 0xffffffff, 0x100,
 470	0x9490, 0xffffffff, 0x3,
 471	0x949c, 0xffffffff, 0x100,
 472	0x9490, 0xffffffff, 0x4,
 473	0x949c, 0xffffffff, 0x100,
 474	0x9490, 0xffffffff, 0x5,
 475	0x949c, 0xffffffff, 0x100,
 476	0x9490, 0xffffffff, 0x6,
 477	0x949c, 0xffffffff, 0x100,
 478	0x9490, 0xffffffff, 0x7,
 479	0x949c, 0xffffffff, 0x100,
 480	0x9490, 0xffffffff, 0x8000,
 481	0x9604, 0xffffffff, 0x0,
 482	0x9654, 0xffffffff, 0x100,
 483	0x9604, 0xffffffff, 0x1,
 484	0x9654, 0xffffffff, 0x100,
 485	0x9604, 0xffffffff, 0x2,
 486	0x9654, 0xffffffff, 0x100,
 487	0x9604, 0xffffffff, 0x3,
 488	0x9654, 0xffffffff, 0x100,
 489	0x9604, 0xffffffff, 0x4,
 490	0x9654, 0xffffffff, 0x100,
 491	0x9604, 0xffffffff, 0x5,
 492	0x9654, 0xffffffff, 0x100,
 493	0x9604, 0xffffffff, 0x6,
 494	0x9654, 0xffffffff, 0x100,
 495	0x9604, 0xffffffff, 0x7,
 496	0x9654, 0xffffffff, 0x100,
 497	0x9604, 0xffffffff, 0x80000000,
 498	0x9030, 0xffffffff, 0x100,
 499	0x9034, 0xffffffff, 0x100,
 500	0x9038, 0xffffffff, 0x100,
 501	0x903c, 0xffffffff, 0x100,
 502	0x9040, 0xffffffff, 0x100,
 503	0xa200, 0xffffffff, 0x100,
 504	0xa204, 0xffffffff, 0x100,
 505	0xa208, 0xffffffff, 0x100,
 506	0xa20c, 0xffffffff, 0x100,
 507	0x971c, 0xffffffff, 0x100,
 508	0x915c, 0xffffffff, 0x00020001,
 509	0x916c, 0xffffffff, 0x00040003,
 510	0x9170, 0xffffffff, 0x00000005,
 511	0x9178, 0xffffffff, 0x00050001,
 512	0x917c, 0xffffffff, 0x00030002,
 513	0x918c, 0xffffffff, 0x00000004,
 514	0x9190, 0xffffffff, 0x00070006,
 515	0x9194, 0xffffffff, 0x00050001,
 516	0x9198, 0xffffffff, 0x00030002,
 517	0x91a8, 0xffffffff, 0x00000004,
 518	0x91ac, 0xffffffff, 0x00070006,
 519	0x91b0, 0xffffffff, 0x00050001,
 520	0x91b4, 0xffffffff, 0x00030002,
 521	0x91c4, 0xffffffff, 0x00000004,
 522	0x91c8, 0xffffffff, 0x00070006,
 523	0x91cc, 0xffffffff, 0x00050001,
 524	0x91d0, 0xffffffff, 0x00030002,
 525	0x91e0, 0xffffffff, 0x00000004,
 526	0x91e4, 0xffffffff, 0x00070006,
 527	0x91e8, 0xffffffff, 0x00000001,
 528	0x91ec, 0xffffffff, 0x00050001,
 529	0x91f0, 0xffffffff, 0x00030002,
 530	0x9200, 0xffffffff, 0x00000004,
 531	0x9204, 0xffffffff, 0x00070006,
 532	0x9208, 0xffffffff, 0x00050001,
 533	0x920c, 0xffffffff, 0x00030002,
 534	0x921c, 0xffffffff, 0x00000004,
 535	0x9220, 0xffffffff, 0x00070006,
 536	0x9224, 0xffffffff, 0x00050001,
 537	0x9228, 0xffffffff, 0x00030002,
 538	0x9238, 0xffffffff, 0x00000004,
 539	0x923c, 0xffffffff, 0x00070006,
 540	0x9240, 0xffffffff, 0x00050001,
 541	0x9244, 0xffffffff, 0x00030002,
 542	0x9254, 0xffffffff, 0x00000004,
 543	0x9258, 0xffffffff, 0x00070006,
 544	0x9294, 0xffffffff, 0x00000001,
 545	0x929c, 0xffffffff, 0x00000002,
 546	0x92a0, 0xffffffff, 0x00040003,
 547	0x92a4, 0xffffffff, 0x00000005
 548};
 549
 550static const u32 rv740_golden_registers[] =
 551{
 552	0x88c4, 0xffffffff, 0x00000082,
 553	0x28a50, 0xfffffffc, 0x00000004,
 554	0x2650, 0x00040000, 0,
 555	0x20bc, 0x00040000, 0,
 556	0x733c, 0xffffffff, 0x00000002,
 557	0x7300, 0xffffffff, 0x001000f0,
 558	0x3f90, 0x00ff0000, 0,
 559	0x9148, 0x00ff0000, 0,
 560	0x3f94, 0x00ff0000, 0,
 561	0x914c, 0x00ff0000, 0,
 562	0x240c, 0xffffffff, 0x00000380,
 563	0x8a14, 0x00000007, 0x00000007,
 564	0x8b24, 0xffffffff, 0x00ff0fff,
 565	0x28a4c, 0xffffffff, 0x00004000,
 566	0xa180, 0xffffffff, 0x00003f3f,
 567	0x8d00, 0xffffffff, 0x0e0e003a,
 568	0x8d04, 0xffffffff, 0x013a0e2a,
 569	0x8c00, 0xffffffff, 0xe400000f,
 570	0x8db0, 0xffffffff, 0x98989898,
 571	0x8db4, 0xffffffff, 0x98989898,
 572	0x8db8, 0xffffffff, 0x98989898,
 573	0x8dbc, 0xffffffff, 0x98989898,
 574	0x8dc0, 0xffffffff, 0x98989898,
 575	0x8dc4, 0xffffffff, 0x98989898,
 576	0x8dc8, 0xffffffff, 0x98989898,
 577	0x8dcc, 0xffffffff, 0x98989898,
 578	0x9058, 0xffffffff, 0x0fffc40f,
 579	0x900c, 0xffffffff, 0x003b033f,
 580	0x28350, 0xffffffff, 0,
 581	0x8cf0, 0x1fffffff, 0x08e00420,
 582	0x9508, 0xffffffff, 0x00000002,
 583	0x88c4, 0xffffffff, 0x000000c2,
 584	0x9698, 0x18000000, 0x18000000
 585};
 586
 587static const u32 rv740_mgcg_init[] =
 588{
 589	0x8bcc, 0xffffffff, 0x13030100,
 590	0x5448, 0xffffffff, 0x100,
 591	0x55e4, 0xffffffff, 0x100,
 592	0x160c, 0xffffffff, 0x100,
 593	0x5644, 0xffffffff, 0x100,
 594	0xc164, 0xffffffff, 0x100,
 595	0x8a18, 0xffffffff, 0x100,
 596	0x897c, 0xffffffff, 0x100,
 597	0x8b28, 0xffffffff, 0x100,
 598	0x9144, 0xffffffff, 0x100,
 599	0x9a1c, 0xffffffff, 0x10000,
 600	0x9a50, 0xffffffff, 0x100,
 601	0x9a1c, 0xffffffff, 0x10001,
 602	0x9a50, 0xffffffff, 0x100,
 603	0x9a1c, 0xffffffff, 0x10002,
 604	0x9a50, 0xffffffff, 0x100,
 605	0x9a1c, 0xffffffff, 0x10003,
 606	0x9a50, 0xffffffff, 0x100,
 607	0x9a1c, 0xffffffff, 0x0,
 608	0x9870, 0xffffffff, 0x100,
 609	0x8d58, 0xffffffff, 0x100,
 610	0x9500, 0xffffffff, 0x0,
 611	0x9510, 0xffffffff, 0x100,
 612	0x9500, 0xffffffff, 0x1,
 613	0x9510, 0xffffffff, 0x100,
 614	0x9500, 0xffffffff, 0x2,
 615	0x9510, 0xffffffff, 0x100,
 616	0x9500, 0xffffffff, 0x3,
 617	0x9510, 0xffffffff, 0x100,
 618	0x9500, 0xffffffff, 0x4,
 619	0x9510, 0xffffffff, 0x100,
 620	0x9500, 0xffffffff, 0x5,
 621	0x9510, 0xffffffff, 0x100,
 622	0x9500, 0xffffffff, 0x6,
 623	0x9510, 0xffffffff, 0x100,
 624	0x9500, 0xffffffff, 0x7,
 625	0x9510, 0xffffffff, 0x100,
 626	0x9500, 0xffffffff, 0x8000,
 627	0x9490, 0xffffffff, 0x0,
 628	0x949c, 0xffffffff, 0x100,
 629	0x9490, 0xffffffff, 0x1,
 630	0x949c, 0xffffffff, 0x100,
 631	0x9490, 0xffffffff, 0x2,
 632	0x949c, 0xffffffff, 0x100,
 633	0x9490, 0xffffffff, 0x3,
 634	0x949c, 0xffffffff, 0x100,
 635	0x9490, 0xffffffff, 0x4,
 636	0x949c, 0xffffffff, 0x100,
 637	0x9490, 0xffffffff, 0x5,
 638	0x949c, 0xffffffff, 0x100,
 639	0x9490, 0xffffffff, 0x6,
 640	0x949c, 0xffffffff, 0x100,
 641	0x9490, 0xffffffff, 0x7,
 642	0x949c, 0xffffffff, 0x100,
 643	0x9490, 0xffffffff, 0x8000,
 644	0x9604, 0xffffffff, 0x0,
 645	0x9654, 0xffffffff, 0x100,
 646	0x9604, 0xffffffff, 0x1,
 647	0x9654, 0xffffffff, 0x100,
 648	0x9604, 0xffffffff, 0x2,
 649	0x9654, 0xffffffff, 0x100,
 650	0x9604, 0xffffffff, 0x3,
 651	0x9654, 0xffffffff, 0x100,
 652	0x9604, 0xffffffff, 0x4,
 653	0x9654, 0xffffffff, 0x100,
 654	0x9604, 0xffffffff, 0x5,
 655	0x9654, 0xffffffff, 0x100,
 656	0x9604, 0xffffffff, 0x6,
 657	0x9654, 0xffffffff, 0x100,
 658	0x9604, 0xffffffff, 0x7,
 659	0x9654, 0xffffffff, 0x100,
 660	0x9604, 0xffffffff, 0x80000000,
 661	0x9030, 0xffffffff, 0x100,
 662	0x9034, 0xffffffff, 0x100,
 663	0x9038, 0xffffffff, 0x100,
 664	0x903c, 0xffffffff, 0x100,
 665	0x9040, 0xffffffff, 0x100,
 666	0xa200, 0xffffffff, 0x100,
 667	0xa204, 0xffffffff, 0x100,
 668	0xa208, 0xffffffff, 0x100,
 669	0xa20c, 0xffffffff, 0x100,
 670	0x971c, 0xffffffff, 0x100,
 671	0x915c, 0xffffffff, 0x00020001,
 672	0x9160, 0xffffffff, 0x00040003,
 673	0x916c, 0xffffffff, 0x00060005,
 674	0x9170, 0xffffffff, 0x00080007,
 675	0x9174, 0xffffffff, 0x000a0009,
 676	0x9178, 0xffffffff, 0x000c000b,
 677	0x917c, 0xffffffff, 0x000e000d,
 678	0x9180, 0xffffffff, 0x0010000f,
 679	0x918c, 0xffffffff, 0x00120011,
 680	0x9190, 0xffffffff, 0x00140013,
 681	0x9194, 0xffffffff, 0x00020001,
 682	0x9198, 0xffffffff, 0x00040003,
 683	0x919c, 0xffffffff, 0x00060005,
 684	0x91a8, 0xffffffff, 0x00080007,
 685	0x91ac, 0xffffffff, 0x000a0009,
 686	0x91b0, 0xffffffff, 0x000c000b,
 687	0x91b4, 0xffffffff, 0x000e000d,
 688	0x91b8, 0xffffffff, 0x0010000f,
 689	0x91c4, 0xffffffff, 0x00120011,
 690	0x91c8, 0xffffffff, 0x00140013,
 691	0x91cc, 0xffffffff, 0x00020001,
 692	0x91d0, 0xffffffff, 0x00040003,
 693	0x91d4, 0xffffffff, 0x00060005,
 694	0x91e0, 0xffffffff, 0x00080007,
 695	0x91e4, 0xffffffff, 0x000a0009,
 696	0x91e8, 0xffffffff, 0x000c000b,
 697	0x91ec, 0xffffffff, 0x00020001,
 698	0x91f0, 0xffffffff, 0x00040003,
 699	0x91f4, 0xffffffff, 0x00060005,
 700	0x9200, 0xffffffff, 0x00080007,
 701	0x9204, 0xffffffff, 0x000a0009,
 702	0x9208, 0xffffffff, 0x000c000b,
 703	0x920c, 0xffffffff, 0x000e000d,
 704	0x9210, 0xffffffff, 0x0010000f,
 705	0x921c, 0xffffffff, 0x00120011,
 706	0x9220, 0xffffffff, 0x00140013,
 707	0x9224, 0xffffffff, 0x00020001,
 708	0x9228, 0xffffffff, 0x00040003,
 709	0x922c, 0xffffffff, 0x00060005,
 710	0x9238, 0xffffffff, 0x00080007,
 711	0x923c, 0xffffffff, 0x000a0009,
 712	0x9240, 0xffffffff, 0x000c000b,
 713	0x9244, 0xffffffff, 0x000e000d,
 714	0x9248, 0xffffffff, 0x0010000f,
 715	0x9254, 0xffffffff, 0x00120011,
 716	0x9258, 0xffffffff, 0x00140013,
 717	0x9294, 0xffffffff, 0x00020001,
 718	0x929c, 0xffffffff, 0x00040003,
 719	0x92a0, 0xffffffff, 0x00060005,
 720	0x92a4, 0xffffffff, 0x00080007
 721};
 722
 723static void rv770_init_golden_registers(struct radeon_device *rdev)
 724{
 725	switch (rdev->family) {
 726	case CHIP_RV770:
 727		radeon_program_register_sequence(rdev,
 728						 r7xx_golden_registers,
 729						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
 730		radeon_program_register_sequence(rdev,
 731						 r7xx_golden_dyn_gpr_registers,
 732						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 733		if (rdev->pdev->device == 0x994e)
 734			radeon_program_register_sequence(rdev,
 735							 rv770ce_golden_registers,
 736							 (const u32)ARRAY_SIZE(rv770ce_golden_registers));
 737		else
 738			radeon_program_register_sequence(rdev,
 739							 rv770_golden_registers,
 740							 (const u32)ARRAY_SIZE(rv770_golden_registers));
 741		radeon_program_register_sequence(rdev,
 742						 rv770_mgcg_init,
 743						 (const u32)ARRAY_SIZE(rv770_mgcg_init));
 744		break;
 745	case CHIP_RV730:
 746		radeon_program_register_sequence(rdev,
 747						 r7xx_golden_registers,
 748						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
 749		radeon_program_register_sequence(rdev,
 750						 r7xx_golden_dyn_gpr_registers,
 751						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 752		radeon_program_register_sequence(rdev,
 753						 rv730_golden_registers,
 754						 (const u32)ARRAY_SIZE(rv730_golden_registers));
 755		radeon_program_register_sequence(rdev,
 756						 rv730_mgcg_init,
 757						 (const u32)ARRAY_SIZE(rv730_mgcg_init));
 758		break;
 759	case CHIP_RV710:
 760		radeon_program_register_sequence(rdev,
 761						 r7xx_golden_registers,
 762						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
 763		radeon_program_register_sequence(rdev,
 764						 r7xx_golden_dyn_gpr_registers,
 765						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 766		radeon_program_register_sequence(rdev,
 767						 rv710_golden_registers,
 768						 (const u32)ARRAY_SIZE(rv710_golden_registers));
 769		radeon_program_register_sequence(rdev,
 770						 rv710_mgcg_init,
 771						 (const u32)ARRAY_SIZE(rv710_mgcg_init));
 772		break;
 773	case CHIP_RV740:
 774		radeon_program_register_sequence(rdev,
 775						 rv740_golden_registers,
 776						 (const u32)ARRAY_SIZE(rv740_golden_registers));
 777		radeon_program_register_sequence(rdev,
 778						 rv740_mgcg_init,
 779						 (const u32)ARRAY_SIZE(rv740_mgcg_init));
 780		break;
 781	default:
 782		break;
 783	}
 784}
 785
 786#define PCIE_BUS_CLK                10000
 787#define TCLK                        (PCIE_BUS_CLK / 10)
 788
 789/**
 790 * rv770_get_xclk - get the xclk
 791 *
 792 * @rdev: radeon_device pointer
 793 *
 794 * Returns the reference clock used by the gfx engine
 795 * (r7xx-cayman).
 796 */
 797u32 rv770_get_xclk(struct radeon_device *rdev)
 798{
 799	u32 reference_clock = rdev->clock.spll.reference_freq;
 800	u32 tmp = RREG32(CG_CLKPIN_CNTL);
 801
 802	if (tmp & MUX_TCLK_TO_XCLK)
 803		return TCLK;
 804
 805	if (tmp & XTALIN_DIVIDE)
 806		return reference_clock / 4;
 807
 808	return reference_clock;
 809}
 810
 811void rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, bool async)
 812{
 813	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 814	struct drm_framebuffer *fb = radeon_crtc->base.primary->fb;
 815	u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
 816	int i;
 817
 818	/* Lock the graphics update lock */
 819	tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
 820	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 821
 822	/* flip at hsync for async, default is vsync */
 823	WREG32(AVIVO_D1GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
 824	       async ? AVIVO_D1GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
 825	/* update pitch */
 826	WREG32(AVIVO_D1GRPH_PITCH + radeon_crtc->crtc_offset,
 827	       fb->pitches[0] / fb->format->cpp[0]);
 828	/* update the scanout addresses */
 829	if (radeon_crtc->crtc_id) {
 830		WREG32(D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 831		WREG32(D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 832	} else {
 833		WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 834		WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 835	}
 836	WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 837	       (u32)crtc_base);
 838	WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 839	       (u32)crtc_base);
 840
 841	/* Wait for update_pending to go high. */
 842	for (i = 0; i < rdev->usec_timeout; i++) {
 843		if (RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING)
 844			break;
 845		udelay(1);
 846	}
 847	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
 848
 849	/* Unlock the lock, so double-buffering can take place inside vblank */
 850	tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
 851	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 852}
 853
 854bool rv770_page_flip_pending(struct radeon_device *rdev, int crtc_id)
 855{
 856	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 857
 858	/* Return current update_pending status: */
 859	return !!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) &
 860		AVIVO_D1GRPH_SURFACE_UPDATE_PENDING);
 861}
 862
 863/* get temperature in millidegrees */
 864int rv770_get_temp(struct radeon_device *rdev)
 865{
 866	u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
 867		ASIC_T_SHIFT;
 868	int actual_temp;
 869
 870	if (temp & 0x400)
 871		actual_temp = -256;
 872	else if (temp & 0x200)
 873		actual_temp = 255;
 874	else if (temp & 0x100) {
 875		actual_temp = temp & 0x1ff;
 876		actual_temp |= ~0x1ff;
 877	} else
 878		actual_temp = temp & 0xff;
 879
 880	return (actual_temp * 1000) / 2;
 881}
 882
 883void rv770_pm_misc(struct radeon_device *rdev)
 884{
 885	int req_ps_idx = rdev->pm.requested_power_state_index;
 886	int req_cm_idx = rdev->pm.requested_clock_mode_index;
 887	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
 888	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 889
 890	if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
 891		/* 0xff01 is a flag rather then an actual voltage */
 892		if (voltage->voltage == 0xff01)
 893			return;
 894		if (voltage->voltage != rdev->pm.current_vddc) {
 895			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
 896			rdev->pm.current_vddc = voltage->voltage;
 897			DRM_DEBUG("Setting: v: %d\n", voltage->voltage);
 898		}
 899	}
 900}
 901
 902/*
 903 * GART
 904 */
 905static int rv770_pcie_gart_enable(struct radeon_device *rdev)
 906{
 907	u32 tmp;
 908	int r, i;
 909
 910	if (rdev->gart.robj == NULL) {
 911		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 912		return -EINVAL;
 913	}
 914	r = radeon_gart_table_vram_pin(rdev);
 915	if (r)
 916		return r;
 917	/* Setup L2 cache */
 918	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 919				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 920				EFFECTIVE_L2_QUEUE_SIZE(7));
 921	WREG32(VM_L2_CNTL2, 0);
 922	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 923	/* Setup TLB control */
 924	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 925		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 926		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 927		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 928	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 929	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 930	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 931	if (rdev->family == CHIP_RV740)
 932		WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
 933	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 934	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 935	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 936	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 937	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 938	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 939	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 940	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 941				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 942	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 943			(u32)(rdev->dummy_page.addr >> 12));
 944	for (i = 1; i < 7; i++)
 945		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 946
 947	r600_pcie_gart_tlb_flush(rdev);
 948	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
 949		 (unsigned)(rdev->mc.gtt_size >> 20),
 950		 (unsigned long long)rdev->gart.table_addr);
 951	rdev->gart.ready = true;
 952	return 0;
 953}
 954
 955static void rv770_pcie_gart_disable(struct radeon_device *rdev)
 956{
 957	u32 tmp;
 958	int i;
 959
 960	/* Disable all tables */
 961	for (i = 0; i < 7; i++)
 962		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 963
 964	/* Setup L2 cache */
 965	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 966				EFFECTIVE_L2_QUEUE_SIZE(7));
 967	WREG32(VM_L2_CNTL2, 0);
 968	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 969	/* Setup TLB control */
 970	tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 971	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 972	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 973	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 974	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 975	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 976	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 977	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 978	radeon_gart_table_vram_unpin(rdev);
 979}
 980
 981static void rv770_pcie_gart_fini(struct radeon_device *rdev)
 982{
 983	radeon_gart_fini(rdev);
 984	rv770_pcie_gart_disable(rdev);
 985	radeon_gart_table_vram_free(rdev);
 986}
 987
 988
 989static void rv770_agp_enable(struct radeon_device *rdev)
 990{
 991	u32 tmp;
 992	int i;
 993
 994	/* Setup L2 cache */
 995	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 996				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 997				EFFECTIVE_L2_QUEUE_SIZE(7));
 998	WREG32(VM_L2_CNTL2, 0);
 999	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
1000	/* Setup TLB control */
1001	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1002		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1003		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
1004		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
1005	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
1006	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
1007	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1008	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1009	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1010	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1011	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1012	for (i = 0; i < 7; i++)
1013		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1014}
1015
1016static void rv770_mc_program(struct radeon_device *rdev)
1017{
1018	struct rv515_mc_save save;
1019	u32 tmp;
1020	int i, j;
1021
1022	/* Initialize HDP */
1023	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1024		WREG32((0x2c14 + j), 0x00000000);
1025		WREG32((0x2c18 + j), 0x00000000);
1026		WREG32((0x2c1c + j), 0x00000000);
1027		WREG32((0x2c20 + j), 0x00000000);
1028		WREG32((0x2c24 + j), 0x00000000);
1029	}
1030	/* r7xx hw bug.  Read from HDP_DEBUG1 rather
1031	 * than writing to HDP_REG_COHERENCY_FLUSH_CNTL
1032	 */
1033	tmp = RREG32(HDP_DEBUG1);
1034
1035	rv515_mc_stop(rdev, &save);
1036	if (r600_mc_wait_for_idle(rdev)) {
1037		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1038	}
1039	/* Lockout access through VGA aperture*/
1040	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1041	/* Update configuration */
1042	if (rdev->flags & RADEON_IS_AGP) {
1043		if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1044			/* VRAM before AGP */
1045			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1046				rdev->mc.vram_start >> 12);
1047			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1048				rdev->mc.gtt_end >> 12);
1049		} else {
1050			/* VRAM after AGP */
1051			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1052				rdev->mc.gtt_start >> 12);
1053			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1054				rdev->mc.vram_end >> 12);
1055		}
1056	} else {
1057		WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1058			rdev->mc.vram_start >> 12);
1059		WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1060			rdev->mc.vram_end >> 12);
1061	}
1062	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1063	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1064	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1065	WREG32(MC_VM_FB_LOCATION, tmp);
1066	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1067	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1068	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1069	if (rdev->flags & RADEON_IS_AGP) {
1070		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1071		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1072		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1073	} else {
1074		WREG32(MC_VM_AGP_BASE, 0);
1075		WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1076		WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1077	}
1078	if (r600_mc_wait_for_idle(rdev)) {
1079		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1080	}
1081	rv515_mc_resume(rdev, &save);
1082	/* we need to own VRAM, so turn off the VGA renderer here
1083	 * to stop it overwriting our objects */
1084	rv515_vga_render_disable(rdev);
1085}
1086
1087
1088/*
1089 * CP.
1090 */
1091void r700_cp_stop(struct radeon_device *rdev)
1092{
1093	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1094		radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1095	WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1096	WREG32(SCRATCH_UMSK, 0);
1097	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1098}
1099
1100static int rv770_cp_load_microcode(struct radeon_device *rdev)
1101{
1102	const __be32 *fw_data;
1103	int i;
1104
1105	if (!rdev->me_fw || !rdev->pfp_fw)
1106		return -EINVAL;
1107
1108	r700_cp_stop(rdev);
1109	WREG32(CP_RB_CNTL,
1110#ifdef __BIG_ENDIAN
1111	       BUF_SWAP_32BIT |
1112#endif
1113	       RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1114
1115	/* Reset cp */
1116	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1117	RREG32(GRBM_SOFT_RESET);
1118	mdelay(15);
1119	WREG32(GRBM_SOFT_RESET, 0);
1120
1121	fw_data = (const __be32 *)rdev->pfp_fw->data;
1122	WREG32(CP_PFP_UCODE_ADDR, 0);
1123	for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
1124		WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1125	WREG32(CP_PFP_UCODE_ADDR, 0);
1126
1127	fw_data = (const __be32 *)rdev->me_fw->data;
1128	WREG32(CP_ME_RAM_WADDR, 0);
1129	for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
1130		WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1131
1132	WREG32(CP_PFP_UCODE_ADDR, 0);
1133	WREG32(CP_ME_RAM_WADDR, 0);
1134	WREG32(CP_ME_RAM_RADDR, 0);
1135	return 0;
1136}
1137
1138void r700_cp_fini(struct radeon_device *rdev)
1139{
1140	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1141	r700_cp_stop(rdev);
1142	radeon_ring_fini(rdev, ring);
1143	radeon_scratch_free(rdev, ring->rptr_save_reg);
1144}
1145
1146void rv770_set_clk_bypass_mode(struct radeon_device *rdev)
1147{
1148	u32 tmp, i;
1149
1150	if (rdev->flags & RADEON_IS_IGP)
1151		return;
1152
1153	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
1154	tmp &= SCLK_MUX_SEL_MASK;
1155	tmp |= SCLK_MUX_SEL(1) | SCLK_MUX_UPDATE;
1156	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1157
1158	for (i = 0; i < rdev->usec_timeout; i++) {
1159		if (RREG32(CG_SPLL_STATUS) & SPLL_CHG_STATUS)
1160			break;
1161		udelay(1);
1162	}
1163
1164	tmp &= ~SCLK_MUX_UPDATE;
1165	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1166
1167	tmp = RREG32(MPLL_CNTL_MODE);
1168	if ((rdev->family == CHIP_RV710) || (rdev->family == CHIP_RV730))
1169		tmp &= ~RV730_MPLL_MCLK_SEL;
1170	else
1171		tmp &= ~MPLL_MCLK_SEL;
1172	WREG32(MPLL_CNTL_MODE, tmp);
1173}
1174
1175/*
1176 * Core functions
1177 */
1178static void rv770_gpu_init(struct radeon_device *rdev)
1179{
1180	int i, j, num_qd_pipes;
1181	u32 ta_aux_cntl;
1182	u32 sx_debug_1;
1183	u32 smx_dc_ctl0;
1184	u32 db_debug3;
1185	u32 num_gs_verts_per_thread;
1186	u32 vgt_gs_per_es;
1187	u32 gs_prim_buffer_depth = 0;
1188	u32 sq_ms_fifo_sizes;
1189	u32 sq_config;
1190	u32 sq_thread_resource_mgmt;
1191	u32 hdp_host_path_cntl;
1192	u32 sq_dyn_gpr_size_simd_ab_0;
1193	u32 gb_tiling_config = 0;
1194	u32 cc_gc_shader_pipe_config = 0;
1195	u32 mc_arb_ramcfg;
1196	u32 db_debug4, tmp;
1197	u32 inactive_pipes, shader_pipe_config;
1198	u32 disabled_rb_mask;
1199	unsigned active_number;
1200
1201	/* setup chip specs */
1202	rdev->config.rv770.tiling_group_size = 256;
1203	switch (rdev->family) {
1204	case CHIP_RV770:
1205		rdev->config.rv770.max_pipes = 4;
1206		rdev->config.rv770.max_tile_pipes = 8;
1207		rdev->config.rv770.max_simds = 10;
1208		rdev->config.rv770.max_backends = 4;
1209		rdev->config.rv770.max_gprs = 256;
1210		rdev->config.rv770.max_threads = 248;
1211		rdev->config.rv770.max_stack_entries = 512;
1212		rdev->config.rv770.max_hw_contexts = 8;
1213		rdev->config.rv770.max_gs_threads = 16 * 2;
1214		rdev->config.rv770.sx_max_export_size = 128;
1215		rdev->config.rv770.sx_max_export_pos_size = 16;
1216		rdev->config.rv770.sx_max_export_smx_size = 112;
1217		rdev->config.rv770.sq_num_cf_insts = 2;
1218
1219		rdev->config.rv770.sx_num_of_sets = 7;
1220		rdev->config.rv770.sc_prim_fifo_size = 0xF9;
1221		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1222		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1223		break;
1224	case CHIP_RV730:
1225		rdev->config.rv770.max_pipes = 2;
1226		rdev->config.rv770.max_tile_pipes = 4;
1227		rdev->config.rv770.max_simds = 8;
1228		rdev->config.rv770.max_backends = 2;
1229		rdev->config.rv770.max_gprs = 128;
1230		rdev->config.rv770.max_threads = 248;
1231		rdev->config.rv770.max_stack_entries = 256;
1232		rdev->config.rv770.max_hw_contexts = 8;
1233		rdev->config.rv770.max_gs_threads = 16 * 2;
1234		rdev->config.rv770.sx_max_export_size = 256;
1235		rdev->config.rv770.sx_max_export_pos_size = 32;
1236		rdev->config.rv770.sx_max_export_smx_size = 224;
1237		rdev->config.rv770.sq_num_cf_insts = 2;
1238
1239		rdev->config.rv770.sx_num_of_sets = 7;
1240		rdev->config.rv770.sc_prim_fifo_size = 0xf9;
1241		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1242		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1243		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1244			rdev->config.rv770.sx_max_export_pos_size -= 16;
1245			rdev->config.rv770.sx_max_export_smx_size += 16;
1246		}
1247		break;
1248	case CHIP_RV710:
1249		rdev->config.rv770.max_pipes = 2;
1250		rdev->config.rv770.max_tile_pipes = 2;
1251		rdev->config.rv770.max_simds = 2;
1252		rdev->config.rv770.max_backends = 1;
1253		rdev->config.rv770.max_gprs = 256;
1254		rdev->config.rv770.max_threads = 192;
1255		rdev->config.rv770.max_stack_entries = 256;
1256		rdev->config.rv770.max_hw_contexts = 4;
1257		rdev->config.rv770.max_gs_threads = 8 * 2;
1258		rdev->config.rv770.sx_max_export_size = 128;
1259		rdev->config.rv770.sx_max_export_pos_size = 16;
1260		rdev->config.rv770.sx_max_export_smx_size = 112;
1261		rdev->config.rv770.sq_num_cf_insts = 1;
1262
1263		rdev->config.rv770.sx_num_of_sets = 7;
1264		rdev->config.rv770.sc_prim_fifo_size = 0x40;
1265		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1266		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1267		break;
1268	case CHIP_RV740:
1269		rdev->config.rv770.max_pipes = 4;
1270		rdev->config.rv770.max_tile_pipes = 4;
1271		rdev->config.rv770.max_simds = 8;
1272		rdev->config.rv770.max_backends = 4;
1273		rdev->config.rv770.max_gprs = 256;
1274		rdev->config.rv770.max_threads = 248;
1275		rdev->config.rv770.max_stack_entries = 512;
1276		rdev->config.rv770.max_hw_contexts = 8;
1277		rdev->config.rv770.max_gs_threads = 16 * 2;
1278		rdev->config.rv770.sx_max_export_size = 256;
1279		rdev->config.rv770.sx_max_export_pos_size = 32;
1280		rdev->config.rv770.sx_max_export_smx_size = 224;
1281		rdev->config.rv770.sq_num_cf_insts = 2;
1282
1283		rdev->config.rv770.sx_num_of_sets = 7;
1284		rdev->config.rv770.sc_prim_fifo_size = 0x100;
1285		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1286		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1287
1288		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1289			rdev->config.rv770.sx_max_export_pos_size -= 16;
1290			rdev->config.rv770.sx_max_export_smx_size += 16;
1291		}
1292		break;
1293	default:
1294		break;
1295	}
1296
1297	/* Initialize HDP */
1298	j = 0;
1299	for (i = 0; i < 32; i++) {
1300		WREG32((0x2c14 + j), 0x00000000);
1301		WREG32((0x2c18 + j), 0x00000000);
1302		WREG32((0x2c1c + j), 0x00000000);
1303		WREG32((0x2c20 + j), 0x00000000);
1304		WREG32((0x2c24 + j), 0x00000000);
1305		j += 0x18;
1306	}
1307
1308	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1309
1310	/* setup tiling, simd, pipe config */
1311	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1312
1313	shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG);
1314	inactive_pipes = (shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> INACTIVE_QD_PIPES_SHIFT;
1315	for (i = 0, tmp = 1, active_number = 0; i < R7XX_MAX_PIPES; i++) {
1316		if (!(inactive_pipes & tmp)) {
1317			active_number++;
1318		}
1319		tmp <<= 1;
1320	}
1321	if (active_number == 1) {
1322		WREG32(SPI_CONFIG_CNTL, DISABLE_INTERP_1);
1323	} else {
1324		WREG32(SPI_CONFIG_CNTL, 0);
1325	}
1326
1327	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1328	tmp = rdev->config.rv770.max_simds -
1329		r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R7XX_MAX_SIMDS_MASK);
1330	rdev->config.rv770.active_simds = tmp;
1331
1332	switch (rdev->config.rv770.max_tile_pipes) {
1333	case 1:
1334	default:
1335		gb_tiling_config = PIPE_TILING(0);
1336		break;
1337	case 2:
1338		gb_tiling_config = PIPE_TILING(1);
1339		break;
1340	case 4:
1341		gb_tiling_config = PIPE_TILING(2);
1342		break;
1343	case 8:
1344		gb_tiling_config = PIPE_TILING(3);
1345		break;
1346	}
1347	rdev->config.rv770.tiling_npipes = rdev->config.rv770.max_tile_pipes;
1348
1349	disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R7XX_MAX_BACKENDS_MASK;
1350	tmp = 0;
1351	for (i = 0; i < rdev->config.rv770.max_backends; i++)
1352		tmp |= (1 << i);
1353	/* if all the backends are disabled, fix it up here */
1354	if ((disabled_rb_mask & tmp) == tmp) {
1355		for (i = 0; i < rdev->config.rv770.max_backends; i++)
1356			disabled_rb_mask &= ~(1 << i);
1357	}
1358	tmp = (gb_tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1359	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.rv770.max_backends,
1360					R7XX_MAX_BACKENDS, disabled_rb_mask);
1361	gb_tiling_config |= tmp << 16;
1362	rdev->config.rv770.backend_map = tmp;
1363
1364	if (rdev->family == CHIP_RV770)
1365		gb_tiling_config |= BANK_TILING(1);
1366	else {
1367		if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
1368			gb_tiling_config |= BANK_TILING(1);
1369		else
1370			gb_tiling_config |= BANK_TILING(0);
1371	}
1372	rdev->config.rv770.tiling_nbanks = 4 << ((gb_tiling_config >> 4) & 0x3);
1373	gb_tiling_config |= GROUP_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1374	if (((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT) > 3) {
1375		gb_tiling_config |= ROW_TILING(3);
1376		gb_tiling_config |= SAMPLE_SPLIT(3);
1377	} else {
1378		gb_tiling_config |=
1379			ROW_TILING(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1380		gb_tiling_config |=
1381			SAMPLE_SPLIT(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1382	}
1383
1384	gb_tiling_config |= BANK_SWAPS(1);
1385	rdev->config.rv770.tile_config = gb_tiling_config;
1386
1387	WREG32(GB_TILING_CONFIG, gb_tiling_config);
1388	WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1389	WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1390	WREG32(DMA_TILING_CONFIG, (gb_tiling_config & 0xffff));
1391	WREG32(DMA_TILING_CONFIG2, (gb_tiling_config & 0xffff));
1392	if (rdev->family == CHIP_RV730) {
1393		WREG32(UVD_UDEC_DB_TILING_CONFIG, (gb_tiling_config & 0xffff));
1394		WREG32(UVD_UDEC_DBW_TILING_CONFIG, (gb_tiling_config & 0xffff));
1395		WREG32(UVD_UDEC_TILING_CONFIG, (gb_tiling_config & 0xffff));
1396	}
1397
1398	WREG32(CGTS_SYS_TCC_DISABLE, 0);
1399	WREG32(CGTS_TCC_DISABLE, 0);
1400	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
1401	WREG32(CGTS_USER_TCC_DISABLE, 0);
1402
1403
1404	num_qd_pipes = R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1405	WREG32(VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & DEALLOC_DIST_MASK);
1406	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1407
1408	/* set HW defaults for 3D engine */
1409	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
1410				     ROQ_IB2_START(0x2b)));
1411
1412	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
1413
1414	ta_aux_cntl = RREG32(TA_CNTL_AUX);
1415	WREG32(TA_CNTL_AUX, ta_aux_cntl | DISABLE_CUBE_ANISO);
1416
1417	sx_debug_1 = RREG32(SX_DEBUG_1);
1418	sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1419	WREG32(SX_DEBUG_1, sx_debug_1);
1420
1421	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1422	smx_dc_ctl0 &= ~CACHE_DEPTH(0x1ff);
1423	smx_dc_ctl0 |= CACHE_DEPTH((rdev->config.rv770.sx_num_of_sets * 64) - 1);
1424	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1425
1426	if (rdev->family != CHIP_RV740)
1427		WREG32(SMX_EVENT_CTL, (ES_FLUSH_CTL(4) |
1428				       GS_FLUSH_CTL(4) |
1429				       ACK_FLUSH_CTL(3) |
1430				       SYNC_FLUSH_CTL));
1431
1432	if (rdev->family != CHIP_RV770)
1433		WREG32(SMX_SAR_CTL0, 0x00003f3f);
1434
1435	db_debug3 = RREG32(DB_DEBUG3);
1436	db_debug3 &= ~DB_CLK_OFF_DELAY(0x1f);
1437	switch (rdev->family) {
1438	case CHIP_RV770:
1439	case CHIP_RV740:
1440		db_debug3 |= DB_CLK_OFF_DELAY(0x1f);
1441		break;
1442	case CHIP_RV710:
1443	case CHIP_RV730:
1444	default:
1445		db_debug3 |= DB_CLK_OFF_DELAY(2);
1446		break;
1447	}
1448	WREG32(DB_DEBUG3, db_debug3);
1449
1450	if (rdev->family != CHIP_RV770) {
1451		db_debug4 = RREG32(DB_DEBUG4);
1452		db_debug4 |= DISABLE_TILE_COVERED_FOR_PS_ITER;
1453		WREG32(DB_DEBUG4, db_debug4);
1454	}
1455
1456	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.rv770.sx_max_export_size / 4) - 1) |
1457					POSITION_BUFFER_SIZE((rdev->config.rv770.sx_max_export_pos_size / 4) - 1) |
1458					SMX_BUFFER_SIZE((rdev->config.rv770.sx_max_export_smx_size / 4) - 1)));
1459
1460	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.rv770.sc_prim_fifo_size) |
1461				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_hiz_tile_fifo_size) |
1462				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_earlyz_tile_fifo_fize)));
1463
1464	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1465
1466	WREG32(VGT_NUM_INSTANCES, 1);
1467
1468	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
1469
1470	WREG32(CP_PERFMON_CNTL, 0);
1471
1472	sq_ms_fifo_sizes = (CACHE_FIFO_SIZE(16 * rdev->config.rv770.sq_num_cf_insts) |
1473			    DONE_FIFO_HIWATER(0xe0) |
1474			    ALU_UPDATE_FIFO_HIWATER(0x8));
1475	switch (rdev->family) {
1476	case CHIP_RV770:
1477	case CHIP_RV730:
1478	case CHIP_RV710:
1479		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x1);
1480		break;
1481	case CHIP_RV740:
1482	default:
1483		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x4);
1484		break;
1485	}
1486	WREG32(SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
1487
1488	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1489	 * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1490	 */
1491	sq_config = RREG32(SQ_CONFIG);
1492	sq_config &= ~(PS_PRIO(3) |
1493		       VS_PRIO(3) |
1494		       GS_PRIO(3) |
1495		       ES_PRIO(3));
1496	sq_config |= (DX9_CONSTS |
1497		      VC_ENABLE |
1498		      EXPORT_SRC_C |
1499		      PS_PRIO(0) |
1500		      VS_PRIO(1) |
1501		      GS_PRIO(2) |
1502		      ES_PRIO(3));
1503	if (rdev->family == CHIP_RV710)
1504		/* no vertex cache */
1505		sq_config &= ~VC_ENABLE;
1506
1507	WREG32(SQ_CONFIG, sq_config);
1508
1509	WREG32(SQ_GPR_RESOURCE_MGMT_1,  (NUM_PS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1510					 NUM_VS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1511					 NUM_CLAUSE_TEMP_GPRS(((rdev->config.rv770.max_gprs * 24)/64)/2)));
1512
1513	WREG32(SQ_GPR_RESOURCE_MGMT_2,  (NUM_GS_GPRS((rdev->config.rv770.max_gprs * 7)/64) |
1514					 NUM_ES_GPRS((rdev->config.rv770.max_gprs * 7)/64)));
1515
1516	sq_thread_resource_mgmt = (NUM_PS_THREADS((rdev->config.rv770.max_threads * 4)/8) |
1517				   NUM_VS_THREADS((rdev->config.rv770.max_threads * 2)/8) |
1518				   NUM_ES_THREADS((rdev->config.rv770.max_threads * 1)/8));
1519	if (((rdev->config.rv770.max_threads * 1) / 8) > rdev->config.rv770.max_gs_threads)
1520		sq_thread_resource_mgmt |= NUM_GS_THREADS(rdev->config.rv770.max_gs_threads);
1521	else
1522		sq_thread_resource_mgmt |= NUM_GS_THREADS((rdev->config.rv770.max_gs_threads * 1)/8);
1523	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1524
1525	WREG32(SQ_STACK_RESOURCE_MGMT_1, (NUM_PS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1526						     NUM_VS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1527
1528	WREG32(SQ_STACK_RESOURCE_MGMT_2, (NUM_GS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1529						     NUM_ES_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1530
1531	sq_dyn_gpr_size_simd_ab_0 = (SIMDA_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1532				     SIMDA_RING1((rdev->config.rv770.max_gprs * 38)/64) |
1533				     SIMDB_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1534				     SIMDB_RING1((rdev->config.rv770.max_gprs * 38)/64));
1535
1536	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
1537	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
1538	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
1539	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
1540	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
1541	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
1542	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
1543	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
1544
1545	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1546					  FORCE_EOV_MAX_REZ_CNT(255)));
1547
1548	if (rdev->family == CHIP_RV710)
1549		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(TC_ONLY) |
1550						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1551	else
1552		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(VC_AND_TC) |
1553						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1554
1555	switch (rdev->family) {
1556	case CHIP_RV770:
1557	case CHIP_RV730:
1558	case CHIP_RV740:
1559		gs_prim_buffer_depth = 384;
1560		break;
1561	case CHIP_RV710:
1562		gs_prim_buffer_depth = 128;
1563		break;
1564	default:
1565		break;
1566	}
1567
1568	num_gs_verts_per_thread = rdev->config.rv770.max_pipes * 16;
1569	vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1570	/* Max value for this is 256 */
1571	if (vgt_gs_per_es > 256)
1572		vgt_gs_per_es = 256;
1573
1574	WREG32(VGT_ES_PER_GS, 128);
1575	WREG32(VGT_GS_PER_ES, vgt_gs_per_es);
1576	WREG32(VGT_GS_PER_VS, 2);
1577
1578	/* more default values. 2D/3D driver should adjust as needed */
1579	WREG32(VGT_GS_VERTEX_REUSE, 16);
1580	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1581	WREG32(VGT_STRMOUT_EN, 0);
1582	WREG32(SX_MISC, 0);
1583	WREG32(PA_SC_MODE_CNTL, 0);
1584	WREG32(PA_SC_EDGERULE, 0xaaaaaaaa);
1585	WREG32(PA_SC_AA_CONFIG, 0);
1586	WREG32(PA_SC_CLIPRECT_RULE, 0xffff);
1587	WREG32(PA_SC_LINE_STIPPLE, 0);
1588	WREG32(SPI_INPUT_Z, 0);
1589	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1590	WREG32(CB_COLOR7_FRAG, 0);
1591
1592	/* clear render buffer base addresses */
1593	WREG32(CB_COLOR0_BASE, 0);
1594	WREG32(CB_COLOR1_BASE, 0);
1595	WREG32(CB_COLOR2_BASE, 0);
1596	WREG32(CB_COLOR3_BASE, 0);
1597	WREG32(CB_COLOR4_BASE, 0);
1598	WREG32(CB_COLOR5_BASE, 0);
1599	WREG32(CB_COLOR6_BASE, 0);
1600	WREG32(CB_COLOR7_BASE, 0);
1601
1602	WREG32(TCP_CNTL, 0);
1603
1604	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1605	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1606
1607	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1608
1609	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1610					  NUM_CLIP_SEQ(3)));
1611	WREG32(VC_ENHANCE, 0);
1612}
1613
1614void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1615{
1616	u64 size_bf, size_af;
1617
1618	if (mc->mc_vram_size > 0xE0000000) {
1619		/* leave room for at least 512M GTT */
1620		dev_warn(rdev->dev, "limiting VRAM\n");
1621		mc->real_vram_size = 0xE0000000;
1622		mc->mc_vram_size = 0xE0000000;
1623	}
1624	if (rdev->flags & RADEON_IS_AGP) {
1625		size_bf = mc->gtt_start;
1626		size_af = mc->mc_mask - mc->gtt_end;
1627		if (size_bf > size_af) {
1628			if (mc->mc_vram_size > size_bf) {
1629				dev_warn(rdev->dev, "limiting VRAM\n");
1630				mc->real_vram_size = size_bf;
1631				mc->mc_vram_size = size_bf;
1632			}
1633			mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1634		} else {
1635			if (mc->mc_vram_size > size_af) {
1636				dev_warn(rdev->dev, "limiting VRAM\n");
1637				mc->real_vram_size = size_af;
1638				mc->mc_vram_size = size_af;
1639			}
1640			mc->vram_start = mc->gtt_end + 1;
1641		}
1642		mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1643		dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1644				mc->mc_vram_size >> 20, mc->vram_start,
1645				mc->vram_end, mc->real_vram_size >> 20);
1646	} else {
1647		radeon_vram_location(rdev, &rdev->mc, 0);
1648		rdev->mc.gtt_base_align = 0;
1649		radeon_gtt_location(rdev, mc);
1650	}
1651}
1652
1653static int rv770_mc_init(struct radeon_device *rdev)
1654{
1655	u32 tmp;
1656	int chansize, numchan;
1657
1658	/* Get VRAM informations */
1659	rdev->mc.vram_is_ddr = true;
1660	tmp = RREG32(MC_ARB_RAMCFG);
1661	if (tmp & CHANSIZE_OVERRIDE) {
1662		chansize = 16;
1663	} else if (tmp & CHANSIZE_MASK) {
1664		chansize = 64;
1665	} else {
1666		chansize = 32;
1667	}
1668	tmp = RREG32(MC_SHARED_CHMAP);
1669	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1670	case 0:
1671	default:
1672		numchan = 1;
1673		break;
1674	case 1:
1675		numchan = 2;
1676		break;
1677	case 2:
1678		numchan = 4;
1679		break;
1680	case 3:
1681		numchan = 8;
1682		break;
1683	}
1684	rdev->mc.vram_width = numchan * chansize;
1685	/* Could aper size report 0 ? */
1686	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1687	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1688	/* Setup GPU memory space */
1689	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1690	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1691	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1692	r700_vram_gtt_location(rdev, &rdev->mc);
1693	radeon_update_bandwidth_info(rdev);
1694
1695	return 0;
1696}
1697
1698static void rv770_uvd_init(struct radeon_device *rdev)
1699{
1700	int r;
1701
1702	if (!rdev->has_uvd)
1703		return;
1704
1705	r = radeon_uvd_init(rdev);
1706	if (r) {
1707		dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
1708		/*
1709		 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
1710		 * to early fails uvd_v2_2_resume() and thus nothing happens
1711		 * there. So it is pointless to try to go through that code
1712		 * hence why we disable uvd here.
1713		 */
1714		rdev->has_uvd = false;
1715		return;
1716	}
1717	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
1718	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
1719}
1720
1721static void rv770_uvd_start(struct radeon_device *rdev)
1722{
1723	int r;
1724
1725	if (!rdev->has_uvd)
1726		return;
1727
1728	r = uvd_v2_2_resume(rdev);
1729	if (r) {
1730		dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
1731		goto error;
1732	}
1733	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
1734	if (r) {
1735		dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
1736		goto error;
1737	}
1738	return;
1739
1740error:
1741	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
1742}
1743
1744static void rv770_uvd_resume(struct radeon_device *rdev)
1745{
1746	struct radeon_ring *ring;
1747	int r;
1748
1749	if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
1750		return;
1751
1752	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
1753	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
1754	if (r) {
1755		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
1756		return;
1757	}
1758	r = uvd_v1_0_init(rdev);
1759	if (r) {
1760		dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
1761		return;
1762	}
1763}
1764
1765static int rv770_startup(struct radeon_device *rdev)
1766{
1767	struct radeon_ring *ring;
1768	int r;
1769
1770	/* enable pcie gen2 link */
1771	rv770_pcie_gen2_enable(rdev);
1772
1773	/* scratch needs to be initialized before MC */
1774	r = r600_vram_scratch_init(rdev);
1775	if (r)
1776		return r;
1777
1778	rv770_mc_program(rdev);
1779
1780	if (rdev->flags & RADEON_IS_AGP) {
1781		rv770_agp_enable(rdev);
1782	} else {
1783		r = rv770_pcie_gart_enable(rdev);
1784		if (r)
1785			return r;
1786	}
1787
1788	rv770_gpu_init(rdev);
1789
1790	/* allocate wb buffer */
1791	r = radeon_wb_init(rdev);
1792	if (r)
1793		return r;
1794
1795	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1796	if (r) {
1797		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1798		return r;
1799	}
1800
1801	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
1802	if (r) {
1803		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1804		return r;
1805	}
1806
1807	rv770_uvd_start(rdev);
1808
1809	/* Enable IRQ */
1810	if (!rdev->irq.installed) {
1811		r = radeon_irq_kms_init(rdev);
1812		if (r)
1813			return r;
1814	}
1815
1816	r = r600_irq_init(rdev);
1817	if (r) {
1818		DRM_ERROR("radeon: IH init failed (%d).\n", r);
1819		radeon_irq_kms_fini(rdev);
1820		return r;
1821	}
1822	r600_irq_set(rdev);
1823
1824	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1825	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1826			     RADEON_CP_PACKET2);
1827	if (r)
1828		return r;
1829
1830	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1831	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
1832			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1833	if (r)
1834		return r;
1835
1836	r = rv770_cp_load_microcode(rdev);
1837	if (r)
1838		return r;
1839	r = r600_cp_resume(rdev);
1840	if (r)
1841		return r;
1842
1843	r = r600_dma_resume(rdev);
1844	if (r)
1845		return r;
1846
1847	rv770_uvd_resume(rdev);
1848
1849	r = radeon_ib_pool_init(rdev);
1850	if (r) {
1851		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1852		return r;
1853	}
1854
1855	r = radeon_audio_init(rdev);
1856	if (r) {
1857		DRM_ERROR("radeon: audio init failed\n");
1858		return r;
1859	}
1860
1861	return 0;
1862}
1863
1864int rv770_resume(struct radeon_device *rdev)
1865{
1866	int r;
1867
1868	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
1869	 * posting will perform necessary task to bring back GPU into good
1870	 * shape.
1871	 */
1872	/* post card */
1873	atom_asic_init(rdev->mode_info.atom_context);
1874
1875	/* init golden registers */
1876	rv770_init_golden_registers(rdev);
1877
1878	if (rdev->pm.pm_method == PM_METHOD_DPM)
1879		radeon_pm_resume(rdev);
1880
1881	rdev->accel_working = true;
1882	r = rv770_startup(rdev);
1883	if (r) {
1884		DRM_ERROR("r600 startup failed on resume\n");
1885		rdev->accel_working = false;
1886		return r;
1887	}
1888
1889	return r;
1890
1891}
1892
1893int rv770_suspend(struct radeon_device *rdev)
1894{
1895	radeon_pm_suspend(rdev);
1896	radeon_audio_fini(rdev);
1897	if (rdev->has_uvd) {
1898		radeon_uvd_suspend(rdev);
1899		uvd_v1_0_fini(rdev);
 
1900	}
1901	r700_cp_stop(rdev);
1902	r600_dma_stop(rdev);
1903	r600_irq_suspend(rdev);
1904	radeon_wb_disable(rdev);
1905	rv770_pcie_gart_disable(rdev);
1906
1907	return 0;
1908}
1909
1910/* Plan is to move initialization in that function and use
1911 * helper function so that radeon_device_init pretty much
1912 * do nothing more than calling asic specific function. This
1913 * should also allow to remove a bunch of callback function
1914 * like vram_info.
1915 */
1916int rv770_init(struct radeon_device *rdev)
1917{
1918	int r;
1919
1920	/* Read BIOS */
1921	if (!radeon_get_bios(rdev)) {
1922		if (ASIC_IS_AVIVO(rdev))
1923			return -EINVAL;
1924	}
1925	/* Must be an ATOMBIOS */
1926	if (!rdev->is_atom_bios) {
1927		dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
1928		return -EINVAL;
1929	}
1930	r = radeon_atombios_init(rdev);
1931	if (r)
1932		return r;
1933	/* Post card if necessary */
1934	if (!radeon_card_posted(rdev)) {
1935		if (!rdev->bios) {
1936			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
1937			return -EINVAL;
1938		}
1939		DRM_INFO("GPU not posted. posting now...\n");
1940		atom_asic_init(rdev->mode_info.atom_context);
1941	}
1942	/* init golden registers */
1943	rv770_init_golden_registers(rdev);
1944	/* Initialize scratch registers */
1945	r600_scratch_init(rdev);
1946	/* Initialize surface registers */
1947	radeon_surface_init(rdev);
1948	/* Initialize clocks */
1949	radeon_get_clock_info(rdev->ddev);
1950	/* Fence driver */
1951	radeon_fence_driver_init(rdev);
 
 
1952	/* initialize AGP */
1953	if (rdev->flags & RADEON_IS_AGP) {
1954		r = radeon_agp_init(rdev);
1955		if (r)
1956			radeon_agp_disable(rdev);
1957	}
1958	r = rv770_mc_init(rdev);
1959	if (r)
1960		return r;
1961	/* Memory manager */
1962	r = radeon_bo_init(rdev);
1963	if (r)
1964		return r;
1965
1966	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1967		r = r600_init_microcode(rdev);
1968		if (r) {
1969			DRM_ERROR("Failed to load firmware!\n");
1970			return r;
1971		}
1972	}
1973
1974	/* Initialize power management */
1975	radeon_pm_init(rdev);
1976
1977	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
1978	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1979
1980	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
1981	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
1982
1983	rv770_uvd_init(rdev);
1984
1985	rdev->ih.ring_obj = NULL;
1986	r600_ih_ring_init(rdev, 64 * 1024);
1987
1988	r = r600_pcie_gart_init(rdev);
1989	if (r)
1990		return r;
1991
1992	rdev->accel_working = true;
1993	r = rv770_startup(rdev);
1994	if (r) {
1995		dev_err(rdev->dev, "disabling GPU acceleration\n");
1996		r700_cp_fini(rdev);
1997		r600_dma_fini(rdev);
1998		r600_irq_fini(rdev);
1999		radeon_wb_fini(rdev);
2000		radeon_ib_pool_fini(rdev);
2001		radeon_irq_kms_fini(rdev);
2002		rv770_pcie_gart_fini(rdev);
2003		rdev->accel_working = false;
2004	}
2005
2006	return 0;
2007}
2008
2009void rv770_fini(struct radeon_device *rdev)
2010{
2011	radeon_pm_fini(rdev);
2012	r700_cp_fini(rdev);
2013	r600_dma_fini(rdev);
2014	r600_irq_fini(rdev);
2015	radeon_wb_fini(rdev);
2016	radeon_ib_pool_fini(rdev);
2017	radeon_irq_kms_fini(rdev);
2018	uvd_v1_0_fini(rdev);
2019	radeon_uvd_fini(rdev);
2020	rv770_pcie_gart_fini(rdev);
2021	r600_vram_scratch_fini(rdev);
2022	radeon_gem_fini(rdev);
2023	radeon_fence_driver_fini(rdev);
2024	radeon_agp_fini(rdev);
2025	radeon_bo_fini(rdev);
2026	radeon_atombios_fini(rdev);
2027	kfree(rdev->bios);
2028	rdev->bios = NULL;
2029}
2030
2031static void rv770_pcie_gen2_enable(struct radeon_device *rdev)
2032{
2033	u32 link_width_cntl, lanes, speed_cntl, tmp;
2034	u16 link_cntl2;
2035
2036	if (radeon_pcie_gen2 == 0)
2037		return;
2038
2039	if (rdev->flags & RADEON_IS_IGP)
2040		return;
2041
2042	if (!(rdev->flags & RADEON_IS_PCIE))
2043		return;
2044
2045	/* x2 cards have a special sequence */
2046	if (ASIC_IS_X2(rdev))
2047		return;
2048
2049	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
2050		(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
2051		return;
2052
2053	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
2054
2055	/* advertise upconfig capability */
2056	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2057	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2058	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2059	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2060	if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
2061		lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
2062		link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
2063				     LC_RECONFIG_ARC_MISSING_ESCAPE);
2064		link_width_cntl |= lanes | LC_RECONFIG_NOW |
2065			LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT;
2066		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2067	} else {
2068		link_width_cntl |= LC_UPCONFIGURE_DIS;
2069		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2070	}
2071
2072	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2073	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
2074	    (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
2075
2076		tmp = RREG32(0x541c);
2077		WREG32(0x541c, tmp | 0x8);
2078		WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
2079		link_cntl2 = RREG16(0x4088);
2080		link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
2081		link_cntl2 |= 0x2;
2082		WREG16(0x4088, link_cntl2);
2083		WREG32(MM_CFGREGS_CNTL, 0);
2084
2085		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2086		speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
2087		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2088
2089		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2090		speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
2091		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2092
2093		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2094		speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
2095		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2096
2097		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2098		speed_cntl |= LC_GEN2_EN_STRAP;
2099		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2100
2101	} else {
2102		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2103		/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
2104		if (1)
2105			link_width_cntl |= LC_UPCONFIGURE_DIS;
2106		else
2107			link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2108		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2109	}
2110}