1/*
   2 * Copyright 2008 Advanced Micro Devices, Inc.
   3 * Copyright 2008 Red Hat Inc.
   4 * Copyright 2009 Jerome Glisse.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 *
  24 * Authors: Dave Airlie
  25 *          Alex Deucher
  26 *          Jerome Glisse
  27 */
  28#include <linux/firmware.h>
  29#include <linux/slab.h>
  30#include <drm/drmP.h>
  31#include "radeon.h"
  32#include "radeon_asic.h"
  33#include "radeon_audio.h"
  34#include <drm/radeon_drm.h>
  35#include "rv770d.h"
  36#include "atom.h"
  37#include "avivod.h"
  38
  39#define R700_PFP_UCODE_SIZE 848
  40#define R700_PM4_UCODE_SIZE 1360
  41
  42static void rv770_gpu_init(struct radeon_device *rdev);
  43void rv770_fini(struct radeon_device *rdev);
  44static void rv770_pcie_gen2_enable(struct radeon_device *rdev);
  45int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk);
  46
  47int rv770_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
  48{
  49	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
  50	int r;
  51
  52	/* RV740 uses evergreen uvd clk programming */
  53	if (rdev->family == CHIP_RV740)
  54		return evergreen_set_uvd_clocks(rdev, vclk, dclk);
  55
  56	/* bypass vclk and dclk with bclk */
  57	WREG32_P(CG_UPLL_FUNC_CNTL_2,
  58		 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
  59		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
  60
  61	if (!vclk || !dclk) {
  62		/* keep the Bypass mode, put PLL to sleep */
  63		WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
  64		return 0;
  65	}
  66
  67	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000,
  68					  43663, 0x03FFFFFE, 1, 30, ~0,
  69					  &fb_div, &vclk_div, &dclk_div);
  70	if (r)
  71		return r;
  72
  73	fb_div |= 1;
  74	vclk_div -= 1;
  75	dclk_div -= 1;
  76
  77	/* set UPLL_FB_DIV to 0x50000 */
  78	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(0x50000), ~UPLL_FB_DIV_MASK);
  79
  80	/* deassert UPLL_RESET and UPLL_SLEEP */
  81	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~(UPLL_RESET_MASK | UPLL_SLEEP_MASK));
  82
  83	/* assert BYPASS EN and FB_DIV[0] <- ??? why? */
  84	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
  85	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(1), ~UPLL_FB_DIV(1));
  86
  87	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
  88	if (r)
  89		return r;
  90
  91	/* assert PLL_RESET */
  92	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
  93
  94	/* set the required FB_DIV, REF_DIV, Post divder values */
  95	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_REF_DIV(1), ~UPLL_REF_DIV_MASK);
  96	WREG32_P(CG_UPLL_FUNC_CNTL_2,
  97		 UPLL_SW_HILEN(vclk_div >> 1) |
  98		 UPLL_SW_LOLEN((vclk_div >> 1) + (vclk_div & 1)) |
  99		 UPLL_SW_HILEN2(dclk_div >> 1) |
 100		 UPLL_SW_LOLEN2((dclk_div >> 1) + (dclk_div & 1)),
 101		 ~UPLL_SW_MASK);
 102
 103	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div),
 104		 ~UPLL_FB_DIV_MASK);
 105
 106	/* give the PLL some time to settle */
 107	mdelay(15);
 108
 109	/* deassert PLL_RESET */
 110	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
 111
 112	mdelay(15);
 113
 114	/* deassert BYPASS EN and FB_DIV[0] <- ??? why? */
 115	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
 116	WREG32_P(CG_UPLL_FUNC_CNTL_3, 0, ~UPLL_FB_DIV(1));
 117
 118	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
 119	if (r)
 120		return r;
 121
 122	/* switch VCLK and DCLK selection */
 123	WREG32_P(CG_UPLL_FUNC_CNTL_2,
 124		 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
 125		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
 126
 127	mdelay(100);
 128
 129	return 0;
 130}
 131
 132static const u32 r7xx_golden_registers[] =
 133{
 134	0x8d00, 0xffffffff, 0x0e0e0074,
 135	0x8d04, 0xffffffff, 0x013a2b34,
 136	0x9508, 0xffffffff, 0x00000002,
 137	0x8b20, 0xffffffff, 0,
 138	0x88c4, 0xffffffff, 0x000000c2,
 139	0x28350, 0xffffffff, 0,
 140	0x9058, 0xffffffff, 0x0fffc40f,
 141	0x240c, 0xffffffff, 0x00000380,
 142	0x733c, 0xffffffff, 0x00000002,
 143	0x2650, 0x00040000, 0,
 144	0x20bc, 0x00040000, 0,
 145	0x7300, 0xffffffff, 0x001000f0
 146};
 147
 148static const u32 r7xx_golden_dyn_gpr_registers[] =
 149{
 150	0x8db0, 0xffffffff, 0x98989898,
 151	0x8db4, 0xffffffff, 0x98989898,
 152	0x8db8, 0xffffffff, 0x98989898,
 153	0x8dbc, 0xffffffff, 0x98989898,
 154	0x8dc0, 0xffffffff, 0x98989898,
 155	0x8dc4, 0xffffffff, 0x98989898,
 156	0x8dc8, 0xffffffff, 0x98989898,
 157	0x8dcc, 0xffffffff, 0x98989898,
 158	0x88c4, 0xffffffff, 0x00000082
 159};
 160
 161static const u32 rv770_golden_registers[] =
 162{
 163	0x562c, 0xffffffff, 0,
 164	0x3f90, 0xffffffff, 0,
 165	0x9148, 0xffffffff, 0,
 166	0x3f94, 0xffffffff, 0,
 167	0x914c, 0xffffffff, 0,
 168	0x9698, 0x18000000, 0x18000000
 169};
 170
 171static const u32 rv770ce_golden_registers[] =
 172{
 173	0x562c, 0xffffffff, 0,
 174	0x3f90, 0xffffffff, 0x00cc0000,
 175	0x9148, 0xffffffff, 0x00cc0000,
 176	0x3f94, 0xffffffff, 0x00cc0000,
 177	0x914c, 0xffffffff, 0x00cc0000,
 178	0x9b7c, 0xffffffff, 0x00fa0000,
 179	0x3f8c, 0xffffffff, 0x00fa0000,
 180	0x9698, 0x18000000, 0x18000000
 181};
 182
 183static const u32 rv770_mgcg_init[] =
 184{
 185	0x8bcc, 0xffffffff, 0x130300f9,
 186	0x5448, 0xffffffff, 0x100,
 187	0x55e4, 0xffffffff, 0x100,
 188	0x160c, 0xffffffff, 0x100,
 189	0x5644, 0xffffffff, 0x100,
 190	0xc164, 0xffffffff, 0x100,
 191	0x8a18, 0xffffffff, 0x100,
 192	0x897c, 0xffffffff, 0x8000100,
 193	0x8b28, 0xffffffff, 0x3c000100,
 194	0x9144, 0xffffffff, 0x100,
 195	0x9a1c, 0xffffffff, 0x10000,
 196	0x9a50, 0xffffffff, 0x100,
 197	0x9a1c, 0xffffffff, 0x10001,
 198	0x9a50, 0xffffffff, 0x100,
 199	0x9a1c, 0xffffffff, 0x10002,
 200	0x9a50, 0xffffffff, 0x100,
 201	0x9a1c, 0xffffffff, 0x10003,
 202	0x9a50, 0xffffffff, 0x100,
 203	0x9a1c, 0xffffffff, 0x0,
 204	0x9870, 0xffffffff, 0x100,
 205	0x8d58, 0xffffffff, 0x100,
 206	0x9500, 0xffffffff, 0x0,
 207	0x9510, 0xffffffff, 0x100,
 208	0x9500, 0xffffffff, 0x1,
 209	0x9510, 0xffffffff, 0x100,
 210	0x9500, 0xffffffff, 0x2,
 211	0x9510, 0xffffffff, 0x100,
 212	0x9500, 0xffffffff, 0x3,
 213	0x9510, 0xffffffff, 0x100,
 214	0x9500, 0xffffffff, 0x4,
 215	0x9510, 0xffffffff, 0x100,
 216	0x9500, 0xffffffff, 0x5,
 217	0x9510, 0xffffffff, 0x100,
 218	0x9500, 0xffffffff, 0x6,
 219	0x9510, 0xffffffff, 0x100,
 220	0x9500, 0xffffffff, 0x7,
 221	0x9510, 0xffffffff, 0x100,
 222	0x9500, 0xffffffff, 0x8,
 223	0x9510, 0xffffffff, 0x100,
 224	0x9500, 0xffffffff, 0x9,
 225	0x9510, 0xffffffff, 0x100,
 226	0x9500, 0xffffffff, 0x8000,
 227	0x9490, 0xffffffff, 0x0,
 228	0x949c, 0xffffffff, 0x100,
 229	0x9490, 0xffffffff, 0x1,
 230	0x949c, 0xffffffff, 0x100,
 231	0x9490, 0xffffffff, 0x2,
 232	0x949c, 0xffffffff, 0x100,
 233	0x9490, 0xffffffff, 0x3,
 234	0x949c, 0xffffffff, 0x100,
 235	0x9490, 0xffffffff, 0x4,
 236	0x949c, 0xffffffff, 0x100,
 237	0x9490, 0xffffffff, 0x5,
 238	0x949c, 0xffffffff, 0x100,
 239	0x9490, 0xffffffff, 0x6,
 240	0x949c, 0xffffffff, 0x100,
 241	0x9490, 0xffffffff, 0x7,
 242	0x949c, 0xffffffff, 0x100,
 243	0x9490, 0xffffffff, 0x8,
 244	0x949c, 0xffffffff, 0x100,
 245	0x9490, 0xffffffff, 0x9,
 246	0x949c, 0xffffffff, 0x100,
 247	0x9490, 0xffffffff, 0x8000,
 248	0x9604, 0xffffffff, 0x0,
 249	0x9654, 0xffffffff, 0x100,
 250	0x9604, 0xffffffff, 0x1,
 251	0x9654, 0xffffffff, 0x100,
 252	0x9604, 0xffffffff, 0x2,
 253	0x9654, 0xffffffff, 0x100,
 254	0x9604, 0xffffffff, 0x3,
 255	0x9654, 0xffffffff, 0x100,
 256	0x9604, 0xffffffff, 0x4,
 257	0x9654, 0xffffffff, 0x100,
 258	0x9604, 0xffffffff, 0x5,
 259	0x9654, 0xffffffff, 0x100,
 260	0x9604, 0xffffffff, 0x6,
 261	0x9654, 0xffffffff, 0x100,
 262	0x9604, 0xffffffff, 0x7,
 263	0x9654, 0xffffffff, 0x100,
 264	0x9604, 0xffffffff, 0x8,
 265	0x9654, 0xffffffff, 0x100,
 266	0x9604, 0xffffffff, 0x9,
 267	0x9654, 0xffffffff, 0x100,
 268	0x9604, 0xffffffff, 0x80000000,
 269	0x9030, 0xffffffff, 0x100,
 270	0x9034, 0xffffffff, 0x100,
 271	0x9038, 0xffffffff, 0x100,
 272	0x903c, 0xffffffff, 0x100,
 273	0x9040, 0xffffffff, 0x100,
 274	0xa200, 0xffffffff, 0x100,
 275	0xa204, 0xffffffff, 0x100,
 276	0xa208, 0xffffffff, 0x100,
 277	0xa20c, 0xffffffff, 0x100,
 278	0x971c, 0xffffffff, 0x100,
 279	0x915c, 0xffffffff, 0x00020001,
 280	0x9160, 0xffffffff, 0x00040003,
 281	0x916c, 0xffffffff, 0x00060005,
 282	0x9170, 0xffffffff, 0x00080007,
 283	0x9174, 0xffffffff, 0x000a0009,
 284	0x9178, 0xffffffff, 0x000c000b,
 285	0x917c, 0xffffffff, 0x000e000d,
 286	0x9180, 0xffffffff, 0x0010000f,
 287	0x918c, 0xffffffff, 0x00120011,
 288	0x9190, 0xffffffff, 0x00140013,
 289	0x9194, 0xffffffff, 0x00020001,
 290	0x9198, 0xffffffff, 0x00040003,
 291	0x919c, 0xffffffff, 0x00060005,
 292	0x91a8, 0xffffffff, 0x00080007,
 293	0x91ac, 0xffffffff, 0x000a0009,
 294	0x91b0, 0xffffffff, 0x000c000b,
 295	0x91b4, 0xffffffff, 0x000e000d,
 296	0x91b8, 0xffffffff, 0x0010000f,
 297	0x91c4, 0xffffffff, 0x00120011,
 298	0x91c8, 0xffffffff, 0x00140013,
 299	0x91cc, 0xffffffff, 0x00020001,
 300	0x91d0, 0xffffffff, 0x00040003,
 301	0x91d4, 0xffffffff, 0x00060005,
 302	0x91e0, 0xffffffff, 0x00080007,
 303	0x91e4, 0xffffffff, 0x000a0009,
 304	0x91e8, 0xffffffff, 0x000c000b,
 305	0x91ec, 0xffffffff, 0x00020001,
 306	0x91f0, 0xffffffff, 0x00040003,
 307	0x91f4, 0xffffffff, 0x00060005,
 308	0x9200, 0xffffffff, 0x00080007,
 309	0x9204, 0xffffffff, 0x000a0009,
 310	0x9208, 0xffffffff, 0x000c000b,
 311	0x920c, 0xffffffff, 0x000e000d,
 312	0x9210, 0xffffffff, 0x0010000f,
 313	0x921c, 0xffffffff, 0x00120011,
 314	0x9220, 0xffffffff, 0x00140013,
 315	0x9224, 0xffffffff, 0x00020001,
 316	0x9228, 0xffffffff, 0x00040003,
 317	0x922c, 0xffffffff, 0x00060005,
 318	0x9238, 0xffffffff, 0x00080007,
 319	0x923c, 0xffffffff, 0x000a0009,
 320	0x9240, 0xffffffff, 0x000c000b,
 321	0x9244, 0xffffffff, 0x000e000d,
 322	0x9248, 0xffffffff, 0x0010000f,
 323	0x9254, 0xffffffff, 0x00120011,
 324	0x9258, 0xffffffff, 0x00140013,
 325	0x925c, 0xffffffff, 0x00020001,
 326	0x9260, 0xffffffff, 0x00040003,
 327	0x9264, 0xffffffff, 0x00060005,
 328	0x9270, 0xffffffff, 0x00080007,
 329	0x9274, 0xffffffff, 0x000a0009,
 330	0x9278, 0xffffffff, 0x000c000b,
 331	0x927c, 0xffffffff, 0x000e000d,
 332	0x9280, 0xffffffff, 0x0010000f,
 333	0x928c, 0xffffffff, 0x00120011,
 334	0x9290, 0xffffffff, 0x00140013,
 335	0x9294, 0xffffffff, 0x00020001,
 336	0x929c, 0xffffffff, 0x00040003,
 337	0x92a0, 0xffffffff, 0x00060005,
 338	0x92a4, 0xffffffff, 0x00080007
 339};
 340
 341static const u32 rv710_golden_registers[] =
 342{
 343	0x3f90, 0x00ff0000, 0x00fc0000,
 344	0x9148, 0x00ff0000, 0x00fc0000,
 345	0x3f94, 0x00ff0000, 0x00fc0000,
 346	0x914c, 0x00ff0000, 0x00fc0000,
 347	0xb4c, 0x00000020, 0x00000020,
 348	0xa180, 0xffffffff, 0x00003f3f
 349};
 350
 351static const u32 rv710_mgcg_init[] =
 352{
 353	0x8bcc, 0xffffffff, 0x13030040,
 354	0x5448, 0xffffffff, 0x100,
 355	0x55e4, 0xffffffff, 0x100,
 356	0x160c, 0xffffffff, 0x100,
 357	0x5644, 0xffffffff, 0x100,
 358	0xc164, 0xffffffff, 0x100,
 359	0x8a18, 0xffffffff, 0x100,
 360	0x897c, 0xffffffff, 0x8000100,
 361	0x8b28, 0xffffffff, 0x3c000100,
 362	0x9144, 0xffffffff, 0x100,
 363	0x9a1c, 0xffffffff, 0x10000,
 364	0x9a50, 0xffffffff, 0x100,
 365	0x9a1c, 0xffffffff, 0x0,
 366	0x9870, 0xffffffff, 0x100,
 367	0x8d58, 0xffffffff, 0x100,
 368	0x9500, 0xffffffff, 0x0,
 369	0x9510, 0xffffffff, 0x100,
 370	0x9500, 0xffffffff, 0x1,
 371	0x9510, 0xffffffff, 0x100,
 372	0x9500, 0xffffffff, 0x8000,
 373	0x9490, 0xffffffff, 0x0,
 374	0x949c, 0xffffffff, 0x100,
 375	0x9490, 0xffffffff, 0x1,
 376	0x949c, 0xffffffff, 0x100,
 377	0x9490, 0xffffffff, 0x8000,
 378	0x9604, 0xffffffff, 0x0,
 379	0x9654, 0xffffffff, 0x100,
 380	0x9604, 0xffffffff, 0x1,
 381	0x9654, 0xffffffff, 0x100,
 382	0x9604, 0xffffffff, 0x80000000,
 383	0x9030, 0xffffffff, 0x100,
 384	0x9034, 0xffffffff, 0x100,
 385	0x9038, 0xffffffff, 0x100,
 386	0x903c, 0xffffffff, 0x100,
 387	0x9040, 0xffffffff, 0x100,
 388	0xa200, 0xffffffff, 0x100,
 389	0xa204, 0xffffffff, 0x100,
 390	0xa208, 0xffffffff, 0x100,
 391	0xa20c, 0xffffffff, 0x100,
 392	0x971c, 0xffffffff, 0x100,
 393	0x915c, 0xffffffff, 0x00020001,
 394	0x9174, 0xffffffff, 0x00000003,
 395	0x9178, 0xffffffff, 0x00050001,
 396	0x917c, 0xffffffff, 0x00030002,
 397	0x918c, 0xffffffff, 0x00000004,
 398	0x9190, 0xffffffff, 0x00070006,
 399	0x9194, 0xffffffff, 0x00050001,
 400	0x9198, 0xffffffff, 0x00030002,
 401	0x91a8, 0xffffffff, 0x00000004,
 402	0x91ac, 0xffffffff, 0x00070006,
 403	0x91e8, 0xffffffff, 0x00000001,
 404	0x9294, 0xffffffff, 0x00000001,
 405	0x929c, 0xffffffff, 0x00000002,
 406	0x92a0, 0xffffffff, 0x00040003,
 407	0x9150, 0xffffffff, 0x4d940000
 408};
 409
 410static const u32 rv730_golden_registers[] =
 411{
 412	0x3f90, 0x00ff0000, 0x00f00000,
 413	0x9148, 0x00ff0000, 0x00f00000,
 414	0x3f94, 0x00ff0000, 0x00f00000,
 415	0x914c, 0x00ff0000, 0x00f00000,
 416	0x900c, 0xffffffff, 0x003b033f,
 417	0xb4c, 0x00000020, 0x00000020,
 418	0xa180, 0xffffffff, 0x00003f3f
 419};
 420
 421static const u32 rv730_mgcg_init[] =
 422{
 423	0x8bcc, 0xffffffff, 0x130300f9,
 424	0x5448, 0xffffffff, 0x100,
 425	0x55e4, 0xffffffff, 0x100,
 426	0x160c, 0xffffffff, 0x100,
 427	0x5644, 0xffffffff, 0x100,
 428	0xc164, 0xffffffff, 0x100,
 429	0x8a18, 0xffffffff, 0x100,
 430	0x897c, 0xffffffff, 0x8000100,
 431	0x8b28, 0xffffffff, 0x3c000100,
 432	0x9144, 0xffffffff, 0x100,
 433	0x9a1c, 0xffffffff, 0x10000,
 434	0x9a50, 0xffffffff, 0x100,
 435	0x9a1c, 0xffffffff, 0x10001,
 436	0x9a50, 0xffffffff, 0x100,
 437	0x9a1c, 0xffffffff, 0x0,
 438	0x9870, 0xffffffff, 0x100,
 439	0x8d58, 0xffffffff, 0x100,
 440	0x9500, 0xffffffff, 0x0,
 441	0x9510, 0xffffffff, 0x100,
 442	0x9500, 0xffffffff, 0x1,
 443	0x9510, 0xffffffff, 0x100,
 444	0x9500, 0xffffffff, 0x2,
 445	0x9510, 0xffffffff, 0x100,
 446	0x9500, 0xffffffff, 0x3,
 447	0x9510, 0xffffffff, 0x100,
 448	0x9500, 0xffffffff, 0x4,
 449	0x9510, 0xffffffff, 0x100,
 450	0x9500, 0xffffffff, 0x5,
 451	0x9510, 0xffffffff, 0x100,
 452	0x9500, 0xffffffff, 0x6,
 453	0x9510, 0xffffffff, 0x100,
 454	0x9500, 0xffffffff, 0x7,
 455	0x9510, 0xffffffff, 0x100,
 456	0x9500, 0xffffffff, 0x8000,
 457	0x9490, 0xffffffff, 0x0,
 458	0x949c, 0xffffffff, 0x100,
 459	0x9490, 0xffffffff, 0x1,
 460	0x949c, 0xffffffff, 0x100,
 461	0x9490, 0xffffffff, 0x2,
 462	0x949c, 0xffffffff, 0x100,
 463	0x9490, 0xffffffff, 0x3,
 464	0x949c, 0xffffffff, 0x100,
 465	0x9490, 0xffffffff, 0x4,
 466	0x949c, 0xffffffff, 0x100,
 467	0x9490, 0xffffffff, 0x5,
 468	0x949c, 0xffffffff, 0x100,
 469	0x9490, 0xffffffff, 0x6,
 470	0x949c, 0xffffffff, 0x100,
 471	0x9490, 0xffffffff, 0x7,
 472	0x949c, 0xffffffff, 0x100,
 473	0x9490, 0xffffffff, 0x8000,
 474	0x9604, 0xffffffff, 0x0,
 475	0x9654, 0xffffffff, 0x100,
 476	0x9604, 0xffffffff, 0x1,
 477	0x9654, 0xffffffff, 0x100,
 478	0x9604, 0xffffffff, 0x2,
 479	0x9654, 0xffffffff, 0x100,
 480	0x9604, 0xffffffff, 0x3,
 481	0x9654, 0xffffffff, 0x100,
 482	0x9604, 0xffffffff, 0x4,
 483	0x9654, 0xffffffff, 0x100,
 484	0x9604, 0xffffffff, 0x5,
 485	0x9654, 0xffffffff, 0x100,
 486	0x9604, 0xffffffff, 0x6,
 487	0x9654, 0xffffffff, 0x100,
 488	0x9604, 0xffffffff, 0x7,
 489	0x9654, 0xffffffff, 0x100,
 490	0x9604, 0xffffffff, 0x80000000,
 491	0x9030, 0xffffffff, 0x100,
 492	0x9034, 0xffffffff, 0x100,
 493	0x9038, 0xffffffff, 0x100,
 494	0x903c, 0xffffffff, 0x100,
 495	0x9040, 0xffffffff, 0x100,
 496	0xa200, 0xffffffff, 0x100,
 497	0xa204, 0xffffffff, 0x100,
 498	0xa208, 0xffffffff, 0x100,
 499	0xa20c, 0xffffffff, 0x100,
 500	0x971c, 0xffffffff, 0x100,
 501	0x915c, 0xffffffff, 0x00020001,
 502	0x916c, 0xffffffff, 0x00040003,
 503	0x9170, 0xffffffff, 0x00000005,
 504	0x9178, 0xffffffff, 0x00050001,
 505	0x917c, 0xffffffff, 0x00030002,
 506	0x918c, 0xffffffff, 0x00000004,
 507	0x9190, 0xffffffff, 0x00070006,
 508	0x9194, 0xffffffff, 0x00050001,
 509	0x9198, 0xffffffff, 0x00030002,
 510	0x91a8, 0xffffffff, 0x00000004,
 511	0x91ac, 0xffffffff, 0x00070006,
 512	0x91b0, 0xffffffff, 0x00050001,
 513	0x91b4, 0xffffffff, 0x00030002,
 514	0x91c4, 0xffffffff, 0x00000004,
 515	0x91c8, 0xffffffff, 0x00070006,
 516	0x91cc, 0xffffffff, 0x00050001,
 517	0x91d0, 0xffffffff, 0x00030002,
 518	0x91e0, 0xffffffff, 0x00000004,
 519	0x91e4, 0xffffffff, 0x00070006,
 520	0x91e8, 0xffffffff, 0x00000001,
 521	0x91ec, 0xffffffff, 0x00050001,
 522	0x91f0, 0xffffffff, 0x00030002,
 523	0x9200, 0xffffffff, 0x00000004,
 524	0x9204, 0xffffffff, 0x00070006,
 525	0x9208, 0xffffffff, 0x00050001,
 526	0x920c, 0xffffffff, 0x00030002,
 527	0x921c, 0xffffffff, 0x00000004,
 528	0x9220, 0xffffffff, 0x00070006,
 529	0x9224, 0xffffffff, 0x00050001,
 530	0x9228, 0xffffffff, 0x00030002,
 531	0x9238, 0xffffffff, 0x00000004,
 532	0x923c, 0xffffffff, 0x00070006,
 533	0x9240, 0xffffffff, 0x00050001,
 534	0x9244, 0xffffffff, 0x00030002,
 535	0x9254, 0xffffffff, 0x00000004,
 536	0x9258, 0xffffffff, 0x00070006,
 537	0x9294, 0xffffffff, 0x00000001,
 538	0x929c, 0xffffffff, 0x00000002,
 539	0x92a0, 0xffffffff, 0x00040003,
 540	0x92a4, 0xffffffff, 0x00000005
 541};
 542
 543static const u32 rv740_golden_registers[] =
 544{
 545	0x88c4, 0xffffffff, 0x00000082,
 546	0x28a50, 0xfffffffc, 0x00000004,
 547	0x2650, 0x00040000, 0,
 548	0x20bc, 0x00040000, 0,
 549	0x733c, 0xffffffff, 0x00000002,
 550	0x7300, 0xffffffff, 0x001000f0,
 551	0x3f90, 0x00ff0000, 0,
 552	0x9148, 0x00ff0000, 0,
 553	0x3f94, 0x00ff0000, 0,
 554	0x914c, 0x00ff0000, 0,
 555	0x240c, 0xffffffff, 0x00000380,
 556	0x8a14, 0x00000007, 0x00000007,
 557	0x8b24, 0xffffffff, 0x00ff0fff,
 558	0x28a4c, 0xffffffff, 0x00004000,
 559	0xa180, 0xffffffff, 0x00003f3f,
 560	0x8d00, 0xffffffff, 0x0e0e003a,
 561	0x8d04, 0xffffffff, 0x013a0e2a,
 562	0x8c00, 0xffffffff, 0xe400000f,
 563	0x8db0, 0xffffffff, 0x98989898,
 564	0x8db4, 0xffffffff, 0x98989898,
 565	0x8db8, 0xffffffff, 0x98989898,
 566	0x8dbc, 0xffffffff, 0x98989898,
 567	0x8dc0, 0xffffffff, 0x98989898,
 568	0x8dc4, 0xffffffff, 0x98989898,
 569	0x8dc8, 0xffffffff, 0x98989898,
 570	0x8dcc, 0xffffffff, 0x98989898,
 571	0x9058, 0xffffffff, 0x0fffc40f,
 572	0x900c, 0xffffffff, 0x003b033f,
 573	0x28350, 0xffffffff, 0,
 574	0x8cf0, 0x1fffffff, 0x08e00420,
 575	0x9508, 0xffffffff, 0x00000002,
 576	0x88c4, 0xffffffff, 0x000000c2,
 577	0x9698, 0x18000000, 0x18000000
 578};
 579
 580static const u32 rv740_mgcg_init[] =
 581{
 582	0x8bcc, 0xffffffff, 0x13030100,
 583	0x5448, 0xffffffff, 0x100,
 584	0x55e4, 0xffffffff, 0x100,
 585	0x160c, 0xffffffff, 0x100,
 586	0x5644, 0xffffffff, 0x100,
 587	0xc164, 0xffffffff, 0x100,
 588	0x8a18, 0xffffffff, 0x100,
 589	0x897c, 0xffffffff, 0x100,
 590	0x8b28, 0xffffffff, 0x100,
 591	0x9144, 0xffffffff, 0x100,
 592	0x9a1c, 0xffffffff, 0x10000,
 593	0x9a50, 0xffffffff, 0x100,
 594	0x9a1c, 0xffffffff, 0x10001,
 595	0x9a50, 0xffffffff, 0x100,
 596	0x9a1c, 0xffffffff, 0x10002,
 597	0x9a50, 0xffffffff, 0x100,
 598	0x9a1c, 0xffffffff, 0x10003,
 599	0x9a50, 0xffffffff, 0x100,
 600	0x9a1c, 0xffffffff, 0x0,
 601	0x9870, 0xffffffff, 0x100,
 602	0x8d58, 0xffffffff, 0x100,
 603	0x9500, 0xffffffff, 0x0,
 604	0x9510, 0xffffffff, 0x100,
 605	0x9500, 0xffffffff, 0x1,
 606	0x9510, 0xffffffff, 0x100,
 607	0x9500, 0xffffffff, 0x2,
 608	0x9510, 0xffffffff, 0x100,
 609	0x9500, 0xffffffff, 0x3,
 610	0x9510, 0xffffffff, 0x100,
 611	0x9500, 0xffffffff, 0x4,
 612	0x9510, 0xffffffff, 0x100,
 613	0x9500, 0xffffffff, 0x5,
 614	0x9510, 0xffffffff, 0x100,
 615	0x9500, 0xffffffff, 0x6,
 616	0x9510, 0xffffffff, 0x100,
 617	0x9500, 0xffffffff, 0x7,
 618	0x9510, 0xffffffff, 0x100,
 619	0x9500, 0xffffffff, 0x8000,
 620	0x9490, 0xffffffff, 0x0,
 621	0x949c, 0xffffffff, 0x100,
 622	0x9490, 0xffffffff, 0x1,
 623	0x949c, 0xffffffff, 0x100,
 624	0x9490, 0xffffffff, 0x2,
 625	0x949c, 0xffffffff, 0x100,
 626	0x9490, 0xffffffff, 0x3,
 627	0x949c, 0xffffffff, 0x100,
 628	0x9490, 0xffffffff, 0x4,
 629	0x949c, 0xffffffff, 0x100,
 630	0x9490, 0xffffffff, 0x5,
 631	0x949c, 0xffffffff, 0x100,
 632	0x9490, 0xffffffff, 0x6,
 633	0x949c, 0xffffffff, 0x100,
 634	0x9490, 0xffffffff, 0x7,
 635	0x949c, 0xffffffff, 0x100,
 636	0x9490, 0xffffffff, 0x8000,
 637	0x9604, 0xffffffff, 0x0,
 638	0x9654, 0xffffffff, 0x100,
 639	0x9604, 0xffffffff, 0x1,
 640	0x9654, 0xffffffff, 0x100,
 641	0x9604, 0xffffffff, 0x2,
 642	0x9654, 0xffffffff, 0x100,
 643	0x9604, 0xffffffff, 0x3,
 644	0x9654, 0xffffffff, 0x100,
 645	0x9604, 0xffffffff, 0x4,
 646	0x9654, 0xffffffff, 0x100,
 647	0x9604, 0xffffffff, 0x5,
 648	0x9654, 0xffffffff, 0x100,
 649	0x9604, 0xffffffff, 0x6,
 650	0x9654, 0xffffffff, 0x100,
 651	0x9604, 0xffffffff, 0x7,
 652	0x9654, 0xffffffff, 0x100,
 653	0x9604, 0xffffffff, 0x80000000,
 654	0x9030, 0xffffffff, 0x100,
 655	0x9034, 0xffffffff, 0x100,
 656	0x9038, 0xffffffff, 0x100,
 657	0x903c, 0xffffffff, 0x100,
 658	0x9040, 0xffffffff, 0x100,
 659	0xa200, 0xffffffff, 0x100,
 660	0xa204, 0xffffffff, 0x100,
 661	0xa208, 0xffffffff, 0x100,
 662	0xa20c, 0xffffffff, 0x100,
 663	0x971c, 0xffffffff, 0x100,
 664	0x915c, 0xffffffff, 0x00020001,
 665	0x9160, 0xffffffff, 0x00040003,
 666	0x916c, 0xffffffff, 0x00060005,
 667	0x9170, 0xffffffff, 0x00080007,
 668	0x9174, 0xffffffff, 0x000a0009,
 669	0x9178, 0xffffffff, 0x000c000b,
 670	0x917c, 0xffffffff, 0x000e000d,
 671	0x9180, 0xffffffff, 0x0010000f,
 672	0x918c, 0xffffffff, 0x00120011,
 673	0x9190, 0xffffffff, 0x00140013,
 674	0x9194, 0xffffffff, 0x00020001,
 675	0x9198, 0xffffffff, 0x00040003,
 676	0x919c, 0xffffffff, 0x00060005,
 677	0x91a8, 0xffffffff, 0x00080007,
 678	0x91ac, 0xffffffff, 0x000a0009,
 679	0x91b0, 0xffffffff, 0x000c000b,
 680	0x91b4, 0xffffffff, 0x000e000d,
 681	0x91b8, 0xffffffff, 0x0010000f,
 682	0x91c4, 0xffffffff, 0x00120011,
 683	0x91c8, 0xffffffff, 0x00140013,
 684	0x91cc, 0xffffffff, 0x00020001,
 685	0x91d0, 0xffffffff, 0x00040003,
 686	0x91d4, 0xffffffff, 0x00060005,
 687	0x91e0, 0xffffffff, 0x00080007,
 688	0x91e4, 0xffffffff, 0x000a0009,
 689	0x91e8, 0xffffffff, 0x000c000b,
 690	0x91ec, 0xffffffff, 0x00020001,
 691	0x91f0, 0xffffffff, 0x00040003,
 692	0x91f4, 0xffffffff, 0x00060005,
 693	0x9200, 0xffffffff, 0x00080007,
 694	0x9204, 0xffffffff, 0x000a0009,
 695	0x9208, 0xffffffff, 0x000c000b,
 696	0x920c, 0xffffffff, 0x000e000d,
 697	0x9210, 0xffffffff, 0x0010000f,
 698	0x921c, 0xffffffff, 0x00120011,
 699	0x9220, 0xffffffff, 0x00140013,
 700	0x9224, 0xffffffff, 0x00020001,
 701	0x9228, 0xffffffff, 0x00040003,
 702	0x922c, 0xffffffff, 0x00060005,
 703	0x9238, 0xffffffff, 0x00080007,
 704	0x923c, 0xffffffff, 0x000a0009,
 705	0x9240, 0xffffffff, 0x000c000b,
 706	0x9244, 0xffffffff, 0x000e000d,
 707	0x9248, 0xffffffff, 0x0010000f,
 708	0x9254, 0xffffffff, 0x00120011,
 709	0x9258, 0xffffffff, 0x00140013,
 710	0x9294, 0xffffffff, 0x00020001,
 711	0x929c, 0xffffffff, 0x00040003,
 712	0x92a0, 0xffffffff, 0x00060005,
 713	0x92a4, 0xffffffff, 0x00080007
 714};
 715
 716static void rv770_init_golden_registers(struct radeon_device *rdev)
 717{
 718	switch (rdev->family) {
 719	case CHIP_RV770:
 720		radeon_program_register_sequence(rdev,
 721						 r7xx_golden_registers,
 722						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
 723		radeon_program_register_sequence(rdev,
 724						 r7xx_golden_dyn_gpr_registers,
 725						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 726		if (rdev->pdev->device == 0x994e)
 727			radeon_program_register_sequence(rdev,
 728							 rv770ce_golden_registers,
 729							 (const u32)ARRAY_SIZE(rv770ce_golden_registers));
 730		else
 731			radeon_program_register_sequence(rdev,
 732							 rv770_golden_registers,
 733							 (const u32)ARRAY_SIZE(rv770_golden_registers));
 734		radeon_program_register_sequence(rdev,
 735						 rv770_mgcg_init,
 736						 (const u32)ARRAY_SIZE(rv770_mgcg_init));
 737		break;
 738	case CHIP_RV730:
 739		radeon_program_register_sequence(rdev,
 740						 r7xx_golden_registers,
 741						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
 742		radeon_program_register_sequence(rdev,
 743						 r7xx_golden_dyn_gpr_registers,
 744						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 745		radeon_program_register_sequence(rdev,
 746						 rv730_golden_registers,
 747						 (const u32)ARRAY_SIZE(rv730_golden_registers));
 748		radeon_program_register_sequence(rdev,
 749						 rv730_mgcg_init,
 750						 (const u32)ARRAY_SIZE(rv730_mgcg_init));
 751		break;
 752	case CHIP_RV710:
 753		radeon_program_register_sequence(rdev,
 754						 r7xx_golden_registers,
 755						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
 756		radeon_program_register_sequence(rdev,
 757						 r7xx_golden_dyn_gpr_registers,
 758						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 759		radeon_program_register_sequence(rdev,
 760						 rv710_golden_registers,
 761						 (const u32)ARRAY_SIZE(rv710_golden_registers));
 762		radeon_program_register_sequence(rdev,
 763						 rv710_mgcg_init,
 764						 (const u32)ARRAY_SIZE(rv710_mgcg_init));
 765		break;
 766	case CHIP_RV740:
 767		radeon_program_register_sequence(rdev,
 768						 rv740_golden_registers,
 769						 (const u32)ARRAY_SIZE(rv740_golden_registers));
 770		radeon_program_register_sequence(rdev,
 771						 rv740_mgcg_init,
 772						 (const u32)ARRAY_SIZE(rv740_mgcg_init));
 773		break;
 774	default:
 775		break;
 776	}
 777}
 778
 779#define PCIE_BUS_CLK                10000
 780#define TCLK                        (PCIE_BUS_CLK / 10)
 781
 782/**
 783 * rv770_get_xclk - get the xclk
 784 *
 785 * @rdev: radeon_device pointer
 786 *
 787 * Returns the reference clock used by the gfx engine
 788 * (r7xx-cayman).
 789 */
 790u32 rv770_get_xclk(struct radeon_device *rdev)
 791{
 792	u32 reference_clock = rdev->clock.spll.reference_freq;
 793	u32 tmp = RREG32(CG_CLKPIN_CNTL);
 794
 795	if (tmp & MUX_TCLK_TO_XCLK)
 796		return TCLK;
 797
 798	if (tmp & XTALIN_DIVIDE)
 799		return reference_clock / 4;
 800
 801	return reference_clock;
 802}
 803
 804void rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
 805{
 806	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 807	u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
 808	int i;
 809
 810	/* Lock the graphics update lock */
 811	tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
 812	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 813
 814	/* update the scanout addresses */
 
 
 815	if (radeon_crtc->crtc_id) {
 816		WREG32(D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 817		WREG32(D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 818	} else {
 819		WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 820		WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 821	}
 822	WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 823	       (u32)crtc_base);
 824	WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 825	       (u32)crtc_base);
 826
 827	/* Wait for update_pending to go high. */
 828	for (i = 0; i < rdev->usec_timeout; i++) {
 829		if (RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING)
 830			break;
 831		udelay(1);
 832	}
 833	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
 834
 835	/* Unlock the lock, so double-buffering can take place inside vblank */
 836	tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
 837	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 838}
 839
 840bool rv770_page_flip_pending(struct radeon_device *rdev, int crtc_id)
 841{
 842	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 843
 844	/* Return current update_pending status: */
 845	return !!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) &
 846		AVIVO_D1GRPH_SURFACE_UPDATE_PENDING);
 847}
 848
 849/* get temperature in millidegrees */
 850int rv770_get_temp(struct radeon_device *rdev)
 851{
 852	u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
 853		ASIC_T_SHIFT;
 854	int actual_temp;
 855
 856	if (temp & 0x400)
 857		actual_temp = -256;
 858	else if (temp & 0x200)
 859		actual_temp = 255;
 860	else if (temp & 0x100) {
 861		actual_temp = temp & 0x1ff;
 862		actual_temp |= ~0x1ff;
 863	} else
 864		actual_temp = temp & 0xff;
 865
 866	return (actual_temp * 1000) / 2;
 867}
 868
 869void rv770_pm_misc(struct radeon_device *rdev)
 870{
 871	int req_ps_idx = rdev->pm.requested_power_state_index;
 872	int req_cm_idx = rdev->pm.requested_clock_mode_index;
 873	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
 874	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 875
 876	if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
 877		/* 0xff01 is a flag rather then an actual voltage */
 878		if (voltage->voltage == 0xff01)
 879			return;
 880		if (voltage->voltage != rdev->pm.current_vddc) {
 881			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
 882			rdev->pm.current_vddc = voltage->voltage;
 883			DRM_DEBUG("Setting: v: %d\n", voltage->voltage);
 884		}
 885	}
 886}
 887
 888/*
 889 * GART
 890 */
 891static int rv770_pcie_gart_enable(struct radeon_device *rdev)
 892{
 893	u32 tmp;
 894	int r, i;
 895
 896	if (rdev->gart.robj == NULL) {
 897		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 898		return -EINVAL;
 899	}
 900	r = radeon_gart_table_vram_pin(rdev);
 901	if (r)
 902		return r;
 903	/* Setup L2 cache */
 904	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 905				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 906				EFFECTIVE_L2_QUEUE_SIZE(7));
 907	WREG32(VM_L2_CNTL2, 0);
 908	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 909	/* Setup TLB control */
 910	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 911		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 912		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 913		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 914	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 915	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 916	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 917	if (rdev->family == CHIP_RV740)
 918		WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
 919	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 920	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 921	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 922	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 923	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 924	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 925	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 926	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 927				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 928	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 929			(u32)(rdev->dummy_page.addr >> 12));
 930	for (i = 1; i < 7; i++)
 931		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 932
 933	r600_pcie_gart_tlb_flush(rdev);
 934	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
 935		 (unsigned)(rdev->mc.gtt_size >> 20),
 936		 (unsigned long long)rdev->gart.table_addr);
 937	rdev->gart.ready = true;
 938	return 0;
 939}
 940
 941static void rv770_pcie_gart_disable(struct radeon_device *rdev)
 942{
 943	u32 tmp;
 944	int i;
 945
 946	/* Disable all tables */
 947	for (i = 0; i < 7; i++)
 948		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 949
 950	/* Setup L2 cache */
 951	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 952				EFFECTIVE_L2_QUEUE_SIZE(7));
 953	WREG32(VM_L2_CNTL2, 0);
 954	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 955	/* Setup TLB control */
 956	tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 957	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 958	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 959	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 960	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 961	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 962	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 963	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 964	radeon_gart_table_vram_unpin(rdev);
 965}
 966
 967static void rv770_pcie_gart_fini(struct radeon_device *rdev)
 968{
 969	radeon_gart_fini(rdev);
 970	rv770_pcie_gart_disable(rdev);
 971	radeon_gart_table_vram_free(rdev);
 972}
 973
 974
 975static void rv770_agp_enable(struct radeon_device *rdev)
 976{
 977	u32 tmp;
 978	int i;
 979
 980	/* Setup L2 cache */
 981	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 982				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 983				EFFECTIVE_L2_QUEUE_SIZE(7));
 984	WREG32(VM_L2_CNTL2, 0);
 985	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 986	/* Setup TLB control */
 987	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 988		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 989		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 990		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 991	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 992	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 993	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 994	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 995	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 996	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 997	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 998	for (i = 0; i < 7; i++)
 999		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1000}
1001
1002static void rv770_mc_program(struct radeon_device *rdev)
1003{
1004	struct rv515_mc_save save;
1005	u32 tmp;
1006	int i, j;
1007
1008	/* Initialize HDP */
1009	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1010		WREG32((0x2c14 + j), 0x00000000);
1011		WREG32((0x2c18 + j), 0x00000000);
1012		WREG32((0x2c1c + j), 0x00000000);
1013		WREG32((0x2c20 + j), 0x00000000);
1014		WREG32((0x2c24 + j), 0x00000000);
1015	}
1016	/* r7xx hw bug.  Read from HDP_DEBUG1 rather
1017	 * than writing to HDP_REG_COHERENCY_FLUSH_CNTL
1018	 */
1019	tmp = RREG32(HDP_DEBUG1);
1020
1021	rv515_mc_stop(rdev, &save);
1022	if (r600_mc_wait_for_idle(rdev)) {
1023		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1024	}
1025	/* Lockout access through VGA aperture*/
1026	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1027	/* Update configuration */
1028	if (rdev->flags & RADEON_IS_AGP) {
1029		if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1030			/* VRAM before AGP */
1031			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1032				rdev->mc.vram_start >> 12);
1033			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1034				rdev->mc.gtt_end >> 12);
1035		} else {
1036			/* VRAM after AGP */
1037			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1038				rdev->mc.gtt_start >> 12);
1039			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1040				rdev->mc.vram_end >> 12);
1041		}
1042	} else {
1043		WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1044			rdev->mc.vram_start >> 12);
1045		WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1046			rdev->mc.vram_end >> 12);
1047	}
1048	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1049	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1050	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1051	WREG32(MC_VM_FB_LOCATION, tmp);
1052	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1053	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1054	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1055	if (rdev->flags & RADEON_IS_AGP) {
1056		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1057		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1058		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1059	} else {
1060		WREG32(MC_VM_AGP_BASE, 0);
1061		WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1062		WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1063	}
1064	if (r600_mc_wait_for_idle(rdev)) {
1065		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1066	}
1067	rv515_mc_resume(rdev, &save);
1068	/* we need to own VRAM, so turn off the VGA renderer here
1069	 * to stop it overwriting our objects */
1070	rv515_vga_render_disable(rdev);
1071}
1072
1073
1074/*
1075 * CP.
1076 */
1077void r700_cp_stop(struct radeon_device *rdev)
1078{
1079	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1080		radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1081	WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1082	WREG32(SCRATCH_UMSK, 0);
1083	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1084}
1085
1086static int rv770_cp_load_microcode(struct radeon_device *rdev)
1087{
1088	const __be32 *fw_data;
1089	int i;
1090
1091	if (!rdev->me_fw || !rdev->pfp_fw)
1092		return -EINVAL;
1093
1094	r700_cp_stop(rdev);
1095	WREG32(CP_RB_CNTL,
1096#ifdef __BIG_ENDIAN
1097	       BUF_SWAP_32BIT |
1098#endif
1099	       RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1100
1101	/* Reset cp */
1102	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1103	RREG32(GRBM_SOFT_RESET);
1104	mdelay(15);
1105	WREG32(GRBM_SOFT_RESET, 0);
1106
1107	fw_data = (const __be32 *)rdev->pfp_fw->data;
1108	WREG32(CP_PFP_UCODE_ADDR, 0);
1109	for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
1110		WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1111	WREG32(CP_PFP_UCODE_ADDR, 0);
1112
1113	fw_data = (const __be32 *)rdev->me_fw->data;
1114	WREG32(CP_ME_RAM_WADDR, 0);
1115	for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
1116		WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1117
1118	WREG32(CP_PFP_UCODE_ADDR, 0);
1119	WREG32(CP_ME_RAM_WADDR, 0);
1120	WREG32(CP_ME_RAM_RADDR, 0);
1121	return 0;
1122}
1123
1124void r700_cp_fini(struct radeon_device *rdev)
1125{
1126	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1127	r700_cp_stop(rdev);
1128	radeon_ring_fini(rdev, ring);
1129	radeon_scratch_free(rdev, ring->rptr_save_reg);
1130}
1131
1132void rv770_set_clk_bypass_mode(struct radeon_device *rdev)
1133{
1134	u32 tmp, i;
1135
1136	if (rdev->flags & RADEON_IS_IGP)
1137		return;
1138
1139	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
1140	tmp &= SCLK_MUX_SEL_MASK;
1141	tmp |= SCLK_MUX_SEL(1) | SCLK_MUX_UPDATE;
1142	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1143
1144	for (i = 0; i < rdev->usec_timeout; i++) {
1145		if (RREG32(CG_SPLL_STATUS) & SPLL_CHG_STATUS)
1146			break;
1147		udelay(1);
1148	}
1149
1150	tmp &= ~SCLK_MUX_UPDATE;
1151	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1152
1153	tmp = RREG32(MPLL_CNTL_MODE);
1154	if ((rdev->family == CHIP_RV710) || (rdev->family == CHIP_RV730))
1155		tmp &= ~RV730_MPLL_MCLK_SEL;
1156	else
1157		tmp &= ~MPLL_MCLK_SEL;
1158	WREG32(MPLL_CNTL_MODE, tmp);
1159}
1160
1161/*
1162 * Core functions
1163 */
1164static void rv770_gpu_init(struct radeon_device *rdev)
1165{
1166	int i, j, num_qd_pipes;
1167	u32 ta_aux_cntl;
1168	u32 sx_debug_1;
1169	u32 smx_dc_ctl0;
1170	u32 db_debug3;
1171	u32 num_gs_verts_per_thread;
1172	u32 vgt_gs_per_es;
1173	u32 gs_prim_buffer_depth = 0;
1174	u32 sq_ms_fifo_sizes;
1175	u32 sq_config;
1176	u32 sq_thread_resource_mgmt;
1177	u32 hdp_host_path_cntl;
1178	u32 sq_dyn_gpr_size_simd_ab_0;
1179	u32 gb_tiling_config = 0;
1180	u32 cc_gc_shader_pipe_config = 0;
1181	u32 mc_arb_ramcfg;
1182	u32 db_debug4, tmp;
1183	u32 inactive_pipes, shader_pipe_config;
1184	u32 disabled_rb_mask;
1185	unsigned active_number;
1186
1187	/* setup chip specs */
1188	rdev->config.rv770.tiling_group_size = 256;
1189	switch (rdev->family) {
1190	case CHIP_RV770:
1191		rdev->config.rv770.max_pipes = 4;
1192		rdev->config.rv770.max_tile_pipes = 8;
1193		rdev->config.rv770.max_simds = 10;
1194		rdev->config.rv770.max_backends = 4;
1195		rdev->config.rv770.max_gprs = 256;
1196		rdev->config.rv770.max_threads = 248;
1197		rdev->config.rv770.max_stack_entries = 512;
1198		rdev->config.rv770.max_hw_contexts = 8;
1199		rdev->config.rv770.max_gs_threads = 16 * 2;
1200		rdev->config.rv770.sx_max_export_size = 128;
1201		rdev->config.rv770.sx_max_export_pos_size = 16;
1202		rdev->config.rv770.sx_max_export_smx_size = 112;
1203		rdev->config.rv770.sq_num_cf_insts = 2;
1204
1205		rdev->config.rv770.sx_num_of_sets = 7;
1206		rdev->config.rv770.sc_prim_fifo_size = 0xF9;
1207		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1208		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1209		break;
1210	case CHIP_RV730:
1211		rdev->config.rv770.max_pipes = 2;
1212		rdev->config.rv770.max_tile_pipes = 4;
1213		rdev->config.rv770.max_simds = 8;
1214		rdev->config.rv770.max_backends = 2;
1215		rdev->config.rv770.max_gprs = 128;
1216		rdev->config.rv770.max_threads = 248;
1217		rdev->config.rv770.max_stack_entries = 256;
1218		rdev->config.rv770.max_hw_contexts = 8;
1219		rdev->config.rv770.max_gs_threads = 16 * 2;
1220		rdev->config.rv770.sx_max_export_size = 256;
1221		rdev->config.rv770.sx_max_export_pos_size = 32;
1222		rdev->config.rv770.sx_max_export_smx_size = 224;
1223		rdev->config.rv770.sq_num_cf_insts = 2;
1224
1225		rdev->config.rv770.sx_num_of_sets = 7;
1226		rdev->config.rv770.sc_prim_fifo_size = 0xf9;
1227		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1228		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1229		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1230			rdev->config.rv770.sx_max_export_pos_size -= 16;
1231			rdev->config.rv770.sx_max_export_smx_size += 16;
1232		}
1233		break;
1234	case CHIP_RV710:
1235		rdev->config.rv770.max_pipes = 2;
1236		rdev->config.rv770.max_tile_pipes = 2;
1237		rdev->config.rv770.max_simds = 2;
1238		rdev->config.rv770.max_backends = 1;
1239		rdev->config.rv770.max_gprs = 256;
1240		rdev->config.rv770.max_threads = 192;
1241		rdev->config.rv770.max_stack_entries = 256;
1242		rdev->config.rv770.max_hw_contexts = 4;
1243		rdev->config.rv770.max_gs_threads = 8 * 2;
1244		rdev->config.rv770.sx_max_export_size = 128;
1245		rdev->config.rv770.sx_max_export_pos_size = 16;
1246		rdev->config.rv770.sx_max_export_smx_size = 112;
1247		rdev->config.rv770.sq_num_cf_insts = 1;
1248
1249		rdev->config.rv770.sx_num_of_sets = 7;
1250		rdev->config.rv770.sc_prim_fifo_size = 0x40;
1251		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1252		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1253		break;
1254	case CHIP_RV740:
1255		rdev->config.rv770.max_pipes = 4;
1256		rdev->config.rv770.max_tile_pipes = 4;
1257		rdev->config.rv770.max_simds = 8;
1258		rdev->config.rv770.max_backends = 4;
1259		rdev->config.rv770.max_gprs = 256;
1260		rdev->config.rv770.max_threads = 248;
1261		rdev->config.rv770.max_stack_entries = 512;
1262		rdev->config.rv770.max_hw_contexts = 8;
1263		rdev->config.rv770.max_gs_threads = 16 * 2;
1264		rdev->config.rv770.sx_max_export_size = 256;
1265		rdev->config.rv770.sx_max_export_pos_size = 32;
1266		rdev->config.rv770.sx_max_export_smx_size = 224;
1267		rdev->config.rv770.sq_num_cf_insts = 2;
1268
1269		rdev->config.rv770.sx_num_of_sets = 7;
1270		rdev->config.rv770.sc_prim_fifo_size = 0x100;
1271		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1272		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1273
1274		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1275			rdev->config.rv770.sx_max_export_pos_size -= 16;
1276			rdev->config.rv770.sx_max_export_smx_size += 16;
1277		}
1278		break;
1279	default:
1280		break;
1281	}
1282
1283	/* Initialize HDP */
1284	j = 0;
1285	for (i = 0; i < 32; i++) {
1286		WREG32((0x2c14 + j), 0x00000000);
1287		WREG32((0x2c18 + j), 0x00000000);
1288		WREG32((0x2c1c + j), 0x00000000);
1289		WREG32((0x2c20 + j), 0x00000000);
1290		WREG32((0x2c24 + j), 0x00000000);
1291		j += 0x18;
1292	}
1293
1294	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1295
1296	/* setup tiling, simd, pipe config */
1297	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1298
1299	shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG);
1300	inactive_pipes = (shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> INACTIVE_QD_PIPES_SHIFT;
1301	for (i = 0, tmp = 1, active_number = 0; i < R7XX_MAX_PIPES; i++) {
1302		if (!(inactive_pipes & tmp)) {
1303			active_number++;
1304		}
1305		tmp <<= 1;
1306	}
1307	if (active_number == 1) {
1308		WREG32(SPI_CONFIG_CNTL, DISABLE_INTERP_1);
1309	} else {
1310		WREG32(SPI_CONFIG_CNTL, 0);
1311	}
1312
1313	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1314	tmp = rdev->config.rv770.max_simds -
1315		r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R7XX_MAX_SIMDS_MASK);
1316	rdev->config.rv770.active_simds = tmp;
1317
1318	switch (rdev->config.rv770.max_tile_pipes) {
1319	case 1:
1320	default:
1321		gb_tiling_config = PIPE_TILING(0);
1322		break;
1323	case 2:
1324		gb_tiling_config = PIPE_TILING(1);
1325		break;
1326	case 4:
1327		gb_tiling_config = PIPE_TILING(2);
1328		break;
1329	case 8:
1330		gb_tiling_config = PIPE_TILING(3);
1331		break;
1332	}
1333	rdev->config.rv770.tiling_npipes = rdev->config.rv770.max_tile_pipes;
1334
1335	disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R7XX_MAX_BACKENDS_MASK;
1336	tmp = 0;
1337	for (i = 0; i < rdev->config.rv770.max_backends; i++)
1338		tmp |= (1 << i);
1339	/* if all the backends are disabled, fix it up here */
1340	if ((disabled_rb_mask & tmp) == tmp) {
1341		for (i = 0; i < rdev->config.rv770.max_backends; i++)
1342			disabled_rb_mask &= ~(1 << i);
1343	}
1344	tmp = (gb_tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1345	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.rv770.max_backends,
1346					R7XX_MAX_BACKENDS, disabled_rb_mask);
1347	gb_tiling_config |= tmp << 16;
1348	rdev->config.rv770.backend_map = tmp;
1349
1350	if (rdev->family == CHIP_RV770)
1351		gb_tiling_config |= BANK_TILING(1);
1352	else {
1353		if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
1354			gb_tiling_config |= BANK_TILING(1);
1355		else
1356			gb_tiling_config |= BANK_TILING(0);
1357	}
1358	rdev->config.rv770.tiling_nbanks = 4 << ((gb_tiling_config >> 4) & 0x3);
1359	gb_tiling_config |= GROUP_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1360	if (((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT) > 3) {
1361		gb_tiling_config |= ROW_TILING(3);
1362		gb_tiling_config |= SAMPLE_SPLIT(3);
1363	} else {
1364		gb_tiling_config |=
1365			ROW_TILING(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1366		gb_tiling_config |=
1367			SAMPLE_SPLIT(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1368	}
1369
1370	gb_tiling_config |= BANK_SWAPS(1);
1371	rdev->config.rv770.tile_config = gb_tiling_config;
1372
1373	WREG32(GB_TILING_CONFIG, gb_tiling_config);
1374	WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1375	WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1376	WREG32(DMA_TILING_CONFIG, (gb_tiling_config & 0xffff));
1377	WREG32(DMA_TILING_CONFIG2, (gb_tiling_config & 0xffff));
1378	if (rdev->family == CHIP_RV730) {
1379		WREG32(UVD_UDEC_DB_TILING_CONFIG, (gb_tiling_config & 0xffff));
1380		WREG32(UVD_UDEC_DBW_TILING_CONFIG, (gb_tiling_config & 0xffff));
1381		WREG32(UVD_UDEC_TILING_CONFIG, (gb_tiling_config & 0xffff));
1382	}
1383
1384	WREG32(CGTS_SYS_TCC_DISABLE, 0);
1385	WREG32(CGTS_TCC_DISABLE, 0);
1386	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
1387	WREG32(CGTS_USER_TCC_DISABLE, 0);
1388
1389
1390	num_qd_pipes = R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1391	WREG32(VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & DEALLOC_DIST_MASK);
1392	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1393
1394	/* set HW defaults for 3D engine */
1395	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
1396				     ROQ_IB2_START(0x2b)));
1397
1398	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
1399
1400	ta_aux_cntl = RREG32(TA_CNTL_AUX);
1401	WREG32(TA_CNTL_AUX, ta_aux_cntl | DISABLE_CUBE_ANISO);
1402
1403	sx_debug_1 = RREG32(SX_DEBUG_1);
1404	sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1405	WREG32(SX_DEBUG_1, sx_debug_1);
1406
1407	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1408	smx_dc_ctl0 &= ~CACHE_DEPTH(0x1ff);
1409	smx_dc_ctl0 |= CACHE_DEPTH((rdev->config.rv770.sx_num_of_sets * 64) - 1);
1410	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1411
1412	if (rdev->family != CHIP_RV740)
1413		WREG32(SMX_EVENT_CTL, (ES_FLUSH_CTL(4) |
1414				       GS_FLUSH_CTL(4) |
1415				       ACK_FLUSH_CTL(3) |
1416				       SYNC_FLUSH_CTL));
1417
1418	if (rdev->family != CHIP_RV770)
1419		WREG32(SMX_SAR_CTL0, 0x00003f3f);
1420
1421	db_debug3 = RREG32(DB_DEBUG3);
1422	db_debug3 &= ~DB_CLK_OFF_DELAY(0x1f);
1423	switch (rdev->family) {
1424	case CHIP_RV770:
1425	case CHIP_RV740:
1426		db_debug3 |= DB_CLK_OFF_DELAY(0x1f);
1427		break;
1428	case CHIP_RV710:
1429	case CHIP_RV730:
1430	default:
1431		db_debug3 |= DB_CLK_OFF_DELAY(2);
1432		break;
1433	}
1434	WREG32(DB_DEBUG3, db_debug3);
1435
1436	if (rdev->family != CHIP_RV770) {
1437		db_debug4 = RREG32(DB_DEBUG4);
1438		db_debug4 |= DISABLE_TILE_COVERED_FOR_PS_ITER;
1439		WREG32(DB_DEBUG4, db_debug4);
1440	}
1441
1442	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.rv770.sx_max_export_size / 4) - 1) |
1443					POSITION_BUFFER_SIZE((rdev->config.rv770.sx_max_export_pos_size / 4) - 1) |
1444					SMX_BUFFER_SIZE((rdev->config.rv770.sx_max_export_smx_size / 4) - 1)));
1445
1446	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.rv770.sc_prim_fifo_size) |
1447				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_hiz_tile_fifo_size) |
1448				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_earlyz_tile_fifo_fize)));
1449
1450	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1451
1452	WREG32(VGT_NUM_INSTANCES, 1);
1453
1454	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
1455
1456	WREG32(CP_PERFMON_CNTL, 0);
1457
1458	sq_ms_fifo_sizes = (CACHE_FIFO_SIZE(16 * rdev->config.rv770.sq_num_cf_insts) |
1459			    DONE_FIFO_HIWATER(0xe0) |
1460			    ALU_UPDATE_FIFO_HIWATER(0x8));
1461	switch (rdev->family) {
1462	case CHIP_RV770:
1463	case CHIP_RV730:
1464	case CHIP_RV710:
1465		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x1);
1466		break;
1467	case CHIP_RV740:
1468	default:
1469		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x4);
1470		break;
1471	}
1472	WREG32(SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
1473
1474	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1475	 * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1476	 */
1477	sq_config = RREG32(SQ_CONFIG);
1478	sq_config &= ~(PS_PRIO(3) |
1479		       VS_PRIO(3) |
1480		       GS_PRIO(3) |
1481		       ES_PRIO(3));
1482	sq_config |= (DX9_CONSTS |
1483		      VC_ENABLE |
1484		      EXPORT_SRC_C |
1485		      PS_PRIO(0) |
1486		      VS_PRIO(1) |
1487		      GS_PRIO(2) |
1488		      ES_PRIO(3));
1489	if (rdev->family == CHIP_RV710)
1490		/* no vertex cache */
1491		sq_config &= ~VC_ENABLE;
1492
1493	WREG32(SQ_CONFIG, sq_config);
1494
1495	WREG32(SQ_GPR_RESOURCE_MGMT_1,  (NUM_PS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1496					 NUM_VS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1497					 NUM_CLAUSE_TEMP_GPRS(((rdev->config.rv770.max_gprs * 24)/64)/2)));
1498
1499	WREG32(SQ_GPR_RESOURCE_MGMT_2,  (NUM_GS_GPRS((rdev->config.rv770.max_gprs * 7)/64) |
1500					 NUM_ES_GPRS((rdev->config.rv770.max_gprs * 7)/64)));
1501
1502	sq_thread_resource_mgmt = (NUM_PS_THREADS((rdev->config.rv770.max_threads * 4)/8) |
1503				   NUM_VS_THREADS((rdev->config.rv770.max_threads * 2)/8) |
1504				   NUM_ES_THREADS((rdev->config.rv770.max_threads * 1)/8));
1505	if (((rdev->config.rv770.max_threads * 1) / 8) > rdev->config.rv770.max_gs_threads)
1506		sq_thread_resource_mgmt |= NUM_GS_THREADS(rdev->config.rv770.max_gs_threads);
1507	else
1508		sq_thread_resource_mgmt |= NUM_GS_THREADS((rdev->config.rv770.max_gs_threads * 1)/8);
1509	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1510
1511	WREG32(SQ_STACK_RESOURCE_MGMT_1, (NUM_PS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1512						     NUM_VS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1513
1514	WREG32(SQ_STACK_RESOURCE_MGMT_2, (NUM_GS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1515						     NUM_ES_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1516
1517	sq_dyn_gpr_size_simd_ab_0 = (SIMDA_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1518				     SIMDA_RING1((rdev->config.rv770.max_gprs * 38)/64) |
1519				     SIMDB_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1520				     SIMDB_RING1((rdev->config.rv770.max_gprs * 38)/64));
1521
1522	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
1523	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
1524	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
1525	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
1526	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
1527	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
1528	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
1529	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
1530
1531	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1532					  FORCE_EOV_MAX_REZ_CNT(255)));
1533
1534	if (rdev->family == CHIP_RV710)
1535		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(TC_ONLY) |
1536						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1537	else
1538		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(VC_AND_TC) |
1539						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1540
1541	switch (rdev->family) {
1542	case CHIP_RV770:
1543	case CHIP_RV730:
1544	case CHIP_RV740:
1545		gs_prim_buffer_depth = 384;
1546		break;
1547	case CHIP_RV710:
1548		gs_prim_buffer_depth = 128;
1549		break;
1550	default:
1551		break;
1552	}
1553
1554	num_gs_verts_per_thread = rdev->config.rv770.max_pipes * 16;
1555	vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1556	/* Max value for this is 256 */
1557	if (vgt_gs_per_es > 256)
1558		vgt_gs_per_es = 256;
1559
1560	WREG32(VGT_ES_PER_GS, 128);
1561	WREG32(VGT_GS_PER_ES, vgt_gs_per_es);
1562	WREG32(VGT_GS_PER_VS, 2);
1563
1564	/* more default values. 2D/3D driver should adjust as needed */
1565	WREG32(VGT_GS_VERTEX_REUSE, 16);
1566	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1567	WREG32(VGT_STRMOUT_EN, 0);
1568	WREG32(SX_MISC, 0);
1569	WREG32(PA_SC_MODE_CNTL, 0);
1570	WREG32(PA_SC_EDGERULE, 0xaaaaaaaa);
1571	WREG32(PA_SC_AA_CONFIG, 0);
1572	WREG32(PA_SC_CLIPRECT_RULE, 0xffff);
1573	WREG32(PA_SC_LINE_STIPPLE, 0);
1574	WREG32(SPI_INPUT_Z, 0);
1575	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1576	WREG32(CB_COLOR7_FRAG, 0);
1577
1578	/* clear render buffer base addresses */
1579	WREG32(CB_COLOR0_BASE, 0);
1580	WREG32(CB_COLOR1_BASE, 0);
1581	WREG32(CB_COLOR2_BASE, 0);
1582	WREG32(CB_COLOR3_BASE, 0);
1583	WREG32(CB_COLOR4_BASE, 0);
1584	WREG32(CB_COLOR5_BASE, 0);
1585	WREG32(CB_COLOR6_BASE, 0);
1586	WREG32(CB_COLOR7_BASE, 0);
1587
1588	WREG32(TCP_CNTL, 0);
1589
1590	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1591	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1592
1593	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1594
1595	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1596					  NUM_CLIP_SEQ(3)));
1597	WREG32(VC_ENHANCE, 0);
1598}
1599
1600void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1601{
1602	u64 size_bf, size_af;
1603
1604	if (mc->mc_vram_size > 0xE0000000) {
1605		/* leave room for at least 512M GTT */
1606		dev_warn(rdev->dev, "limiting VRAM\n");
1607		mc->real_vram_size = 0xE0000000;
1608		mc->mc_vram_size = 0xE0000000;
1609	}
1610	if (rdev->flags & RADEON_IS_AGP) {
1611		size_bf = mc->gtt_start;
1612		size_af = mc->mc_mask - mc->gtt_end;
1613		if (size_bf > size_af) {
1614			if (mc->mc_vram_size > size_bf) {
1615				dev_warn(rdev->dev, "limiting VRAM\n");
1616				mc->real_vram_size = size_bf;
1617				mc->mc_vram_size = size_bf;
1618			}
1619			mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1620		} else {
1621			if (mc->mc_vram_size > size_af) {
1622				dev_warn(rdev->dev, "limiting VRAM\n");
1623				mc->real_vram_size = size_af;
1624				mc->mc_vram_size = size_af;
1625			}
1626			mc->vram_start = mc->gtt_end + 1;
1627		}
1628		mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1629		dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1630				mc->mc_vram_size >> 20, mc->vram_start,
1631				mc->vram_end, mc->real_vram_size >> 20);
1632	} else {
1633		radeon_vram_location(rdev, &rdev->mc, 0);
1634		rdev->mc.gtt_base_align = 0;
1635		radeon_gtt_location(rdev, mc);
1636	}
1637}
1638
1639static int rv770_mc_init(struct radeon_device *rdev)
1640{
1641	u32 tmp;
1642	int chansize, numchan;
1643
1644	/* Get VRAM informations */
1645	rdev->mc.vram_is_ddr = true;
1646	tmp = RREG32(MC_ARB_RAMCFG);
1647	if (tmp & CHANSIZE_OVERRIDE) {
1648		chansize = 16;
1649	} else if (tmp & CHANSIZE_MASK) {
1650		chansize = 64;
1651	} else {
1652		chansize = 32;
1653	}
1654	tmp = RREG32(MC_SHARED_CHMAP);
1655	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1656	case 0:
1657	default:
1658		numchan = 1;
1659		break;
1660	case 1:
1661		numchan = 2;
1662		break;
1663	case 2:
1664		numchan = 4;
1665		break;
1666	case 3:
1667		numchan = 8;
1668		break;
1669	}
1670	rdev->mc.vram_width = numchan * chansize;
1671	/* Could aper size report 0 ? */
1672	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1673	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1674	/* Setup GPU memory space */
1675	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1676	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1677	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1678	r700_vram_gtt_location(rdev, &rdev->mc);
1679	radeon_update_bandwidth_info(rdev);
1680
1681	return 0;
1682}
1683
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1684static int rv770_startup(struct radeon_device *rdev)
1685{
1686	struct radeon_ring *ring;
1687	int r;
1688
1689	/* enable pcie gen2 link */
1690	rv770_pcie_gen2_enable(rdev);
1691
1692	/* scratch needs to be initialized before MC */
1693	r = r600_vram_scratch_init(rdev);
1694	if (r)
1695		return r;
1696
1697	rv770_mc_program(rdev);
1698
1699	if (rdev->flags & RADEON_IS_AGP) {
1700		rv770_agp_enable(rdev);
1701	} else {
1702		r = rv770_pcie_gart_enable(rdev);
1703		if (r)
1704			return r;
1705	}
1706
1707	rv770_gpu_init(rdev);
1708
1709	/* allocate wb buffer */
1710	r = radeon_wb_init(rdev);
1711	if (r)
1712		return r;
1713
1714	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1715	if (r) {
1716		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1717		return r;
1718	}
1719
1720	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
1721	if (r) {
1722		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1723		return r;
1724	}
1725
1726	r = uvd_v2_2_resume(rdev);
1727	if (!r) {
1728		r = radeon_fence_driver_start_ring(rdev,
1729						   R600_RING_TYPE_UVD_INDEX);
1730		if (r)
1731			dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
1732	}
1733
1734	if (r)
1735		rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
1736
1737	/* Enable IRQ */
1738	if (!rdev->irq.installed) {
1739		r = radeon_irq_kms_init(rdev);
1740		if (r)
1741			return r;
1742	}
1743
1744	r = r600_irq_init(rdev);
1745	if (r) {
1746		DRM_ERROR("radeon: IH init failed (%d).\n", r);
1747		radeon_irq_kms_fini(rdev);
1748		return r;
1749	}
1750	r600_irq_set(rdev);
1751
1752	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1753	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1754			     RADEON_CP_PACKET2);
1755	if (r)
1756		return r;
1757
1758	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1759	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
1760			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1761	if (r)
1762		return r;
1763
1764	r = rv770_cp_load_microcode(rdev);
1765	if (r)
1766		return r;
1767	r = r600_cp_resume(rdev);
1768	if (r)
1769		return r;
1770
1771	r = r600_dma_resume(rdev);
1772	if (r)
1773		return r;
1774
1775	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
1776	if (ring->ring_size) {
1777		r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
1778				     RADEON_CP_PACKET2);
1779		if (!r)
1780			r = uvd_v1_0_init(rdev);
1781
1782		if (r)
1783			DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
1784	}
1785
1786	r = radeon_ib_pool_init(rdev);
1787	if (r) {
1788		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1789		return r;
1790	}
1791
1792	r = radeon_audio_init(rdev);
1793	if (r) {
1794		DRM_ERROR("radeon: audio init failed\n");
1795		return r;
1796	}
1797
1798	return 0;
1799}
1800
1801int rv770_resume(struct radeon_device *rdev)
1802{
1803	int r;
1804
1805	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
1806	 * posting will perform necessary task to bring back GPU into good
1807	 * shape.
1808	 */
1809	/* post card */
1810	atom_asic_init(rdev->mode_info.atom_context);
1811
1812	/* init golden registers */
1813	rv770_init_golden_registers(rdev);
1814
1815	if (rdev->pm.pm_method == PM_METHOD_DPM)
1816		radeon_pm_resume(rdev);
1817
1818	rdev->accel_working = true;
1819	r = rv770_startup(rdev);
1820	if (r) {
1821		DRM_ERROR("r600 startup failed on resume\n");
1822		rdev->accel_working = false;
1823		return r;
1824	}
1825
1826	return r;
1827
1828}
1829
1830int rv770_suspend(struct radeon_device *rdev)
1831{
1832	radeon_pm_suspend(rdev);
1833	radeon_audio_fini(rdev);
1834	uvd_v1_0_fini(rdev);
1835	radeon_uvd_suspend(rdev);
 
 
1836	r700_cp_stop(rdev);
1837	r600_dma_stop(rdev);
1838	r600_irq_suspend(rdev);
1839	radeon_wb_disable(rdev);
1840	rv770_pcie_gart_disable(rdev);
1841
1842	return 0;
1843}
1844
1845/* Plan is to move initialization in that function and use
1846 * helper function so that radeon_device_init pretty much
1847 * do nothing more than calling asic specific function. This
1848 * should also allow to remove a bunch of callback function
1849 * like vram_info.
1850 */
1851int rv770_init(struct radeon_device *rdev)
1852{
1853	int r;
1854
1855	/* Read BIOS */
1856	if (!radeon_get_bios(rdev)) {
1857		if (ASIC_IS_AVIVO(rdev))
1858			return -EINVAL;
1859	}
1860	/* Must be an ATOMBIOS */
1861	if (!rdev->is_atom_bios) {
1862		dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
1863		return -EINVAL;
1864	}
1865	r = radeon_atombios_init(rdev);
1866	if (r)
1867		return r;
1868	/* Post card if necessary */
1869	if (!radeon_card_posted(rdev)) {
1870		if (!rdev->bios) {
1871			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
1872			return -EINVAL;
1873		}
1874		DRM_INFO("GPU not posted. posting now...\n");
1875		atom_asic_init(rdev->mode_info.atom_context);
1876	}
1877	/* init golden registers */
1878	rv770_init_golden_registers(rdev);
1879	/* Initialize scratch registers */
1880	r600_scratch_init(rdev);
1881	/* Initialize surface registers */
1882	radeon_surface_init(rdev);
1883	/* Initialize clocks */
1884	radeon_get_clock_info(rdev->ddev);
1885	/* Fence driver */
1886	r = radeon_fence_driver_init(rdev);
1887	if (r)
1888		return r;
1889	/* initialize AGP */
1890	if (rdev->flags & RADEON_IS_AGP) {
1891		r = radeon_agp_init(rdev);
1892		if (r)
1893			radeon_agp_disable(rdev);
1894	}
1895	r = rv770_mc_init(rdev);
1896	if (r)
1897		return r;
1898	/* Memory manager */
1899	r = radeon_bo_init(rdev);
1900	if (r)
1901		return r;
1902
1903	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1904		r = r600_init_microcode(rdev);
1905		if (r) {
1906			DRM_ERROR("Failed to load firmware!\n");
1907			return r;
1908		}
1909	}
1910
1911	/* Initialize power management */
1912	radeon_pm_init(rdev);
1913
1914	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
1915	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1916
1917	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
1918	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
1919
1920	r = radeon_uvd_init(rdev);
1921	if (!r) {
1922		rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
1923		r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
1924			       4096);
1925	}
1926
1927	rdev->ih.ring_obj = NULL;
1928	r600_ih_ring_init(rdev, 64 * 1024);
1929
1930	r = r600_pcie_gart_init(rdev);
1931	if (r)
1932		return r;
1933
1934	rdev->accel_working = true;
1935	r = rv770_startup(rdev);
1936	if (r) {
1937		dev_err(rdev->dev, "disabling GPU acceleration\n");
1938		r700_cp_fini(rdev);
1939		r600_dma_fini(rdev);
1940		r600_irq_fini(rdev);
1941		radeon_wb_fini(rdev);
1942		radeon_ib_pool_fini(rdev);
1943		radeon_irq_kms_fini(rdev);
1944		rv770_pcie_gart_fini(rdev);
1945		rdev->accel_working = false;
1946	}
1947
1948	return 0;
1949}
1950
1951void rv770_fini(struct radeon_device *rdev)
1952{
1953	radeon_pm_fini(rdev);
1954	r700_cp_fini(rdev);
1955	r600_dma_fini(rdev);
1956	r600_irq_fini(rdev);
1957	radeon_wb_fini(rdev);
1958	radeon_ib_pool_fini(rdev);
1959	radeon_irq_kms_fini(rdev);
1960	uvd_v1_0_fini(rdev);
1961	radeon_uvd_fini(rdev);
1962	rv770_pcie_gart_fini(rdev);
1963	r600_vram_scratch_fini(rdev);
1964	radeon_gem_fini(rdev);
1965	radeon_fence_driver_fini(rdev);
1966	radeon_agp_fini(rdev);
1967	radeon_bo_fini(rdev);
1968	radeon_atombios_fini(rdev);
1969	kfree(rdev->bios);
1970	rdev->bios = NULL;
1971}
1972
1973static void rv770_pcie_gen2_enable(struct radeon_device *rdev)
1974{
1975	u32 link_width_cntl, lanes, speed_cntl, tmp;
1976	u16 link_cntl2;
1977
1978	if (radeon_pcie_gen2 == 0)
1979		return;
1980
1981	if (rdev->flags & RADEON_IS_IGP)
1982		return;
1983
1984	if (!(rdev->flags & RADEON_IS_PCIE))
1985		return;
1986
1987	/* x2 cards have a special sequence */
1988	if (ASIC_IS_X2(rdev))
1989		return;
1990
1991	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
1992		(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
1993		return;
1994
1995	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
1996
1997	/* advertise upconfig capability */
1998	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1999	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2000	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2001	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2002	if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
2003		lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
2004		link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
2005				     LC_RECONFIG_ARC_MISSING_ESCAPE);
2006		link_width_cntl |= lanes | LC_RECONFIG_NOW |
2007			LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT;
2008		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2009	} else {
2010		link_width_cntl |= LC_UPCONFIGURE_DIS;
2011		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2012	}
2013
2014	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2015	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
2016	    (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
2017
2018		tmp = RREG32(0x541c);
2019		WREG32(0x541c, tmp | 0x8);
2020		WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
2021		link_cntl2 = RREG16(0x4088);
2022		link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
2023		link_cntl2 |= 0x2;
2024		WREG16(0x4088, link_cntl2);
2025		WREG32(MM_CFGREGS_CNTL, 0);
2026
2027		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2028		speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
2029		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2030
2031		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2032		speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
2033		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2034
2035		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2036		speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
2037		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2038
2039		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2040		speed_cntl |= LC_GEN2_EN_STRAP;
2041		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2042
2043	} else {
2044		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2045		/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
2046		if (1)
2047			link_width_cntl |= LC_UPCONFIGURE_DIS;
2048		else
2049			link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2050		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2051	}
2052}