1/*
   2 * Copyright 2008 Advanced Micro Devices, Inc.
   3 * Copyright 2008 Red Hat Inc.
   4 * Copyright 2009 Jerome Glisse.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 *
  24 * Authors: Dave Airlie
  25 *          Alex Deucher
  26 *          Jerome Glisse
  27 */
  28#include <linux/firmware.h>
  29#include <linux/platform_device.h>
  30#include <linux/slab.h>
  31#include "drmP.h"
  32#include "radeon.h"
  33#include "radeon_asic.h"
  34#include "radeon_drm.h"
  35#include "rv770d.h"
  36#include "atom.h"
  37#include "avivod.h"
  38
  39#define R700_PFP_UCODE_SIZE 848
  40#define R700_PM4_UCODE_SIZE 1360
  41
  42static void rv770_gpu_init(struct radeon_device *rdev);
  43void rv770_fini(struct radeon_device *rdev);
  44static void rv770_pcie_gen2_enable(struct radeon_device *rdev);
  45
  46u32 rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
  47{
  48	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
  49	u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
  50
  51	/* Lock the graphics update lock */
  52	tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
  53	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
  54
  55	/* update the scanout addresses */
  56	if (radeon_crtc->crtc_id) {
  57		WREG32(D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
  58		WREG32(D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
  59	} else {
  60		WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
  61		WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
  62	}
  63	WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
  64	       (u32)crtc_base);
  65	WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
  66	       (u32)crtc_base);
  67
  68	/* Wait for update_pending to go high. */
  69	while (!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING));
  70	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
  71
  72	/* Unlock the lock, so double-buffering can take place inside vblank */
  73	tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
  74	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
  75
  76	/* Return current update_pending status: */
  77	return RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING;
  78}
  79
  80/* get temperature in millidegrees */
  81int rv770_get_temp(struct radeon_device *rdev)
  82{
  83	u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
  84		ASIC_T_SHIFT;
  85	int actual_temp;
  86
  87	if (temp & 0x400)
  88		actual_temp = -256;
  89	else if (temp & 0x200)
  90		actual_temp = 255;
  91	else if (temp & 0x100) {
  92		actual_temp = temp & 0x1ff;
  93		actual_temp |= ~0x1ff;
  94	} else
  95		actual_temp = temp & 0xff;
  96
  97	return (actual_temp * 1000) / 2;
  98}
  99
 100void rv770_pm_misc(struct radeon_device *rdev)
 101{
 102	int req_ps_idx = rdev->pm.requested_power_state_index;
 103	int req_cm_idx = rdev->pm.requested_clock_mode_index;
 104	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
 105	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 106
 107	if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
 108		/* 0xff01 is a flag rather then an actual voltage */
 109		if (voltage->voltage == 0xff01)
 110			return;
 111		if (voltage->voltage != rdev->pm.current_vddc) {
 112			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
 113			rdev->pm.current_vddc = voltage->voltage;
 114			DRM_DEBUG("Setting: v: %d\n", voltage->voltage);
 115		}
 116	}
 117}
 118
 119/*
 120 * GART
 121 */
 122int rv770_pcie_gart_enable(struct radeon_device *rdev)
 123{
 124	u32 tmp;
 125	int r, i;
 126
 127	if (rdev->gart.table.vram.robj == NULL) {
 128		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 129		return -EINVAL;
 130	}
 131	r = radeon_gart_table_vram_pin(rdev);
 132	if (r)
 133		return r;
 134	radeon_gart_restore(rdev);
 135	/* Setup L2 cache */
 136	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 137				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 138				EFFECTIVE_L2_QUEUE_SIZE(7));
 139	WREG32(VM_L2_CNTL2, 0);
 140	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 141	/* Setup TLB control */
 142	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 143		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 144		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 145		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 146	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 147	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 148	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 149	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 150	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 151	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 152	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 153	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 154	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 155	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 156	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 157				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 158	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 159			(u32)(rdev->dummy_page.addr >> 12));
 160	for (i = 1; i < 7; i++)
 161		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 162
 163	r600_pcie_gart_tlb_flush(rdev);
 164	rdev->gart.ready = true;
 165	return 0;
 166}
 167
 168void rv770_pcie_gart_disable(struct radeon_device *rdev)
 169{
 170	u32 tmp;
 171	int i, r;
 172
 173	/* Disable all tables */
 174	for (i = 0; i < 7; i++)
 175		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 176
 177	/* Setup L2 cache */
 178	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 179				EFFECTIVE_L2_QUEUE_SIZE(7));
 180	WREG32(VM_L2_CNTL2, 0);
 181	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 182	/* Setup TLB control */
 183	tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 184	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 185	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 186	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 187	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 188	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 189	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 190	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 191	if (rdev->gart.table.vram.robj) {
 192		r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
 193		if (likely(r == 0)) {
 194			radeon_bo_kunmap(rdev->gart.table.vram.robj);
 195			radeon_bo_unpin(rdev->gart.table.vram.robj);
 196			radeon_bo_unreserve(rdev->gart.table.vram.robj);
 197		}
 198	}
 199}
 200
 201void rv770_pcie_gart_fini(struct radeon_device *rdev)
 202{
 203	radeon_gart_fini(rdev);
 204	rv770_pcie_gart_disable(rdev);
 205	radeon_gart_table_vram_free(rdev);
 206}
 207
 208
 209void rv770_agp_enable(struct radeon_device *rdev)
 210{
 211	u32 tmp;
 212	int i;
 213
 214	/* Setup L2 cache */
 215	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 216				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 217				EFFECTIVE_L2_QUEUE_SIZE(7));
 218	WREG32(VM_L2_CNTL2, 0);
 219	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 220	/* Setup TLB control */
 221	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 222		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 223		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 224		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 225	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 226	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 227	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 228	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 229	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 230	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 231	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 232	for (i = 0; i < 7; i++)
 233		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 234}
 235
 236static void rv770_mc_program(struct radeon_device *rdev)
 237{
 238	struct rv515_mc_save save;
 239	u32 tmp;
 240	int i, j;
 241
 242	/* Initialize HDP */
 243	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 244		WREG32((0x2c14 + j), 0x00000000);
 245		WREG32((0x2c18 + j), 0x00000000);
 246		WREG32((0x2c1c + j), 0x00000000);
 247		WREG32((0x2c20 + j), 0x00000000);
 248		WREG32((0x2c24 + j), 0x00000000);
 249	}
 250	/* r7xx hw bug.  Read from HDP_DEBUG1 rather
 251	 * than writing to HDP_REG_COHERENCY_FLUSH_CNTL
 252	 */
 253	tmp = RREG32(HDP_DEBUG1);
 254
 255	rv515_mc_stop(rdev, &save);
 256	if (r600_mc_wait_for_idle(rdev)) {
 257		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 258	}
 259	/* Lockout access through VGA aperture*/
 260	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
 261	/* Update configuration */
 262	if (rdev->flags & RADEON_IS_AGP) {
 263		if (rdev->mc.vram_start < rdev->mc.gtt_start) {
 264			/* VRAM before AGP */
 265			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 266				rdev->mc.vram_start >> 12);
 267			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 268				rdev->mc.gtt_end >> 12);
 269		} else {
 270			/* VRAM after AGP */
 271			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 272				rdev->mc.gtt_start >> 12);
 273			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 274				rdev->mc.vram_end >> 12);
 275		}
 276	} else {
 277		WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 278			rdev->mc.vram_start >> 12);
 279		WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 280			rdev->mc.vram_end >> 12);
 281	}
 282	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
 283	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
 284	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
 285	WREG32(MC_VM_FB_LOCATION, tmp);
 286	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
 287	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
 288	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
 289	if (rdev->flags & RADEON_IS_AGP) {
 290		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
 291		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
 292		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
 293	} else {
 294		WREG32(MC_VM_AGP_BASE, 0);
 295		WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
 296		WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
 297	}
 298	if (r600_mc_wait_for_idle(rdev)) {
 299		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 300	}
 301	rv515_mc_resume(rdev, &save);
 302	/* we need to own VRAM, so turn off the VGA renderer here
 303	 * to stop it overwriting our objects */
 304	rv515_vga_render_disable(rdev);
 305}
 306
 307
 308/*
 309 * CP.
 310 */
 311void r700_cp_stop(struct radeon_device *rdev)
 312{
 313	radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
 314	WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
 315	WREG32(SCRATCH_UMSK, 0);
 316}
 317
 318static int rv770_cp_load_microcode(struct radeon_device *rdev)
 319{
 320	const __be32 *fw_data;
 321	int i;
 322
 323	if (!rdev->me_fw || !rdev->pfp_fw)
 324		return -EINVAL;
 325
 326	r700_cp_stop(rdev);
 327	WREG32(CP_RB_CNTL,
 328#ifdef __BIG_ENDIAN
 329	       BUF_SWAP_32BIT |
 330#endif
 331	       RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
 332
 333	/* Reset cp */
 334	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
 335	RREG32(GRBM_SOFT_RESET);
 336	mdelay(15);
 337	WREG32(GRBM_SOFT_RESET, 0);
 338
 339	fw_data = (const __be32 *)rdev->pfp_fw->data;
 340	WREG32(CP_PFP_UCODE_ADDR, 0);
 341	for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
 342		WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
 343	WREG32(CP_PFP_UCODE_ADDR, 0);
 344
 345	fw_data = (const __be32 *)rdev->me_fw->data;
 346	WREG32(CP_ME_RAM_WADDR, 0);
 347	for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
 348		WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
 349
 350	WREG32(CP_PFP_UCODE_ADDR, 0);
 351	WREG32(CP_ME_RAM_WADDR, 0);
 352	WREG32(CP_ME_RAM_RADDR, 0);
 353	return 0;
 354}
 355
 356void r700_cp_fini(struct radeon_device *rdev)
 357{
 358	r700_cp_stop(rdev);
 359	radeon_ring_fini(rdev);
 360}
 361
 362/*
 363 * Core functions
 364 */
 365static u32 r700_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
 366					     u32 num_tile_pipes,
 367					     u32 num_backends,
 368					     u32 backend_disable_mask)
 369{
 370	u32 backend_map = 0;
 371	u32 enabled_backends_mask;
 372	u32 enabled_backends_count;
 373	u32 cur_pipe;
 374	u32 swizzle_pipe[R7XX_MAX_PIPES];
 375	u32 cur_backend;
 376	u32 i;
 377	bool force_no_swizzle;
 378
 379	if (num_tile_pipes > R7XX_MAX_PIPES)
 380		num_tile_pipes = R7XX_MAX_PIPES;
 381	if (num_tile_pipes < 1)
 382		num_tile_pipes = 1;
 383	if (num_backends > R7XX_MAX_BACKENDS)
 384		num_backends = R7XX_MAX_BACKENDS;
 385	if (num_backends < 1)
 386		num_backends = 1;
 387
 388	enabled_backends_mask = 0;
 389	enabled_backends_count = 0;
 390	for (i = 0; i < R7XX_MAX_BACKENDS; ++i) {
 391		if (((backend_disable_mask >> i) & 1) == 0) {
 392			enabled_backends_mask |= (1 << i);
 393			++enabled_backends_count;
 394		}
 395		if (enabled_backends_count == num_backends)
 396			break;
 397	}
 398
 399	if (enabled_backends_count == 0) {
 400		enabled_backends_mask = 1;
 401		enabled_backends_count = 1;
 402	}
 403
 404	if (enabled_backends_count != num_backends)
 405		num_backends = enabled_backends_count;
 406
 407	switch (rdev->family) {
 408	case CHIP_RV770:
 409	case CHIP_RV730:
 410		force_no_swizzle = false;
 411		break;
 412	case CHIP_RV710:
 413	case CHIP_RV740:
 414	default:
 415		force_no_swizzle = true;
 416		break;
 417	}
 418
 419	memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * R7XX_MAX_PIPES);
 420	switch (num_tile_pipes) {
 421	case 1:
 422		swizzle_pipe[0] = 0;
 423		break;
 424	case 2:
 425		swizzle_pipe[0] = 0;
 426		swizzle_pipe[1] = 1;
 427		break;
 428	case 3:
 429		if (force_no_swizzle) {
 430			swizzle_pipe[0] = 0;
 431			swizzle_pipe[1] = 1;
 432			swizzle_pipe[2] = 2;
 433		} else {
 434			swizzle_pipe[0] = 0;
 435			swizzle_pipe[1] = 2;
 436			swizzle_pipe[2] = 1;
 437		}
 438		break;
 439	case 4:
 440		if (force_no_swizzle) {
 441			swizzle_pipe[0] = 0;
 442			swizzle_pipe[1] = 1;
 443			swizzle_pipe[2] = 2;
 444			swizzle_pipe[3] = 3;
 445		} else {
 446			swizzle_pipe[0] = 0;
 447			swizzle_pipe[1] = 2;
 448			swizzle_pipe[2] = 3;
 449			swizzle_pipe[3] = 1;
 450		}
 451		break;
 452	case 5:
 453		if (force_no_swizzle) {
 454			swizzle_pipe[0] = 0;
 455			swizzle_pipe[1] = 1;
 456			swizzle_pipe[2] = 2;
 457			swizzle_pipe[3] = 3;
 458			swizzle_pipe[4] = 4;
 459		} else {
 460			swizzle_pipe[0] = 0;
 461			swizzle_pipe[1] = 2;
 462			swizzle_pipe[2] = 4;
 463			swizzle_pipe[3] = 1;
 464			swizzle_pipe[4] = 3;
 465		}
 466		break;
 467	case 6:
 468		if (force_no_swizzle) {
 469			swizzle_pipe[0] = 0;
 470			swizzle_pipe[1] = 1;
 471			swizzle_pipe[2] = 2;
 472			swizzle_pipe[3] = 3;
 473			swizzle_pipe[4] = 4;
 474			swizzle_pipe[5] = 5;
 475		} else {
 476			swizzle_pipe[0] = 0;
 477			swizzle_pipe[1] = 2;
 478			swizzle_pipe[2] = 4;
 479			swizzle_pipe[3] = 5;
 480			swizzle_pipe[4] = 3;
 481			swizzle_pipe[5] = 1;
 482		}
 483		break;
 484	case 7:
 485		if (force_no_swizzle) {
 486			swizzle_pipe[0] = 0;
 487			swizzle_pipe[1] = 1;
 488			swizzle_pipe[2] = 2;
 489			swizzle_pipe[3] = 3;
 490			swizzle_pipe[4] = 4;
 491			swizzle_pipe[5] = 5;
 492			swizzle_pipe[6] = 6;
 493		} else {
 494			swizzle_pipe[0] = 0;
 495			swizzle_pipe[1] = 2;
 496			swizzle_pipe[2] = 4;
 497			swizzle_pipe[3] = 6;
 498			swizzle_pipe[4] = 3;
 499			swizzle_pipe[5] = 1;
 500			swizzle_pipe[6] = 5;
 501		}
 502		break;
 503	case 8:
 504		if (force_no_swizzle) {
 505			swizzle_pipe[0] = 0;
 506			swizzle_pipe[1] = 1;
 507			swizzle_pipe[2] = 2;
 508			swizzle_pipe[3] = 3;
 509			swizzle_pipe[4] = 4;
 510			swizzle_pipe[5] = 5;
 511			swizzle_pipe[6] = 6;
 512			swizzle_pipe[7] = 7;
 513		} else {
 514			swizzle_pipe[0] = 0;
 515			swizzle_pipe[1] = 2;
 516			swizzle_pipe[2] = 4;
 517			swizzle_pipe[3] = 6;
 518			swizzle_pipe[4] = 3;
 519			swizzle_pipe[5] = 1;
 520			swizzle_pipe[6] = 7;
 521			swizzle_pipe[7] = 5;
 522		}
 523		break;
 524	}
 525
 526	cur_backend = 0;
 527	for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
 528		while (((1 << cur_backend) & enabled_backends_mask) == 0)
 529			cur_backend = (cur_backend + 1) % R7XX_MAX_BACKENDS;
 530
 531		backend_map |= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
 532
 533		cur_backend = (cur_backend + 1) % R7XX_MAX_BACKENDS;
 534	}
 535
 536	return backend_map;
 537}
 538
 539static void rv770_gpu_init(struct radeon_device *rdev)
 540{
 541	int i, j, num_qd_pipes;
 542	u32 ta_aux_cntl;
 543	u32 sx_debug_1;
 544	u32 smx_dc_ctl0;
 545	u32 db_debug3;
 546	u32 num_gs_verts_per_thread;
 547	u32 vgt_gs_per_es;
 548	u32 gs_prim_buffer_depth = 0;
 549	u32 sq_ms_fifo_sizes;
 550	u32 sq_config;
 551	u32 sq_thread_resource_mgmt;
 552	u32 hdp_host_path_cntl;
 553	u32 sq_dyn_gpr_size_simd_ab_0;
 554	u32 backend_map;
 555	u32 gb_tiling_config = 0;
 556	u32 cc_rb_backend_disable = 0;
 557	u32 cc_gc_shader_pipe_config = 0;
 558	u32 mc_arb_ramcfg;
 559	u32 db_debug4;
 560
 561	/* setup chip specs */
 562	switch (rdev->family) {
 563	case CHIP_RV770:
 564		rdev->config.rv770.max_pipes = 4;
 565		rdev->config.rv770.max_tile_pipes = 8;
 566		rdev->config.rv770.max_simds = 10;
 567		rdev->config.rv770.max_backends = 4;
 568		rdev->config.rv770.max_gprs = 256;
 569		rdev->config.rv770.max_threads = 248;
 570		rdev->config.rv770.max_stack_entries = 512;
 571		rdev->config.rv770.max_hw_contexts = 8;
 572		rdev->config.rv770.max_gs_threads = 16 * 2;
 573		rdev->config.rv770.sx_max_export_size = 128;
 574		rdev->config.rv770.sx_max_export_pos_size = 16;
 575		rdev->config.rv770.sx_max_export_smx_size = 112;
 576		rdev->config.rv770.sq_num_cf_insts = 2;
 577
 578		rdev->config.rv770.sx_num_of_sets = 7;
 579		rdev->config.rv770.sc_prim_fifo_size = 0xF9;
 580		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
 581		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
 582		break;
 583	case CHIP_RV730:
 584		rdev->config.rv770.max_pipes = 2;
 585		rdev->config.rv770.max_tile_pipes = 4;
 586		rdev->config.rv770.max_simds = 8;
 587		rdev->config.rv770.max_backends = 2;
 588		rdev->config.rv770.max_gprs = 128;
 589		rdev->config.rv770.max_threads = 248;
 590		rdev->config.rv770.max_stack_entries = 256;
 591		rdev->config.rv770.max_hw_contexts = 8;
 592		rdev->config.rv770.max_gs_threads = 16 * 2;
 593		rdev->config.rv770.sx_max_export_size = 256;
 594		rdev->config.rv770.sx_max_export_pos_size = 32;
 595		rdev->config.rv770.sx_max_export_smx_size = 224;
 596		rdev->config.rv770.sq_num_cf_insts = 2;
 597
 598		rdev->config.rv770.sx_num_of_sets = 7;
 599		rdev->config.rv770.sc_prim_fifo_size = 0xf9;
 600		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
 601		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
 602		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
 603			rdev->config.rv770.sx_max_export_pos_size -= 16;
 604			rdev->config.rv770.sx_max_export_smx_size += 16;
 605		}
 606		break;
 607	case CHIP_RV710:
 608		rdev->config.rv770.max_pipes = 2;
 609		rdev->config.rv770.max_tile_pipes = 2;
 610		rdev->config.rv770.max_simds = 2;
 611		rdev->config.rv770.max_backends = 1;
 612		rdev->config.rv770.max_gprs = 256;
 613		rdev->config.rv770.max_threads = 192;
 614		rdev->config.rv770.max_stack_entries = 256;
 615		rdev->config.rv770.max_hw_contexts = 4;
 616		rdev->config.rv770.max_gs_threads = 8 * 2;
 617		rdev->config.rv770.sx_max_export_size = 128;
 618		rdev->config.rv770.sx_max_export_pos_size = 16;
 619		rdev->config.rv770.sx_max_export_smx_size = 112;
 620		rdev->config.rv770.sq_num_cf_insts = 1;
 621
 622		rdev->config.rv770.sx_num_of_sets = 7;
 623		rdev->config.rv770.sc_prim_fifo_size = 0x40;
 624		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
 625		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
 626		break;
 627	case CHIP_RV740:
 628		rdev->config.rv770.max_pipes = 4;
 629		rdev->config.rv770.max_tile_pipes = 4;
 630		rdev->config.rv770.max_simds = 8;
 631		rdev->config.rv770.max_backends = 4;
 632		rdev->config.rv770.max_gprs = 256;
 633		rdev->config.rv770.max_threads = 248;
 634		rdev->config.rv770.max_stack_entries = 512;
 635		rdev->config.rv770.max_hw_contexts = 8;
 636		rdev->config.rv770.max_gs_threads = 16 * 2;
 637		rdev->config.rv770.sx_max_export_size = 256;
 638		rdev->config.rv770.sx_max_export_pos_size = 32;
 639		rdev->config.rv770.sx_max_export_smx_size = 224;
 640		rdev->config.rv770.sq_num_cf_insts = 2;
 641
 642		rdev->config.rv770.sx_num_of_sets = 7;
 643		rdev->config.rv770.sc_prim_fifo_size = 0x100;
 644		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
 645		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
 646
 647		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
 648			rdev->config.rv770.sx_max_export_pos_size -= 16;
 649			rdev->config.rv770.sx_max_export_smx_size += 16;
 650		}
 651		break;
 652	default:
 653		break;
 654	}
 655
 656	/* Initialize HDP */
 657	j = 0;
 658	for (i = 0; i < 32; i++) {
 659		WREG32((0x2c14 + j), 0x00000000);
 660		WREG32((0x2c18 + j), 0x00000000);
 661		WREG32((0x2c1c + j), 0x00000000);
 662		WREG32((0x2c20 + j), 0x00000000);
 663		WREG32((0x2c24 + j), 0x00000000);
 664		j += 0x18;
 665	}
 666
 667	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 668
 669	/* setup tiling, simd, pipe config */
 670	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
 671
 672	switch (rdev->config.rv770.max_tile_pipes) {
 673	case 1:
 674	default:
 675		gb_tiling_config |= PIPE_TILING(0);
 676		break;
 677	case 2:
 678		gb_tiling_config |= PIPE_TILING(1);
 679		break;
 680	case 4:
 681		gb_tiling_config |= PIPE_TILING(2);
 682		break;
 683	case 8:
 684		gb_tiling_config |= PIPE_TILING(3);
 685		break;
 686	}
 687	rdev->config.rv770.tiling_npipes = rdev->config.rv770.max_tile_pipes;
 688
 689	if (rdev->family == CHIP_RV770)
 690		gb_tiling_config |= BANK_TILING(1);
 691	else
 692		gb_tiling_config |= BANK_TILING((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
 693	rdev->config.rv770.tiling_nbanks = 4 << ((gb_tiling_config >> 4) & 0x3);
 694	gb_tiling_config |= GROUP_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
 695	if ((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT)
 696		rdev->config.rv770.tiling_group_size = 512;
 697	else
 698		rdev->config.rv770.tiling_group_size = 256;
 699	if (((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT) > 3) {
 700		gb_tiling_config |= ROW_TILING(3);
 701		gb_tiling_config |= SAMPLE_SPLIT(3);
 702	} else {
 703		gb_tiling_config |=
 704			ROW_TILING(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
 705		gb_tiling_config |=
 706			SAMPLE_SPLIT(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
 707	}
 708
 709	gb_tiling_config |= BANK_SWAPS(1);
 710
 711	cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
 712	cc_rb_backend_disable |=
 713		BACKEND_DISABLE((R7XX_MAX_BACKENDS_MASK << rdev->config.rv770.max_backends) & R7XX_MAX_BACKENDS_MASK);
 714
 715	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
 716	cc_gc_shader_pipe_config |=
 717		INACTIVE_QD_PIPES((R7XX_MAX_PIPES_MASK << rdev->config.rv770.max_pipes) & R7XX_MAX_PIPES_MASK);
 718	cc_gc_shader_pipe_config |=
 719		INACTIVE_SIMDS((R7XX_MAX_SIMDS_MASK << rdev->config.rv770.max_simds) & R7XX_MAX_SIMDS_MASK);
 720
 721	if (rdev->family == CHIP_RV740)
 722		backend_map = 0x28;
 723	else
 724		backend_map = r700_get_tile_pipe_to_backend_map(rdev,
 725								rdev->config.rv770.max_tile_pipes,
 726								(R7XX_MAX_BACKENDS -
 727								 r600_count_pipe_bits((cc_rb_backend_disable &
 728										       R7XX_MAX_BACKENDS_MASK) >> 16)),
 729								(cc_rb_backend_disable >> 16));
 730
 731	rdev->config.rv770.tile_config = gb_tiling_config;
 732	rdev->config.rv770.backend_map = backend_map;
 733	gb_tiling_config |= BACKEND_MAP(backend_map);
 734
 735	WREG32(GB_TILING_CONFIG, gb_tiling_config);
 736	WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff));
 737	WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff));
 738
 739	WREG32(CC_RB_BACKEND_DISABLE,      cc_rb_backend_disable);
 740	WREG32(CC_GC_SHADER_PIPE_CONFIG,   cc_gc_shader_pipe_config);
 741	WREG32(GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
 742	WREG32(CC_SYS_RB_BACKEND_DISABLE,  cc_rb_backend_disable);
 743
 744	WREG32(CGTS_SYS_TCC_DISABLE, 0);
 745	WREG32(CGTS_TCC_DISABLE, 0);
 746	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
 747	WREG32(CGTS_USER_TCC_DISABLE, 0);
 748
 749	num_qd_pipes =
 750		R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
 751	WREG32(VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & DEALLOC_DIST_MASK);
 752	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & VTX_REUSE_DEPTH_MASK);
 753
 754	/* set HW defaults for 3D engine */
 755	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
 756				     ROQ_IB2_START(0x2b)));
 757
 758	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
 759
 760	ta_aux_cntl = RREG32(TA_CNTL_AUX);
 761	WREG32(TA_CNTL_AUX, ta_aux_cntl | DISABLE_CUBE_ANISO);
 762
 763	sx_debug_1 = RREG32(SX_DEBUG_1);
 764	sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
 765	WREG32(SX_DEBUG_1, sx_debug_1);
 766
 767	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
 768	smx_dc_ctl0 &= ~CACHE_DEPTH(0x1ff);
 769	smx_dc_ctl0 |= CACHE_DEPTH((rdev->config.rv770.sx_num_of_sets * 64) - 1);
 770	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
 771
 772	if (rdev->family != CHIP_RV740)
 773		WREG32(SMX_EVENT_CTL, (ES_FLUSH_CTL(4) |
 774				       GS_FLUSH_CTL(4) |
 775				       ACK_FLUSH_CTL(3) |
 776				       SYNC_FLUSH_CTL));
 777
 778	db_debug3 = RREG32(DB_DEBUG3);
 779	db_debug3 &= ~DB_CLK_OFF_DELAY(0x1f);
 780	switch (rdev->family) {
 781	case CHIP_RV770:
 782	case CHIP_RV740:
 783		db_debug3 |= DB_CLK_OFF_DELAY(0x1f);
 784		break;
 785	case CHIP_RV710:
 786	case CHIP_RV730:
 787	default:
 788		db_debug3 |= DB_CLK_OFF_DELAY(2);
 789		break;
 790	}
 791	WREG32(DB_DEBUG3, db_debug3);
 792
 793	if (rdev->family != CHIP_RV770) {
 794		db_debug4 = RREG32(DB_DEBUG4);
 795		db_debug4 |= DISABLE_TILE_COVERED_FOR_PS_ITER;
 796		WREG32(DB_DEBUG4, db_debug4);
 797	}
 798
 799	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.rv770.sx_max_export_size / 4) - 1) |
 800					POSITION_BUFFER_SIZE((rdev->config.rv770.sx_max_export_pos_size / 4) - 1) |
 801					SMX_BUFFER_SIZE((rdev->config.rv770.sx_max_export_smx_size / 4) - 1)));
 802
 803	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.rv770.sc_prim_fifo_size) |
 804				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_hiz_tile_fifo_size) |
 805				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_earlyz_tile_fifo_fize)));
 806
 807	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
 808
 809	WREG32(VGT_NUM_INSTANCES, 1);
 810
 811	WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
 812
 813	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
 814
 815	WREG32(CP_PERFMON_CNTL, 0);
 816
 817	sq_ms_fifo_sizes = (CACHE_FIFO_SIZE(16 * rdev->config.rv770.sq_num_cf_insts) |
 818			    DONE_FIFO_HIWATER(0xe0) |
 819			    ALU_UPDATE_FIFO_HIWATER(0x8));
 820	switch (rdev->family) {
 821	case CHIP_RV770:
 822	case CHIP_RV730:
 823	case CHIP_RV710:
 824		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x1);
 825		break;
 826	case CHIP_RV740:
 827	default:
 828		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x4);
 829		break;
 830	}
 831	WREG32(SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
 832
 833	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
 834	 * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
 835	 */
 836	sq_config = RREG32(SQ_CONFIG);
 837	sq_config &= ~(PS_PRIO(3) |
 838		       VS_PRIO(3) |
 839		       GS_PRIO(3) |
 840		       ES_PRIO(3));
 841	sq_config |= (DX9_CONSTS |
 842		      VC_ENABLE |
 843		      EXPORT_SRC_C |
 844		      PS_PRIO(0) |
 845		      VS_PRIO(1) |
 846		      GS_PRIO(2) |
 847		      ES_PRIO(3));
 848	if (rdev->family == CHIP_RV710)
 849		/* no vertex cache */
 850		sq_config &= ~VC_ENABLE;
 851
 852	WREG32(SQ_CONFIG, sq_config);
 853
 854	WREG32(SQ_GPR_RESOURCE_MGMT_1,  (NUM_PS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
 855					 NUM_VS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
 856					 NUM_CLAUSE_TEMP_GPRS(((rdev->config.rv770.max_gprs * 24)/64)/2)));
 857
 858	WREG32(SQ_GPR_RESOURCE_MGMT_2,  (NUM_GS_GPRS((rdev->config.rv770.max_gprs * 7)/64) |
 859					 NUM_ES_GPRS((rdev->config.rv770.max_gprs * 7)/64)));
 860
 861	sq_thread_resource_mgmt = (NUM_PS_THREADS((rdev->config.rv770.max_threads * 4)/8) |
 862				   NUM_VS_THREADS((rdev->config.rv770.max_threads * 2)/8) |
 863				   NUM_ES_THREADS((rdev->config.rv770.max_threads * 1)/8));
 864	if (((rdev->config.rv770.max_threads * 1) / 8) > rdev->config.rv770.max_gs_threads)
 865		sq_thread_resource_mgmt |= NUM_GS_THREADS(rdev->config.rv770.max_gs_threads);
 866	else
 867		sq_thread_resource_mgmt |= NUM_GS_THREADS((rdev->config.rv770.max_gs_threads * 1)/8);
 868	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
 869
 870	WREG32(SQ_STACK_RESOURCE_MGMT_1, (NUM_PS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
 871						     NUM_VS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
 872
 873	WREG32(SQ_STACK_RESOURCE_MGMT_2, (NUM_GS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
 874						     NUM_ES_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
 875
 876	sq_dyn_gpr_size_simd_ab_0 = (SIMDA_RING0((rdev->config.rv770.max_gprs * 38)/64) |
 877				     SIMDA_RING1((rdev->config.rv770.max_gprs * 38)/64) |
 878				     SIMDB_RING0((rdev->config.rv770.max_gprs * 38)/64) |
 879				     SIMDB_RING1((rdev->config.rv770.max_gprs * 38)/64));
 880
 881	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
 882	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
 883	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
 884	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
 885	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
 886	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
 887	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
 888	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
 889
 890	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
 891					  FORCE_EOV_MAX_REZ_CNT(255)));
 892
 893	if (rdev->family == CHIP_RV710)
 894		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(TC_ONLY) |
 895						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
 896	else
 897		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(VC_AND_TC) |
 898						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
 899
 900	switch (rdev->family) {
 901	case CHIP_RV770:
 902	case CHIP_RV730:
 903	case CHIP_RV740:
 904		gs_prim_buffer_depth = 384;
 905		break;
 906	case CHIP_RV710:
 907		gs_prim_buffer_depth = 128;
 908		break;
 909	default:
 910		break;
 911	}
 912
 913	num_gs_verts_per_thread = rdev->config.rv770.max_pipes * 16;
 914	vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
 915	/* Max value for this is 256 */
 916	if (vgt_gs_per_es > 256)
 917		vgt_gs_per_es = 256;
 918
 919	WREG32(VGT_ES_PER_GS, 128);
 920	WREG32(VGT_GS_PER_ES, vgt_gs_per_es);
 921	WREG32(VGT_GS_PER_VS, 2);
 922
 923	/* more default values. 2D/3D driver should adjust as needed */
 924	WREG32(VGT_GS_VERTEX_REUSE, 16);
 925	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
 926	WREG32(VGT_STRMOUT_EN, 0);
 927	WREG32(SX_MISC, 0);
 928	WREG32(PA_SC_MODE_CNTL, 0);
 929	WREG32(PA_SC_EDGERULE, 0xaaaaaaaa);
 930	WREG32(PA_SC_AA_CONFIG, 0);
 931	WREG32(PA_SC_CLIPRECT_RULE, 0xffff);
 932	WREG32(PA_SC_LINE_STIPPLE, 0);
 933	WREG32(SPI_INPUT_Z, 0);
 934	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
 935	WREG32(CB_COLOR7_FRAG, 0);
 936
 937	/* clear render buffer base addresses */
 938	WREG32(CB_COLOR0_BASE, 0);
 939	WREG32(CB_COLOR1_BASE, 0);
 940	WREG32(CB_COLOR2_BASE, 0);
 941	WREG32(CB_COLOR3_BASE, 0);
 942	WREG32(CB_COLOR4_BASE, 0);
 943	WREG32(CB_COLOR5_BASE, 0);
 944	WREG32(CB_COLOR6_BASE, 0);
 945	WREG32(CB_COLOR7_BASE, 0);
 946
 947	WREG32(TCP_CNTL, 0);
 948
 949	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
 950	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
 951
 952	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
 953
 954	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
 955					  NUM_CLIP_SEQ(3)));
 956
 957}
 958
 959static int rv770_vram_scratch_init(struct radeon_device *rdev)
 960{
 961	int r;
 962	u64 gpu_addr;
 963
 964	if (rdev->vram_scratch.robj == NULL) {
 965		r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
 966				     PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
 967				     &rdev->vram_scratch.robj);
 968		if (r) {
 969			return r;
 970		}
 971	}
 972
 973	r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
 974	if (unlikely(r != 0))
 975		return r;
 976	r = radeon_bo_pin(rdev->vram_scratch.robj,
 977			  RADEON_GEM_DOMAIN_VRAM, &gpu_addr);
 978	if (r) {
 979		radeon_bo_unreserve(rdev->vram_scratch.robj);
 980		return r;
 981	}
 982	r = radeon_bo_kmap(rdev->vram_scratch.robj,
 983				(void **)&rdev->vram_scratch.ptr);
 984	if (r)
 985		radeon_bo_unpin(rdev->vram_scratch.robj);
 986	radeon_bo_unreserve(rdev->vram_scratch.robj);
 987
 988	return r;
 989}
 990
 991static void rv770_vram_scratch_fini(struct radeon_device *rdev)
 992{
 993	int r;
 994
 995	if (rdev->vram_scratch.robj == NULL) {
 996		return;
 997	}
 998	r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
 999	if (likely(r == 0)) {
1000		radeon_bo_kunmap(rdev->vram_scratch.robj);
1001		radeon_bo_unpin(rdev->vram_scratch.robj);
1002		radeon_bo_unreserve(rdev->vram_scratch.robj);
1003	}
1004	radeon_bo_unref(&rdev->vram_scratch.robj);
1005}
1006
1007void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1008{
1009	u64 size_bf, size_af;
1010
1011	if (mc->mc_vram_size > 0xE0000000) {
1012		/* leave room for at least 512M GTT */
1013		dev_warn(rdev->dev, "limiting VRAM\n");
1014		mc->real_vram_size = 0xE0000000;
1015		mc->mc_vram_size = 0xE0000000;
1016	}
1017	if (rdev->flags & RADEON_IS_AGP) {
1018		size_bf = mc->gtt_start;
1019		size_af = 0xFFFFFFFF - mc->gtt_end + 1;
1020		if (size_bf > size_af) {
1021			if (mc->mc_vram_size > size_bf) {
1022				dev_warn(rdev->dev, "limiting VRAM\n");
1023				mc->real_vram_size = size_bf;
1024				mc->mc_vram_size = size_bf;
1025			}
1026			mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1027		} else {
1028			if (mc->mc_vram_size > size_af) {
1029				dev_warn(rdev->dev, "limiting VRAM\n");
1030				mc->real_vram_size = size_af;
1031				mc->mc_vram_size = size_af;
1032			}
1033			mc->vram_start = mc->gtt_end;
1034		}
1035		mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1036		dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1037				mc->mc_vram_size >> 20, mc->vram_start,
1038				mc->vram_end, mc->real_vram_size >> 20);
1039	} else {
1040		radeon_vram_location(rdev, &rdev->mc, 0);
1041		rdev->mc.gtt_base_align = 0;
1042		radeon_gtt_location(rdev, mc);
1043	}
1044}
1045
1046int rv770_mc_init(struct radeon_device *rdev)
1047{
1048	u32 tmp;
1049	int chansize, numchan;
1050
1051	/* Get VRAM informations */
1052	rdev->mc.vram_is_ddr = true;
1053	tmp = RREG32(MC_ARB_RAMCFG);
1054	if (tmp & CHANSIZE_OVERRIDE) {
1055		chansize = 16;
1056	} else if (tmp & CHANSIZE_MASK) {
1057		chansize = 64;
1058	} else {
1059		chansize = 32;
1060	}
1061	tmp = RREG32(MC_SHARED_CHMAP);
1062	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1063	case 0:
1064	default:
1065		numchan = 1;
1066		break;
1067	case 1:
1068		numchan = 2;
1069		break;
1070	case 2:
1071		numchan = 4;
1072		break;
1073	case 3:
1074		numchan = 8;
1075		break;
1076	}
1077	rdev->mc.vram_width = numchan * chansize;
1078	/* Could aper size report 0 ? */
1079	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1080	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1081	/* Setup GPU memory space */
1082	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1083	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1084	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1085	r700_vram_gtt_location(rdev, &rdev->mc);
1086	radeon_update_bandwidth_info(rdev);
1087
1088	return 0;
1089}
1090
1091static int rv770_startup(struct radeon_device *rdev)
1092{
1093	int r;
1094
1095	/* enable pcie gen2 link */
1096	rv770_pcie_gen2_enable(rdev);
1097
1098	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1099		r = r600_init_microcode(rdev);
1100		if (r) {
1101			DRM_ERROR("Failed to load firmware!\n");
1102			return r;
1103		}
1104	}
1105
1106	rv770_mc_program(rdev);
1107	if (rdev->flags & RADEON_IS_AGP) {
1108		rv770_agp_enable(rdev);
1109	} else {
1110		r = rv770_pcie_gart_enable(rdev);
1111		if (r)
1112			return r;
1113	}
1114	r = rv770_vram_scratch_init(rdev);
1115	if (r)
1116		return r;
1117	rv770_gpu_init(rdev);
1118	r = r600_blit_init(rdev);
1119	if (r) {
1120		r600_blit_fini(rdev);
1121		rdev->asic->copy = NULL;
1122		dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
1123	}
1124
1125	/* allocate wb buffer */
1126	r = radeon_wb_init(rdev);
1127	if (r)
1128		return r;
1129
1130	/* Enable IRQ */
1131	r = r600_irq_init(rdev);
1132	if (r) {
1133		DRM_ERROR("radeon: IH init failed (%d).\n", r);
1134		radeon_irq_kms_fini(rdev);
1135		return r;
1136	}
1137	r600_irq_set(rdev);
1138
1139	r = radeon_ring_init(rdev, rdev->cp.ring_size);
1140	if (r)
1141		return r;
1142	r = rv770_cp_load_microcode(rdev);
1143	if (r)
1144		return r;
1145	r = r600_cp_resume(rdev);
1146	if (r)
1147		return r;
1148
1149	return 0;
1150}
1151
1152int rv770_resume(struct radeon_device *rdev)
1153{
1154	int r;
1155
1156	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
1157	 * posting will perform necessary task to bring back GPU into good
1158	 * shape.
1159	 */
1160	/* post card */
1161	atom_asic_init(rdev->mode_info.atom_context);
1162
1163	r = rv770_startup(rdev);
1164	if (r) {
1165		DRM_ERROR("r600 startup failed on resume\n");
1166		return r;
1167	}
1168
1169	r = r600_ib_test(rdev);
1170	if (r) {
1171		DRM_ERROR("radeon: failed testing IB (%d).\n", r);
1172		return r;
1173	}
1174
1175	r = r600_audio_init(rdev);
1176	if (r) {
1177		dev_err(rdev->dev, "radeon: audio init failed\n");
1178		return r;
1179	}
1180
1181	return r;
1182
1183}
1184
1185int rv770_suspend(struct radeon_device *rdev)
1186{
1187	int r;
1188
1189	r600_audio_fini(rdev);
1190	/* FIXME: we should wait for ring to be empty */
1191	r700_cp_stop(rdev);
1192	rdev->cp.ready = false;
1193	r600_irq_suspend(rdev);
1194	radeon_wb_disable(rdev);
1195	rv770_pcie_gart_disable(rdev);
1196	/* unpin shaders bo */
1197	if (rdev->r600_blit.shader_obj) {
1198		r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
1199		if (likely(r == 0)) {
1200			radeon_bo_unpin(rdev->r600_blit.shader_obj);
1201			radeon_bo_unreserve(rdev->r600_blit.shader_obj);
1202		}
1203	}
1204	return 0;
1205}
1206
1207/* Plan is to move initialization in that function and use
1208 * helper function so that radeon_device_init pretty much
1209 * do nothing more than calling asic specific function. This
1210 * should also allow to remove a bunch of callback function
1211 * like vram_info.
1212 */
1213int rv770_init(struct radeon_device *rdev)
1214{
1215	int r;
1216
1217	/* This don't do much */
1218	r = radeon_gem_init(rdev);
1219	if (r)
1220		return r;
1221	/* Read BIOS */
1222	if (!radeon_get_bios(rdev)) {
1223		if (ASIC_IS_AVIVO(rdev))
1224			return -EINVAL;
1225	}
1226	/* Must be an ATOMBIOS */
1227	if (!rdev->is_atom_bios) {
1228		dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
1229		return -EINVAL;
1230	}
1231	r = radeon_atombios_init(rdev);
1232	if (r)
1233		return r;
1234	/* Post card if necessary */
1235	if (!radeon_card_posted(rdev)) {
1236		if (!rdev->bios) {
1237			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
1238			return -EINVAL;
1239		}
1240		DRM_INFO("GPU not posted. posting now...\n");
1241		atom_asic_init(rdev->mode_info.atom_context);
1242	}
1243	/* Initialize scratch registers */
1244	r600_scratch_init(rdev);
1245	/* Initialize surface registers */
1246	radeon_surface_init(rdev);
1247	/* Initialize clocks */
1248	radeon_get_clock_info(rdev->ddev);
1249	/* Fence driver */
1250	r = radeon_fence_driver_init(rdev);
1251	if (r)
1252		return r;
1253	/* initialize AGP */
1254	if (rdev->flags & RADEON_IS_AGP) {
1255		r = radeon_agp_init(rdev);
1256		if (r)
1257			radeon_agp_disable(rdev);
1258	}
1259	r = rv770_mc_init(rdev);
1260	if (r)
1261		return r;
1262	/* Memory manager */
1263	r = radeon_bo_init(rdev);
1264	if (r)
1265		return r;
1266
1267	r = radeon_irq_kms_init(rdev);
1268	if (r)
1269		return r;
1270
1271	rdev->cp.ring_obj = NULL;
1272	r600_ring_init(rdev, 1024 * 1024);
1273
1274	rdev->ih.ring_obj = NULL;
1275	r600_ih_ring_init(rdev, 64 * 1024);
1276
1277	r = r600_pcie_gart_init(rdev);
1278	if (r)
1279		return r;
1280
1281	rdev->accel_working = true;
1282	r = rv770_startup(rdev);
1283	if (r) {
1284		dev_err(rdev->dev, "disabling GPU acceleration\n");
1285		r700_cp_fini(rdev);
1286		r600_irq_fini(rdev);
1287		radeon_wb_fini(rdev);
1288		radeon_irq_kms_fini(rdev);
1289		rv770_pcie_gart_fini(rdev);
1290		rdev->accel_working = false;
1291	}
1292	if (rdev->accel_working) {
1293		r = radeon_ib_pool_init(rdev);
1294		if (r) {
1295			dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1296			rdev->accel_working = false;
1297		} else {
1298			r = r600_ib_test(rdev);
1299			if (r) {
1300				dev_err(rdev->dev, "IB test failed (%d).\n", r);
1301				rdev->accel_working = false;
1302			}
1303		}
1304	}
1305
1306	r = r600_audio_init(rdev);
1307	if (r) {
1308		dev_err(rdev->dev, "radeon: audio init failed\n");
1309		return r;
1310	}
1311
1312	return 0;
1313}
1314
1315void rv770_fini(struct radeon_device *rdev)
1316{
1317	r600_blit_fini(rdev);
1318	r700_cp_fini(rdev);
1319	r600_irq_fini(rdev);
1320	radeon_wb_fini(rdev);
1321	radeon_ib_pool_fini(rdev);
1322	radeon_irq_kms_fini(rdev);
1323	rv770_pcie_gart_fini(rdev);
1324	rv770_vram_scratch_fini(rdev);
1325	radeon_gem_fini(rdev);
1326	radeon_fence_driver_fini(rdev);
1327	radeon_agp_fini(rdev);
1328	radeon_bo_fini(rdev);
1329	radeon_atombios_fini(rdev);
1330	kfree(rdev->bios);
1331	rdev->bios = NULL;
1332}
1333
1334static void rv770_pcie_gen2_enable(struct radeon_device *rdev)
1335{
1336	u32 link_width_cntl, lanes, speed_cntl, tmp;
1337	u16 link_cntl2;
1338
1339	if (radeon_pcie_gen2 == 0)
1340		return;
1341
1342	if (rdev->flags & RADEON_IS_IGP)
1343		return;
1344
1345	if (!(rdev->flags & RADEON_IS_PCIE))
1346		return;
1347
1348	/* x2 cards have a special sequence */
1349	if (ASIC_IS_X2(rdev))
1350		return;
1351
1352	/* advertise upconfig capability */
1353	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
1354	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
1355	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1356	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
1357	if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
1358		lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
1359		link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
1360				     LC_RECONFIG_ARC_MISSING_ESCAPE);
1361		link_width_cntl |= lanes | LC_RECONFIG_NOW |
1362			LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT;
1363		WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1364	} else {
1365		link_width_cntl |= LC_UPCONFIGURE_DIS;
1366		WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1367	}
1368
1369	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
1370	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
1371	    (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
1372
1373		tmp = RREG32(0x541c);
1374		WREG32(0x541c, tmp | 0x8);
1375		WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
1376		link_cntl2 = RREG16(0x4088);
1377		link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
1378		link_cntl2 |= 0x2;
1379		WREG16(0x4088, link_cntl2);
1380		WREG32(MM_CFGREGS_CNTL, 0);
1381
1382		speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
1383		speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
1384		WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
1385
1386		speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
1387		speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
1388		WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
1389
1390		speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
1391		speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
1392		WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
1393
1394		speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
1395		speed_cntl |= LC_GEN2_EN_STRAP;
1396		WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
1397
1398	} else {
1399		link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
1400		/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
1401		if (1)
1402			link_width_cntl |= LC_UPCONFIGURE_DIS;
1403		else
1404			link_width_cntl &= ~LC_UPCONFIGURE_DIS;
1405		WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1406	}
1407}