1/*
   2 * Copyright 2008 Advanced Micro Devices, Inc.
   3 * Copyright 2008 Red Hat Inc.
   4 * Copyright 2009 Jerome Glisse.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 *
  24 * Authors: Dave Airlie
  25 *          Alex Deucher
  26 *          Jerome Glisse
  27 */
  28
  29#include <linux/console.h>
  30#include <linux/efi.h>
  31#include <linux/pci.h>
  32#include <linux/pm_runtime.h>
  33#include <linux/slab.h>
  34#include <linux/vga_switcheroo.h>
  35#include <linux/vgaarb.h>
  36
  37#include <drm/drm_cache.h>
  38#include <drm/drm_client_event.h>
  39#include <drm/drm_crtc_helper.h>
  40#include <drm/drm_device.h>
  41#include <drm/drm_file.h>
  42#include <drm/drm_framebuffer.h>
  43#include <drm/drm_probe_helper.h>
  44#include <drm/radeon_drm.h>
  45
  46#include "radeon_device.h"
  47#include "radeon_reg.h"
  48#include "radeon.h"
  49#include "atom.h"
  50
  51static const char radeon_family_name[][16] = {
  52	"R100",
  53	"RV100",
  54	"RS100",
  55	"RV200",
  56	"RS200",
  57	"R200",
  58	"RV250",
  59	"RS300",
  60	"RV280",
  61	"R300",
  62	"R350",
  63	"RV350",
  64	"RV380",
  65	"R420",
  66	"R423",
  67	"RV410",
  68	"RS400",
  69	"RS480",
  70	"RS600",
  71	"RS690",
  72	"RS740",
  73	"RV515",
  74	"R520",
  75	"RV530",
  76	"RV560",
  77	"RV570",
  78	"R580",
  79	"R600",
  80	"RV610",
  81	"RV630",
  82	"RV670",
  83	"RV620",
  84	"RV635",
  85	"RS780",
  86	"RS880",
  87	"RV770",
  88	"RV730",
  89	"RV710",
  90	"RV740",
  91	"CEDAR",
  92	"REDWOOD",
  93	"JUNIPER",
  94	"CYPRESS",
  95	"HEMLOCK",
  96	"PALM",
  97	"SUMO",
  98	"SUMO2",
  99	"BARTS",
 100	"TURKS",
 101	"CAICOS",
 102	"CAYMAN",
 103	"ARUBA",
 104	"TAHITI",
 105	"PITCAIRN",
 106	"VERDE",
 107	"OLAND",
 108	"HAINAN",
 109	"BONAIRE",
 110	"KAVERI",
 111	"KABINI",
 112	"HAWAII",
 113	"MULLINS",
 114	"LAST",
 115};
 116
 117#if defined(CONFIG_VGA_SWITCHEROO)
 118bool radeon_has_atpx_dgpu_power_cntl(void);
 119bool radeon_is_atpx_hybrid(void);
 120#else
 121static inline bool radeon_has_atpx_dgpu_power_cntl(void) { return false; }
 122static inline bool radeon_is_atpx_hybrid(void) { return false; }
 123#endif
 124
 125#define RADEON_PX_QUIRK_DISABLE_PX  (1 << 0)
 126
 127struct radeon_px_quirk {
 128	u32 chip_vendor;
 129	u32 chip_device;
 130	u32 subsys_vendor;
 131	u32 subsys_device;
 132	u32 px_quirk_flags;
 133};
 134
 135static struct radeon_px_quirk radeon_px_quirk_list[] = {
 136	/* Acer aspire 5560g (CPU: AMD A4-3305M; GPU: AMD Radeon HD 6480g + 7470m)
 137	 * https://bugzilla.kernel.org/show_bug.cgi?id=74551
 138	 */
 139	{ PCI_VENDOR_ID_ATI, 0x6760, 0x1025, 0x0672, RADEON_PX_QUIRK_DISABLE_PX },
 140	/* Asus K73TA laptop with AMD A6-3400M APU and Radeon 6550 GPU
 141	 * https://bugzilla.kernel.org/show_bug.cgi?id=51381
 142	 */
 143	{ PCI_VENDOR_ID_ATI, 0x6741, 0x1043, 0x108c, RADEON_PX_QUIRK_DISABLE_PX },
 144	/* Asus K53TK laptop with AMD A6-3420M APU and Radeon 7670m GPU
 145	 * https://bugzilla.kernel.org/show_bug.cgi?id=51381
 146	 */
 147	{ PCI_VENDOR_ID_ATI, 0x6840, 0x1043, 0x2122, RADEON_PX_QUIRK_DISABLE_PX },
 148	/* Asus K53TK laptop with AMD A6-3420M APU and Radeon 7670m GPU
 149	 * https://bugs.freedesktop.org/show_bug.cgi?id=101491
 150	 */
 151	{ PCI_VENDOR_ID_ATI, 0x6741, 0x1043, 0x2122, RADEON_PX_QUIRK_DISABLE_PX },
 152	/* Asus K73TK laptop with AMD A6-3420M APU and Radeon 7670m GPU
 153	 * https://bugzilla.kernel.org/show_bug.cgi?id=51381#c52
 154	 */
 155	{ PCI_VENDOR_ID_ATI, 0x6840, 0x1043, 0x2123, RADEON_PX_QUIRK_DISABLE_PX },
 156	{ 0, 0, 0, 0, 0 },
 157};
 158
 159bool radeon_is_px(struct drm_device *dev)
 160{
 161	struct radeon_device *rdev = dev->dev_private;
 162
 163	if (rdev->flags & RADEON_IS_PX)
 164		return true;
 165	return false;
 166}
 167
 168static void radeon_device_handle_px_quirks(struct radeon_device *rdev)
 169{
 170	struct radeon_px_quirk *p = radeon_px_quirk_list;
 171
 172	/* Apply PX quirks */
 173	while (p && p->chip_device != 0) {
 174		if (rdev->pdev->vendor == p->chip_vendor &&
 175		    rdev->pdev->device == p->chip_device &&
 176		    rdev->pdev->subsystem_vendor == p->subsys_vendor &&
 177		    rdev->pdev->subsystem_device == p->subsys_device) {
 178			rdev->px_quirk_flags = p->px_quirk_flags;
 179			break;
 180		}
 181		++p;
 182	}
 183
 184	if (rdev->px_quirk_flags & RADEON_PX_QUIRK_DISABLE_PX)
 185		rdev->flags &= ~RADEON_IS_PX;
 186
 187	/* disable PX is the system doesn't support dGPU power control or hybrid gfx */
 188	if (!radeon_is_atpx_hybrid() &&
 189	    !radeon_has_atpx_dgpu_power_cntl())
 190		rdev->flags &= ~RADEON_IS_PX;
 191}
 192
 193/**
 194 * radeon_program_register_sequence - program an array of registers.
 195 *
 196 * @rdev: radeon_device pointer
 197 * @registers: pointer to the register array
 198 * @array_size: size of the register array
 199 *
 200 * Programs an array or registers with and and or masks.
 201 * This is a helper for setting golden registers.
 202 */
 203void radeon_program_register_sequence(struct radeon_device *rdev,
 204				      const u32 *registers,
 205				      const u32 array_size)
 206{
 207	u32 tmp, reg, and_mask, or_mask;
 208	int i;
 209
 210	if (array_size % 3)
 211		return;
 212
 213	for (i = 0; i < array_size; i +=3) {
 214		reg = registers[i + 0];
 215		and_mask = registers[i + 1];
 216		or_mask = registers[i + 2];
 217
 218		if (and_mask == 0xffffffff) {
 219			tmp = or_mask;
 220		} else {
 221			tmp = RREG32(reg);
 222			tmp &= ~and_mask;
 223			tmp |= or_mask;
 224		}
 225		WREG32(reg, tmp);
 226	}
 227}
 228
 229void radeon_pci_config_reset(struct radeon_device *rdev)
 230{
 231	pci_write_config_dword(rdev->pdev, 0x7c, RADEON_ASIC_RESET_DATA);
 232}
 233
 234/**
 235 * radeon_surface_init - Clear GPU surface registers.
 236 *
 237 * @rdev: radeon_device pointer
 238 *
 239 * Clear GPU surface registers (r1xx-r5xx).
 240 */
 241void radeon_surface_init(struct radeon_device *rdev)
 242{
 243	/* FIXME: check this out */
 244	if (rdev->family < CHIP_R600) {
 245		int i;
 246
 247		for (i = 0; i < RADEON_GEM_MAX_SURFACES; i++) {
 248			if (rdev->surface_regs[i].bo)
 249				radeon_bo_get_surface_reg(rdev->surface_regs[i].bo);
 250			else
 251				radeon_clear_surface_reg(rdev, i);
 252		}
 253		/* enable surfaces */
 254		WREG32(RADEON_SURFACE_CNTL, 0);
 255	}
 256}
 257
 258/*
 259 * GPU scratch registers helpers function.
 260 */
 261/**
 262 * radeon_scratch_init - Init scratch register driver information.
 263 *
 264 * @rdev: radeon_device pointer
 265 *
 266 * Init CP scratch register driver information (r1xx-r5xx)
 267 */
 268void radeon_scratch_init(struct radeon_device *rdev)
 269{
 270	int i;
 271
 272	/* FIXME: check this out */
 273	if (rdev->family < CHIP_R300) {
 274		rdev->scratch.num_reg = 5;
 275	} else {
 276		rdev->scratch.num_reg = 7;
 277	}
 278	rdev->scratch.reg_base = RADEON_SCRATCH_REG0;
 279	for (i = 0; i < rdev->scratch.num_reg; i++) {
 280		rdev->scratch.free[i] = true;
 281		rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
 282	}
 283}
 284
 285/**
 286 * radeon_scratch_get - Allocate a scratch register
 287 *
 288 * @rdev: radeon_device pointer
 289 * @reg: scratch register mmio offset
 290 *
 291 * Allocate a CP scratch register for use by the driver (all asics).
 292 * Returns 0 on success or -EINVAL on failure.
 293 */
 294int radeon_scratch_get(struct radeon_device *rdev, uint32_t *reg)
 295{
 296	int i;
 297
 298	for (i = 0; i < rdev->scratch.num_reg; i++) {
 299		if (rdev->scratch.free[i]) {
 300			rdev->scratch.free[i] = false;
 301			*reg = rdev->scratch.reg[i];
 302			return 0;
 303		}
 304	}
 305	return -EINVAL;
 306}
 307
 308/**
 309 * radeon_scratch_free - Free a scratch register
 310 *
 311 * @rdev: radeon_device pointer
 312 * @reg: scratch register mmio offset
 313 *
 314 * Free a CP scratch register allocated for use by the driver (all asics)
 315 */
 316void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg)
 317{
 318	int i;
 319
 320	for (i = 0; i < rdev->scratch.num_reg; i++) {
 321		if (rdev->scratch.reg[i] == reg) {
 322			rdev->scratch.free[i] = true;
 323			return;
 324		}
 325	}
 326}
 327
 328/*
 329 * GPU doorbell aperture helpers function.
 330 */
 331/**
 332 * radeon_doorbell_init - Init doorbell driver information.
 333 *
 334 * @rdev: radeon_device pointer
 335 *
 336 * Init doorbell driver information (CIK)
 337 * Returns 0 on success, error on failure.
 338 */
 339static int radeon_doorbell_init(struct radeon_device *rdev)
 340{
 341	/* doorbell bar mapping */
 342	rdev->doorbell.base = pci_resource_start(rdev->pdev, 2);
 343	rdev->doorbell.size = pci_resource_len(rdev->pdev, 2);
 344
 345	rdev->doorbell.num_doorbells = min_t(u32, rdev->doorbell.size / sizeof(u32), RADEON_MAX_DOORBELLS);
 346	if (rdev->doorbell.num_doorbells == 0)
 347		return -EINVAL;
 348
 349	rdev->doorbell.ptr = ioremap(rdev->doorbell.base, rdev->doorbell.num_doorbells * sizeof(u32));
 350	if (rdev->doorbell.ptr == NULL) {
 351		return -ENOMEM;
 352	}
 353	DRM_INFO("doorbell mmio base: 0x%08X\n", (uint32_t)rdev->doorbell.base);
 354	DRM_INFO("doorbell mmio size: %u\n", (unsigned)rdev->doorbell.size);
 355
 356	memset(&rdev->doorbell.used, 0, sizeof(rdev->doorbell.used));
 357
 358	return 0;
 359}
 360
 361/**
 362 * radeon_doorbell_fini - Tear down doorbell driver information.
 363 *
 364 * @rdev: radeon_device pointer
 365 *
 366 * Tear down doorbell driver information (CIK)
 367 */
 368static void radeon_doorbell_fini(struct radeon_device *rdev)
 369{
 370	iounmap(rdev->doorbell.ptr);
 371	rdev->doorbell.ptr = NULL;
 372}
 373
 374/**
 375 * radeon_doorbell_get - Allocate a doorbell entry
 376 *
 377 * @rdev: radeon_device pointer
 378 * @doorbell: doorbell index
 379 *
 380 * Allocate a doorbell for use by the driver (all asics).
 381 * Returns 0 on success or -EINVAL on failure.
 382 */
 383int radeon_doorbell_get(struct radeon_device *rdev, u32 *doorbell)
 384{
 385	unsigned long offset = find_first_zero_bit(rdev->doorbell.used, rdev->doorbell.num_doorbells);
 386	if (offset < rdev->doorbell.num_doorbells) {
 387		__set_bit(offset, rdev->doorbell.used);
 388		*doorbell = offset;
 389		return 0;
 390	} else {
 391		return -EINVAL;
 392	}
 393}
 394
 395/**
 396 * radeon_doorbell_free - Free a doorbell entry
 397 *
 398 * @rdev: radeon_device pointer
 399 * @doorbell: doorbell index
 400 *
 401 * Free a doorbell allocated for use by the driver (all asics)
 402 */
 403void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell)
 404{
 405	if (doorbell < rdev->doorbell.num_doorbells)
 406		__clear_bit(doorbell, rdev->doorbell.used);
 407}
 408
 409/*
 410 * radeon_wb_*()
 411 * Writeback is the method by which the GPU updates special pages
 412 * in memory with the status of certain GPU events (fences, ring pointers,
 413 * etc.).
 414 */
 415
 416/**
 417 * radeon_wb_disable - Disable Writeback
 418 *
 419 * @rdev: radeon_device pointer
 420 *
 421 * Disables Writeback (all asics).  Used for suspend.
 422 */
 423void radeon_wb_disable(struct radeon_device *rdev)
 424{
 425	rdev->wb.enabled = false;
 426}
 427
 428/**
 429 * radeon_wb_fini - Disable Writeback and free memory
 430 *
 431 * @rdev: radeon_device pointer
 432 *
 433 * Disables Writeback and frees the Writeback memory (all asics).
 434 * Used at driver shutdown.
 435 */
 436void radeon_wb_fini(struct radeon_device *rdev)
 437{
 438	radeon_wb_disable(rdev);
 439	if (rdev->wb.wb_obj) {
 440		if (!radeon_bo_reserve(rdev->wb.wb_obj, false)) {
 441			radeon_bo_kunmap(rdev->wb.wb_obj);
 442			radeon_bo_unpin(rdev->wb.wb_obj);
 443			radeon_bo_unreserve(rdev->wb.wb_obj);
 444		}
 445		radeon_bo_unref(&rdev->wb.wb_obj);
 446		rdev->wb.wb = NULL;
 447		rdev->wb.wb_obj = NULL;
 448	}
 449}
 450
 451/**
 452 * radeon_wb_init- Init Writeback driver info and allocate memory
 453 *
 454 * @rdev: radeon_device pointer
 455 *
 456 * Disables Writeback and frees the Writeback memory (all asics).
 457 * Used at driver startup.
 458 * Returns 0 on success or an -error on failure.
 459 */
 460int radeon_wb_init(struct radeon_device *rdev)
 461{
 462	int r;
 463
 464	if (rdev->wb.wb_obj == NULL) {
 465		r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
 466				     RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
 467				     &rdev->wb.wb_obj);
 468		if (r) {
 469			dev_warn(rdev->dev, "(%d) create WB bo failed\n", r);
 470			return r;
 471		}
 472		r = radeon_bo_reserve(rdev->wb.wb_obj, false);
 473		if (unlikely(r != 0)) {
 474			radeon_wb_fini(rdev);
 475			return r;
 476		}
 477		r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,
 478				&rdev->wb.gpu_addr);
 479		if (r) {
 480			radeon_bo_unreserve(rdev->wb.wb_obj);
 481			dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r);
 482			radeon_wb_fini(rdev);
 483			return r;
 484		}
 485		r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);
 486		radeon_bo_unreserve(rdev->wb.wb_obj);
 487		if (r) {
 488			dev_warn(rdev->dev, "(%d) map WB bo failed\n", r);
 489			radeon_wb_fini(rdev);
 490			return r;
 491		}
 492	}
 493
 494	/* clear wb memory */
 495	memset((char *)rdev->wb.wb, 0, RADEON_GPU_PAGE_SIZE);
 496	/* disable event_write fences */
 497	rdev->wb.use_event = false;
 498	/* disabled via module param */
 499	if (radeon_no_wb == 1) {
 500		rdev->wb.enabled = false;
 501	} else {
 502		if (rdev->flags & RADEON_IS_AGP) {
 503			/* often unreliable on AGP */
 504			rdev->wb.enabled = false;
 505		} else if (rdev->family < CHIP_R300) {
 506			/* often unreliable on pre-r300 */
 507			rdev->wb.enabled = false;
 508		} else {
 509			rdev->wb.enabled = true;
 510			/* event_write fences are only available on r600+ */
 511			if (rdev->family >= CHIP_R600) {
 512				rdev->wb.use_event = true;
 513			}
 514		}
 515	}
 516	/* always use writeback/events on NI, APUs */
 517	if (rdev->family >= CHIP_PALM) {
 518		rdev->wb.enabled = true;
 519		rdev->wb.use_event = true;
 520	}
 521
 522	dev_info(rdev->dev, "WB %sabled\n", rdev->wb.enabled ? "en" : "dis");
 523
 524	return 0;
 525}
 526
 527/**
 528 * radeon_vram_location - try to find VRAM location
 529 * @rdev: radeon device structure holding all necessary informations
 530 * @mc: memory controller structure holding memory informations
 531 * @base: base address at which to put VRAM
 532 *
 533 * Function will place try to place VRAM at base address provided
 534 * as parameter (which is so far either PCI aperture address or
 535 * for IGP TOM base address).
 536 *
 537 * If there is not enough space to fit the unvisible VRAM in the 32bits
 538 * address space then we limit the VRAM size to the aperture.
 539 *
 540 * If we are using AGP and if the AGP aperture doesn't allow us to have
 541 * room for all the VRAM than we restrict the VRAM to the PCI aperture
 542 * size and print a warning.
 543 *
 544 * This function will never fails, worst case are limiting VRAM.
 545 *
 546 * Note: GTT start, end, size should be initialized before calling this
 547 * function on AGP platform.
 548 *
 549 * Note 1: We don't explicitly enforce VRAM start to be aligned on VRAM size,
 550 * this shouldn't be a problem as we are using the PCI aperture as a reference.
 551 * Otherwise this would be needed for rv280, all r3xx, and all r4xx, but
 552 * not IGP.
 553 *
 554 * Note 2: we use mc_vram_size as on some board we need to program the mc to
 555 * cover the whole aperture even if VRAM size is inferior to aperture size
 556 * Novell bug 204882 + along with lots of ubuntu ones
 557 *
 558 * Note 3: when limiting vram it's safe to overwritte real_vram_size because
 559 * we are not in case where real_vram_size is inferior to mc_vram_size (ie
 560 * note afected by bogus hw of Novell bug 204882 + along with lots of ubuntu
 561 * ones)
 562 *
 563 * Note 4: IGP TOM addr should be the same as the aperture addr, we don't
 564 * explicitly check for that thought.
 565 *
 566 * FIXME: when reducing VRAM size align new size on power of 2.
 567 */
 568void radeon_vram_location(struct radeon_device *rdev, struct radeon_mc *mc, u64 base)
 569{
 570	uint64_t limit = (uint64_t)radeon_vram_limit << 20;
 571
 572	mc->vram_start = base;
 573	if (mc->mc_vram_size > (rdev->mc.mc_mask - base + 1)) {
 574		dev_warn(rdev->dev, "limiting VRAM to PCI aperture size\n");
 575		mc->real_vram_size = mc->aper_size;
 576		mc->mc_vram_size = mc->aper_size;
 577	}
 578	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 579	if (rdev->flags & RADEON_IS_AGP && mc->vram_end > mc->gtt_start && mc->vram_start <= mc->gtt_end) {
 580		dev_warn(rdev->dev, "limiting VRAM to PCI aperture size\n");
 581		mc->real_vram_size = mc->aper_size;
 582		mc->mc_vram_size = mc->aper_size;
 583	}
 584	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 585	if (limit && limit < mc->real_vram_size)
 586		mc->real_vram_size = limit;
 587	dev_info(rdev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
 588			mc->mc_vram_size >> 20, mc->vram_start,
 589			mc->vram_end, mc->real_vram_size >> 20);
 590}
 591
 592/**
 593 * radeon_gtt_location - try to find GTT location
 594 * @rdev: radeon device structure holding all necessary informations
 595 * @mc: memory controller structure holding memory informations
 596 *
 597 * Function will place try to place GTT before or after VRAM.
 598 *
 599 * If GTT size is bigger than space left then we ajust GTT size.
 600 * Thus function will never fails.
 601 *
 602 * FIXME: when reducing GTT size align new size on power of 2.
 603 */
 604void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
 605{
 606	u64 size_af, size_bf;
 607
 608	size_af = ((rdev->mc.mc_mask - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align;
 609	size_bf = mc->vram_start & ~mc->gtt_base_align;
 610	if (size_bf > size_af) {
 611		if (mc->gtt_size > size_bf) {
 612			dev_warn(rdev->dev, "limiting GTT\n");
 613			mc->gtt_size = size_bf;
 614		}
 615		mc->gtt_start = (mc->vram_start & ~mc->gtt_base_align) - mc->gtt_size;
 616	} else {
 617		if (mc->gtt_size > size_af) {
 618			dev_warn(rdev->dev, "limiting GTT\n");
 619			mc->gtt_size = size_af;
 620		}
 621		mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align;
 622	}
 623	mc->gtt_end = mc->gtt_start + mc->gtt_size - 1;
 624	dev_info(rdev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
 625			mc->gtt_size >> 20, mc->gtt_start, mc->gtt_end);
 626}
 627
 628/*
 629 * GPU helpers function.
 630 */
 631
 632/*
 633 * radeon_device_is_virtual - check if we are running is a virtual environment
 634 *
 635 * Check if the asic has been passed through to a VM (all asics).
 636 * Used at driver startup.
 637 * Returns true if virtual or false if not.
 638 */
 639bool radeon_device_is_virtual(void)
 640{
 641#ifdef CONFIG_X86
 642	return boot_cpu_has(X86_FEATURE_HYPERVISOR);
 643#else
 644	return false;
 645#endif
 646}
 647
 648/**
 649 * radeon_card_posted - check if the hw has already been initialized
 650 *
 651 * @rdev: radeon_device pointer
 652 *
 653 * Check if the asic has been initialized (all asics).
 654 * Used at driver startup.
 655 * Returns true if initialized or false if not.
 656 */
 657bool radeon_card_posted(struct radeon_device *rdev)
 658{
 659	uint32_t reg;
 660
 661	/* for pass through, always force asic_init for CI */
 662	if (rdev->family >= CHIP_BONAIRE &&
 663	    radeon_device_is_virtual())
 664		return false;
 665
 666	/* required for EFI mode on macbook2,1 which uses an r5xx asic */
 667	if (efi_enabled(EFI_BOOT) &&
 668	    (rdev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE) &&
 669	    (rdev->family < CHIP_R600))
 670		return false;
 671
 672	if (ASIC_IS_NODCE(rdev))
 673		goto check_memsize;
 674
 675	/* first check CRTCs */
 676	if (ASIC_IS_DCE4(rdev)) {
 677		reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) |
 678			RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
 679			if (rdev->num_crtc >= 4) {
 680				reg |= RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) |
 681					RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
 682			}
 683			if (rdev->num_crtc >= 6) {
 684				reg |= RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) |
 685					RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
 686			}
 687		if (reg & EVERGREEN_CRTC_MASTER_EN)
 688			return true;
 689	} else if (ASIC_IS_AVIVO(rdev)) {
 690		reg = RREG32(AVIVO_D1CRTC_CONTROL) |
 691		      RREG32(AVIVO_D2CRTC_CONTROL);
 692		if (reg & AVIVO_CRTC_EN) {
 693			return true;
 694		}
 695	} else {
 696		reg = RREG32(RADEON_CRTC_GEN_CNTL) |
 697		      RREG32(RADEON_CRTC2_GEN_CNTL);
 698		if (reg & RADEON_CRTC_EN) {
 699			return true;
 700		}
 701	}
 702
 703check_memsize:
 704	/* then check MEM_SIZE, in case the crtcs are off */
 705	if (rdev->family >= CHIP_R600)
 706		reg = RREG32(R600_CONFIG_MEMSIZE);
 707	else
 708		reg = RREG32(RADEON_CONFIG_MEMSIZE);
 709
 710	if (reg)
 711		return true;
 712
 713	return false;
 714
 715}
 716
 717/**
 718 * radeon_update_bandwidth_info - update display bandwidth params
 719 *
 720 * @rdev: radeon_device pointer
 721 *
 722 * Used when sclk/mclk are switched or display modes are set.
 723 * params are used to calculate display watermarks (all asics)
 724 */
 725void radeon_update_bandwidth_info(struct radeon_device *rdev)
 726{
 727	fixed20_12 a;
 728	u32 sclk = rdev->pm.current_sclk;
 729	u32 mclk = rdev->pm.current_mclk;
 730
 731	/* sclk/mclk in Mhz */
 732	a.full = dfixed_const(100);
 733	rdev->pm.sclk.full = dfixed_const(sclk);
 734	rdev->pm.sclk.full = dfixed_div(rdev->pm.sclk, a);
 735	rdev->pm.mclk.full = dfixed_const(mclk);
 736	rdev->pm.mclk.full = dfixed_div(rdev->pm.mclk, a);
 737
 738	if (rdev->flags & RADEON_IS_IGP) {
 739		a.full = dfixed_const(16);
 740		/* core_bandwidth = sclk(Mhz) * 16 */
 741		rdev->pm.core_bandwidth.full = dfixed_div(rdev->pm.sclk, a);
 742	}
 743}
 744
 745/**
 746 * radeon_boot_test_post_card - check and possibly initialize the hw
 747 *
 748 * @rdev: radeon_device pointer
 749 *
 750 * Check if the asic is initialized and if not, attempt to initialize
 751 * it (all asics).
 752 * Returns true if initialized or false if not.
 753 */
 754bool radeon_boot_test_post_card(struct radeon_device *rdev)
 755{
 756	if (radeon_card_posted(rdev))
 757		return true;
 758
 759	if (rdev->bios) {
 760		DRM_INFO("GPU not posted. posting now...\n");
 761		if (rdev->is_atom_bios)
 762			atom_asic_init(rdev->mode_info.atom_context);
 763		else
 764			radeon_combios_asic_init(rdev_to_drm(rdev));
 765		return true;
 766	} else {
 767		dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
 768		return false;
 769	}
 770}
 771
 772/**
 773 * radeon_dummy_page_init - init dummy page used by the driver
 774 *
 775 * @rdev: radeon_device pointer
 776 *
 777 * Allocate the dummy page used by the driver (all asics).
 778 * This dummy page is used by the driver as a filler for gart entries
 779 * when pages are taken out of the GART
 780 * Returns 0 on sucess, -ENOMEM on failure.
 781 */
 782int radeon_dummy_page_init(struct radeon_device *rdev)
 783{
 784	if (rdev->dummy_page.page)
 785		return 0;
 786	rdev->dummy_page.page = alloc_page(GFP_DMA32 | GFP_KERNEL | __GFP_ZERO);
 787	if (rdev->dummy_page.page == NULL)
 788		return -ENOMEM;
 789	rdev->dummy_page.addr = dma_map_page(&rdev->pdev->dev, rdev->dummy_page.page,
 790					0, PAGE_SIZE, DMA_BIDIRECTIONAL);
 791	if (dma_mapping_error(&rdev->pdev->dev, rdev->dummy_page.addr)) {
 792		dev_err(&rdev->pdev->dev, "Failed to DMA MAP the dummy page\n");
 793		__free_page(rdev->dummy_page.page);
 794		rdev->dummy_page.page = NULL;
 795		return -ENOMEM;
 796	}
 797	rdev->dummy_page.entry = radeon_gart_get_page_entry(rdev->dummy_page.addr,
 798							    RADEON_GART_PAGE_DUMMY);
 799	return 0;
 800}
 801
 802/**
 803 * radeon_dummy_page_fini - free dummy page used by the driver
 804 *
 805 * @rdev: radeon_device pointer
 806 *
 807 * Frees the dummy page used by the driver (all asics).
 808 */
 809void radeon_dummy_page_fini(struct radeon_device *rdev)
 810{
 811	if (rdev->dummy_page.page == NULL)
 812		return;
 813	dma_unmap_page(&rdev->pdev->dev, rdev->dummy_page.addr, PAGE_SIZE,
 814		       DMA_BIDIRECTIONAL);
 815	__free_page(rdev->dummy_page.page);
 816	rdev->dummy_page.page = NULL;
 817}
 818
 819
 820/* ATOM accessor methods */
 821/*
 822 * ATOM is an interpreted byte code stored in tables in the vbios.  The
 823 * driver registers callbacks to access registers and the interpreter
 824 * in the driver parses the tables and executes then to program specific
 825 * actions (set display modes, asic init, etc.).  See radeon_atombios.c,
 826 * atombios.h, and atom.c
 827 */
 828
 829/**
 830 * cail_pll_read - read PLL register
 831 *
 832 * @info: atom card_info pointer
 833 * @reg: PLL register offset
 834 *
 835 * Provides a PLL register accessor for the atom interpreter (r4xx+).
 836 * Returns the value of the PLL register.
 837 */
 838static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)
 839{
 840	struct radeon_device *rdev = info->dev->dev_private;
 841	uint32_t r;
 842
 843	r = rdev->pll_rreg(rdev, reg);
 844	return r;
 845}
 846
 847/**
 848 * cail_pll_write - write PLL register
 849 *
 850 * @info: atom card_info pointer
 851 * @reg: PLL register offset
 852 * @val: value to write to the pll register
 853 *
 854 * Provides a PLL register accessor for the atom interpreter (r4xx+).
 855 */
 856static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)
 857{
 858	struct radeon_device *rdev = info->dev->dev_private;
 859
 860	rdev->pll_wreg(rdev, reg, val);
 861}
 862
 863/**
 864 * cail_mc_read - read MC (Memory Controller) register
 865 *
 866 * @info: atom card_info pointer
 867 * @reg: MC register offset
 868 *
 869 * Provides an MC register accessor for the atom interpreter (r4xx+).
 870 * Returns the value of the MC register.
 871 */
 872static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
 873{
 874	struct radeon_device *rdev = info->dev->dev_private;
 875	uint32_t r;
 876
 877	r = rdev->mc_rreg(rdev, reg);
 878	return r;
 879}
 880
 881/**
 882 * cail_mc_write - write MC (Memory Controller) register
 883 *
 884 * @info: atom card_info pointer
 885 * @reg: MC register offset
 886 * @val: value to write to the pll register
 887 *
 888 * Provides a MC register accessor for the atom interpreter (r4xx+).
 889 */
 890static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
 891{
 892	struct radeon_device *rdev = info->dev->dev_private;
 893
 894	rdev->mc_wreg(rdev, reg, val);
 895}
 896
 897/**
 898 * cail_reg_write - write MMIO register
 899 *
 900 * @info: atom card_info pointer
 901 * @reg: MMIO register offset
 902 * @val: value to write to the pll register
 903 *
 904 * Provides a MMIO register accessor for the atom interpreter (r4xx+).
 905 */
 906static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
 907{
 908	struct radeon_device *rdev = info->dev->dev_private;
 909
 910	WREG32(reg*4, val);
 911}
 912
 913/**
 914 * cail_reg_read - read MMIO register
 915 *
 916 * @info: atom card_info pointer
 917 * @reg: MMIO register offset
 918 *
 919 * Provides an MMIO register accessor for the atom interpreter (r4xx+).
 920 * Returns the value of the MMIO register.
 921 */
 922static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
 923{
 924	struct radeon_device *rdev = info->dev->dev_private;
 925	uint32_t r;
 926
 927	r = RREG32(reg*4);
 928	return r;
 929}
 930
 931/**
 932 * cail_ioreg_write - write IO register
 933 *
 934 * @info: atom card_info pointer
 935 * @reg: IO register offset
 936 * @val: value to write to the pll register
 937 *
 938 * Provides a IO register accessor for the atom interpreter (r4xx+).
 939 */
 940static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)
 941{
 942	struct radeon_device *rdev = info->dev->dev_private;
 943
 944	WREG32_IO(reg*4, val);
 945}
 946
 947/**
 948 * cail_ioreg_read - read IO register
 949 *
 950 * @info: atom card_info pointer
 951 * @reg: IO register offset
 952 *
 953 * Provides an IO register accessor for the atom interpreter (r4xx+).
 954 * Returns the value of the IO register.
 955 */
 956static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
 957{
 958	struct radeon_device *rdev = info->dev->dev_private;
 959	uint32_t r;
 960
 961	r = RREG32_IO(reg*4);
 962	return r;
 963}
 964
 965/**
 966 * radeon_atombios_init - init the driver info and callbacks for atombios
 967 *
 968 * @rdev: radeon_device pointer
 969 *
 970 * Initializes the driver info and register access callbacks for the
 971 * ATOM interpreter (r4xx+).
 972 * Returns 0 on sucess, -ENOMEM on failure.
 973 * Called at driver startup.
 974 */
 975int radeon_atombios_init(struct radeon_device *rdev)
 976{
 977	struct card_info *atom_card_info =
 978	    kzalloc(sizeof(struct card_info), GFP_KERNEL);
 979
 980	if (!atom_card_info)
 981		return -ENOMEM;
 982
 983	rdev->mode_info.atom_card_info = atom_card_info;
 984	atom_card_info->dev = rdev_to_drm(rdev);
 985	atom_card_info->reg_read = cail_reg_read;
 986	atom_card_info->reg_write = cail_reg_write;
 987	/* needed for iio ops */
 988	if (rdev->rio_mem) {
 989		atom_card_info->ioreg_read = cail_ioreg_read;
 990		atom_card_info->ioreg_write = cail_ioreg_write;
 991	} else {
 992		DRM_ERROR("Unable to find PCI I/O BAR; using MMIO for ATOM IIO\n");
 993		atom_card_info->ioreg_read = cail_reg_read;
 994		atom_card_info->ioreg_write = cail_reg_write;
 995	}
 996	atom_card_info->mc_read = cail_mc_read;
 997	atom_card_info->mc_write = cail_mc_write;
 998	atom_card_info->pll_read = cail_pll_read;
 999	atom_card_info->pll_write = cail_pll_write;
1000
1001	rdev->mode_info.atom_context = atom_parse(atom_card_info, rdev->bios);
1002	if (!rdev->mode_info.atom_context) {
1003		radeon_atombios_fini(rdev);
1004		return -ENOMEM;
1005	}
1006
1007	mutex_init(&rdev->mode_info.atom_context->mutex);
1008	mutex_init(&rdev->mode_info.atom_context->scratch_mutex);
1009	radeon_atom_initialize_bios_scratch_regs(rdev_to_drm(rdev));
1010	atom_allocate_fb_scratch(rdev->mode_info.atom_context);
1011	return 0;
1012}
1013
1014/**
1015 * radeon_atombios_fini - free the driver info and callbacks for atombios
1016 *
1017 * @rdev: radeon_device pointer
1018 *
1019 * Frees the driver info and register access callbacks for the ATOM
1020 * interpreter (r4xx+).
1021 * Called at driver shutdown.
1022 */
1023void radeon_atombios_fini(struct radeon_device *rdev)
1024{
1025	if (rdev->mode_info.atom_context) {
1026		kfree(rdev->mode_info.atom_context->scratch);
1027		kfree(rdev->mode_info.atom_context->iio);
1028	}
1029	kfree(rdev->mode_info.atom_context);
1030	rdev->mode_info.atom_context = NULL;
1031	kfree(rdev->mode_info.atom_card_info);
1032	rdev->mode_info.atom_card_info = NULL;
1033}
1034
1035/* COMBIOS */
1036/*
1037 * COMBIOS is the bios format prior to ATOM. It provides
1038 * command tables similar to ATOM, but doesn't have a unified
1039 * parser.  See radeon_combios.c
1040 */
1041
1042/**
1043 * radeon_combios_init - init the driver info for combios
1044 *
1045 * @rdev: radeon_device pointer
1046 *
1047 * Initializes the driver info for combios (r1xx-r3xx).
1048 * Returns 0 on sucess.
1049 * Called at driver startup.
1050 */
1051int radeon_combios_init(struct radeon_device *rdev)
1052{
1053	radeon_combios_initialize_bios_scratch_regs(rdev_to_drm(rdev));
1054	return 0;
1055}
1056
1057/**
1058 * radeon_combios_fini - free the driver info for combios
1059 *
1060 * @rdev: radeon_device pointer
1061 *
1062 * Frees the driver info for combios (r1xx-r3xx).
1063 * Called at driver shutdown.
1064 */
1065void radeon_combios_fini(struct radeon_device *rdev)
1066{
1067}
1068
1069/* if we get transitioned to only one device, take VGA back */
1070/**
1071 * radeon_vga_set_decode - enable/disable vga decode
1072 *
1073 * @pdev: PCI device
1074 * @state: enable/disable vga decode
1075 *
1076 * Enable/disable vga decode (all asics).
1077 * Returns VGA resource flags.
1078 */
1079static unsigned int radeon_vga_set_decode(struct pci_dev *pdev, bool state)
1080{
1081	struct drm_device *dev = pci_get_drvdata(pdev);
1082	struct radeon_device *rdev = dev->dev_private;
1083	radeon_vga_set_state(rdev, state);
1084	if (state)
1085		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1086		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1087	else
1088		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1089}
1090
1091/**
1092 * radeon_gart_size_auto - Determine a sensible default GART size
1093 *                         according to ASIC family.
1094 *
1095 * @family: ASIC family name
1096 */
1097static int radeon_gart_size_auto(enum radeon_family family)
1098{
1099	/* default to a larger gart size on newer asics */
1100	if (family >= CHIP_TAHITI)
1101		return 2048;
1102	else if (family >= CHIP_RV770)
1103		return 1024;
1104	else
1105		return 512;
1106}
1107
1108/**
1109 * radeon_check_arguments - validate module params
1110 *
1111 * @rdev: radeon_device pointer
1112 *
1113 * Validates certain module parameters and updates
1114 * the associated values used by the driver (all asics).
1115 */
1116static void radeon_check_arguments(struct radeon_device *rdev)
1117{
1118	/* vramlimit must be a power of two */
1119	if (radeon_vram_limit != 0 && !is_power_of_2(radeon_vram_limit)) {
1120		dev_warn(rdev->dev, "vram limit (%d) must be a power of 2\n",
1121				radeon_vram_limit);
1122		radeon_vram_limit = 0;
1123	}
1124
1125	if (radeon_gart_size == -1) {
1126		radeon_gart_size = radeon_gart_size_auto(rdev->family);
1127	}
1128	/* gtt size must be power of two and greater or equal to 32M */
1129	if (radeon_gart_size < 32) {
1130		dev_warn(rdev->dev, "gart size (%d) too small\n",
1131				radeon_gart_size);
1132		radeon_gart_size = radeon_gart_size_auto(rdev->family);
1133	} else if (!is_power_of_2(radeon_gart_size)) {
1134		dev_warn(rdev->dev, "gart size (%d) must be a power of 2\n",
1135				radeon_gart_size);
1136		radeon_gart_size = radeon_gart_size_auto(rdev->family);
1137	}
1138	rdev->mc.gtt_size = (uint64_t)radeon_gart_size << 20;
1139
1140	/* AGP mode can only be -1, 1, 2, 4, 8 */
1141	switch (radeon_agpmode) {
1142	case -1:
1143	case 0:
1144	case 1:
1145	case 2:
1146	case 4:
1147	case 8:
1148		break;
1149	default:
1150		dev_warn(rdev->dev, "invalid AGP mode %d (valid mode: "
1151				"-1, 0, 1, 2, 4, 8)\n", radeon_agpmode);
1152		radeon_agpmode = 0;
1153		break;
1154	}
1155
1156	if (!is_power_of_2(radeon_vm_size)) {
1157		dev_warn(rdev->dev, "VM size (%d) must be a power of 2\n",
1158			 radeon_vm_size);
1159		radeon_vm_size = 4;
1160	}
1161
1162	if (radeon_vm_size < 1) {
1163		dev_warn(rdev->dev, "VM size (%d) too small, min is 1GB\n",
1164			 radeon_vm_size);
1165		radeon_vm_size = 4;
1166	}
1167
1168	/*
1169	 * Max GPUVM size for Cayman, SI and CI are 40 bits.
1170	 */
1171	if (radeon_vm_size > 1024) {
1172		dev_warn(rdev->dev, "VM size (%d) too large, max is 1TB\n",
1173			 radeon_vm_size);
1174		radeon_vm_size = 4;
1175	}
1176
1177	/* defines number of bits in page table versus page directory,
1178	 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1179	 * page table and the remaining bits are in the page directory */
1180	if (radeon_vm_block_size == -1) {
1181
1182		/* Total bits covered by PD + PTs */
1183		unsigned bits = ilog2(radeon_vm_size) + 18;
1184
1185		/* Make sure the PD is 4K in size up to 8GB address space.
1186		   Above that split equal between PD and PTs */
1187		if (radeon_vm_size <= 8)
1188			radeon_vm_block_size = bits - 9;
1189		else
1190			radeon_vm_block_size = (bits + 3) / 2;
1191
1192	} else if (radeon_vm_block_size < 9) {
1193		dev_warn(rdev->dev, "VM page table size (%d) too small\n",
1194			 radeon_vm_block_size);
1195		radeon_vm_block_size = 9;
1196	}
1197
1198	if (radeon_vm_block_size > 24 ||
1199	    (radeon_vm_size * 1024) < (1ull << radeon_vm_block_size)) {
1200		dev_warn(rdev->dev, "VM page table size (%d) too large\n",
1201			 radeon_vm_block_size);
1202		radeon_vm_block_size = 9;
1203	}
1204}
1205
1206/**
1207 * radeon_switcheroo_set_state - set switcheroo state
1208 *
1209 * @pdev: pci dev pointer
1210 * @state: vga_switcheroo state
1211 *
1212 * Callback for the switcheroo driver.  Suspends or resumes
1213 * the asics before or after it is powered up using ACPI methods.
1214 */
1215static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1216{
1217	struct drm_device *dev = pci_get_drvdata(pdev);
1218
1219	if (radeon_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1220		return;
1221
1222	if (state == VGA_SWITCHEROO_ON) {
1223		pr_info("radeon: switched on\n");
1224		/* don't suspend or resume card normally */
1225		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1226
1227		radeon_resume_kms(dev, true, true);
1228
1229		dev->switch_power_state = DRM_SWITCH_POWER_ON;
1230		drm_kms_helper_poll_enable(dev);
1231	} else {
1232		pr_info("radeon: switched off\n");
1233		drm_kms_helper_poll_disable(dev);
1234		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1235		radeon_suspend_kms(dev, true, true, false);
1236		dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1237	}
1238}
1239
1240/**
1241 * radeon_switcheroo_can_switch - see if switcheroo state can change
1242 *
1243 * @pdev: pci dev pointer
1244 *
1245 * Callback for the switcheroo driver.  Check of the switcheroo
1246 * state can be changed.
1247 * Returns true if the state can be changed, false if not.
1248 */
1249static bool radeon_switcheroo_can_switch(struct pci_dev *pdev)
1250{
1251	struct drm_device *dev = pci_get_drvdata(pdev);
1252
1253	/*
1254	 * FIXME: open_count is protected by drm_global_mutex but that would lead to
1255	 * locking inversion with the driver load path. And the access here is
1256	 * completely racy anyway. So don't bother with locking for now.
1257	 */
1258	return atomic_read(&dev->open_count) == 0;
1259}
1260
1261static const struct vga_switcheroo_client_ops radeon_switcheroo_ops = {
1262	.set_gpu_state = radeon_switcheroo_set_state,
1263	.reprobe = NULL,
1264	.can_switch = radeon_switcheroo_can_switch,
1265};
1266
1267/**
1268 * radeon_device_init - initialize the driver
1269 *
1270 * @rdev: radeon_device pointer
1271 * @ddev: drm dev pointer
1272 * @pdev: pci dev pointer
1273 * @flags: driver flags
1274 *
1275 * Initializes the driver info and hw (all asics).
1276 * Returns 0 for success or an error on failure.
1277 * Called at driver startup.
1278 */
1279int radeon_device_init(struct radeon_device *rdev,
1280		       struct drm_device *ddev,
1281		       struct pci_dev *pdev,
1282		       uint32_t flags)
1283{
1284	int r, i;
1285	int dma_bits;
1286	bool runtime = false;
1287
1288	rdev->shutdown = false;
 
 
 
1289	rdev->flags = flags;
1290	rdev->family = flags & RADEON_FAMILY_MASK;
1291	rdev->is_atom_bios = false;
1292	rdev->usec_timeout = RADEON_MAX_USEC_TIMEOUT;
1293	rdev->mc.gtt_size = 512 * 1024 * 1024;
1294	rdev->accel_working = false;
1295	/* set up ring ids */
1296	for (i = 0; i < RADEON_NUM_RINGS; i++) {
1297		rdev->ring[i].idx = i;
1298	}
1299	rdev->fence_context = dma_fence_context_alloc(RADEON_NUM_RINGS);
1300
1301	DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
1302		 radeon_family_name[rdev->family], pdev->vendor, pdev->device,
1303		 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
1304
1305	/* mutex initialization are all done here so we
1306	 * can recall function without having locking issues */
1307	mutex_init(&rdev->ring_lock);
1308	mutex_init(&rdev->dc_hw_i2c_mutex);
1309	atomic_set(&rdev->ih.lock, 0);
1310	mutex_init(&rdev->gem.mutex);
1311	mutex_init(&rdev->pm.mutex);
1312	mutex_init(&rdev->gpu_clock_mutex);
1313	mutex_init(&rdev->srbm_mutex);
1314	mutex_init(&rdev->audio.component_mutex);
1315	init_rwsem(&rdev->pm.mclk_lock);
1316	init_rwsem(&rdev->exclusive_lock);
1317	init_waitqueue_head(&rdev->irq.vblank_queue);
1318	r = radeon_gem_init(rdev);
1319	if (r)
1320		return r;
1321
1322	radeon_check_arguments(rdev);
1323	/* Adjust VM size here.
1324	 * Max GPUVM size for cayman+ is 40 bits.
1325	 */
1326	rdev->vm_manager.max_pfn = radeon_vm_size << 18;
1327
1328	/* Set asic functions */
1329	r = radeon_asic_init(rdev);
1330	if (r)
1331		return r;
1332
1333	/* all of the newer IGP chips have an internal gart
1334	 * However some rs4xx report as AGP, so remove that here.
1335	 */
1336	if ((rdev->family >= CHIP_RS400) &&
1337	    (rdev->flags & RADEON_IS_IGP)) {
1338		rdev->flags &= ~RADEON_IS_AGP;
1339	}
1340
1341	if (rdev->flags & RADEON_IS_AGP && radeon_agpmode == -1) {
1342		radeon_agp_disable(rdev);
1343	}
1344
1345	/* Set the internal MC address mask
1346	 * This is the max address of the GPU's
1347	 * internal address space.
1348	 */
1349	if (rdev->family >= CHIP_CAYMAN)
1350		rdev->mc.mc_mask = 0xffffffffffULL; /* 40 bit MC */
1351	else if (rdev->family >= CHIP_CEDAR)
1352		rdev->mc.mc_mask = 0xfffffffffULL; /* 36 bit MC */
1353	else
1354		rdev->mc.mc_mask = 0xffffffffULL; /* 32 bit MC */
1355
1356	/* set DMA mask.
1357	 * PCIE - can handle 40-bits.
1358	 * IGP - can handle 40-bits
1359	 * AGP - generally dma32 is safest
1360	 * PCI - dma32 for legacy pci gart, 40 bits on newer asics
1361	 */
1362	dma_bits = 40;
1363	if (rdev->flags & RADEON_IS_AGP)
1364		dma_bits = 32;
1365	if ((rdev->flags & RADEON_IS_PCI) &&
1366	    (rdev->family <= CHIP_RS740))
1367		dma_bits = 32;
1368#ifdef CONFIG_PPC64
1369	if (rdev->family == CHIP_CEDAR)
1370		dma_bits = 32;
1371#endif
1372
1373	r = dma_set_mask_and_coherent(&rdev->pdev->dev, DMA_BIT_MASK(dma_bits));
1374	if (r) {
1375		pr_warn("radeon: No suitable DMA available\n");
1376		return r;
1377	}
1378	rdev->need_swiotlb = drm_need_swiotlb(dma_bits);
1379
1380	/* Registers mapping */
1381	/* TODO: block userspace mapping of io register */
1382	spin_lock_init(&rdev->mmio_idx_lock);
1383	spin_lock_init(&rdev->smc_idx_lock);
1384	spin_lock_init(&rdev->pll_idx_lock);
1385	spin_lock_init(&rdev->mc_idx_lock);
1386	spin_lock_init(&rdev->pcie_idx_lock);
1387	spin_lock_init(&rdev->pciep_idx_lock);
1388	spin_lock_init(&rdev->pif_idx_lock);
1389	spin_lock_init(&rdev->cg_idx_lock);
1390	spin_lock_init(&rdev->uvd_idx_lock);
1391	spin_lock_init(&rdev->rcu_idx_lock);
1392	spin_lock_init(&rdev->didt_idx_lock);
1393	spin_lock_init(&rdev->end_idx_lock);
1394	if (rdev->family >= CHIP_BONAIRE) {
1395		rdev->rmmio_base = pci_resource_start(rdev->pdev, 5);
1396		rdev->rmmio_size = pci_resource_len(rdev->pdev, 5);
1397	} else {
1398		rdev->rmmio_base = pci_resource_start(rdev->pdev, 2);
1399		rdev->rmmio_size = pci_resource_len(rdev->pdev, 2);
1400	}
1401	rdev->rmmio = ioremap(rdev->rmmio_base, rdev->rmmio_size);
1402	if (rdev->rmmio == NULL)
1403		return -ENOMEM;
1404
1405	/* doorbell bar mapping */
1406	if (rdev->family >= CHIP_BONAIRE)
1407		radeon_doorbell_init(rdev);
1408
1409	/* io port mapping */
1410	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
1411		if (pci_resource_flags(rdev->pdev, i) & IORESOURCE_IO) {
1412			rdev->rio_mem_size = pci_resource_len(rdev->pdev, i);
1413			rdev->rio_mem = pci_iomap(rdev->pdev, i, rdev->rio_mem_size);
1414			break;
1415		}
1416	}
1417	if (rdev->rio_mem == NULL)
1418		DRM_ERROR("Unable to find PCI I/O BAR\n");
1419
1420	if (rdev->flags & RADEON_IS_PX)
1421		radeon_device_handle_px_quirks(rdev);
1422
1423	/* if we have > 1 VGA cards, then disable the radeon VGA resources */
1424	/* this will fail for cards that aren't VGA class devices, just
1425	 * ignore it */
1426	vga_client_register(rdev->pdev, radeon_vga_set_decode);
1427
1428	if (rdev->flags & RADEON_IS_PX)
1429		runtime = true;
1430	if (!pci_is_thunderbolt_attached(rdev->pdev))
1431		vga_switcheroo_register_client(rdev->pdev,
1432					       &radeon_switcheroo_ops, runtime);
1433	if (runtime)
1434		vga_switcheroo_init_domain_pm_ops(rdev->dev, &rdev->vga_pm_domain);
1435
1436	r = radeon_init(rdev);
1437	if (r)
1438		goto failed;
1439
1440	radeon_gem_debugfs_init(rdev);
1441
1442	if (rdev->flags & RADEON_IS_AGP && !rdev->accel_working) {
1443		/* Acceleration not working on AGP card try again
1444		 * with fallback to PCI or PCIE GART
1445		 */
1446		radeon_asic_reset(rdev);
1447		radeon_fini(rdev);
1448		radeon_agp_disable(rdev);
1449		r = radeon_init(rdev);
1450		if (r)
1451			goto failed;
1452	}
1453
1454	radeon_audio_component_init(rdev);
1455
1456	r = radeon_ib_ring_tests(rdev);
1457	if (r)
1458		DRM_ERROR("ib ring test failed (%d).\n", r);
1459
1460	/*
1461	 * Turks/Thames GPU will freeze whole laptop if DPM is not restarted
1462	 * after the CP ring have chew one packet at least. Hence here we stop
1463	 * and restart DPM after the radeon_ib_ring_tests().
1464	 */
1465	if (rdev->pm.dpm_enabled &&
1466	    (rdev->pm.pm_method == PM_METHOD_DPM) &&
1467	    (rdev->family == CHIP_TURKS) &&
1468	    (rdev->flags & RADEON_IS_MOBILITY)) {
1469		mutex_lock(&rdev->pm.mutex);
1470		radeon_dpm_disable(rdev);
1471		radeon_dpm_enable(rdev);
1472		mutex_unlock(&rdev->pm.mutex);
1473	}
1474
1475	if ((radeon_testing & 1)) {
1476		if (rdev->accel_working)
1477			radeon_test_moves(rdev);
1478		else
1479			DRM_INFO("radeon: acceleration disabled, skipping move tests\n");
1480	}
1481	if ((radeon_testing & 2)) {
1482		if (rdev->accel_working)
1483			radeon_test_syncing(rdev);
1484		else
1485			DRM_INFO("radeon: acceleration disabled, skipping sync tests\n");
1486	}
1487	if (radeon_benchmarking) {
1488		if (rdev->accel_working)
1489			radeon_benchmark(rdev, radeon_benchmarking);
1490		else
1491			DRM_INFO("radeon: acceleration disabled, skipping benchmarks\n");
1492	}
1493	return 0;
1494
1495failed:
1496	/* balance pm_runtime_get_sync() in radeon_driver_unload_kms() */
1497	if (radeon_is_px(ddev))
1498		pm_runtime_put_noidle(ddev->dev);
1499	if (runtime)
1500		vga_switcheroo_fini_domain_pm_ops(rdev->dev);
1501	return r;
1502}
1503
1504/**
1505 * radeon_device_fini - tear down the driver
1506 *
1507 * @rdev: radeon_device pointer
1508 *
1509 * Tear down the driver info (all asics).
1510 * Called at driver shutdown.
1511 */
1512void radeon_device_fini(struct radeon_device *rdev)
1513{
1514	DRM_INFO("radeon: finishing device.\n");
1515	rdev->shutdown = true;
1516	/* evict vram memory */
1517	radeon_bo_evict_vram(rdev);
1518	radeon_audio_component_fini(rdev);
1519	radeon_fini(rdev);
1520	if (!pci_is_thunderbolt_attached(rdev->pdev))
1521		vga_switcheroo_unregister_client(rdev->pdev);
1522	if (rdev->flags & RADEON_IS_PX)
1523		vga_switcheroo_fini_domain_pm_ops(rdev->dev);
1524	vga_client_unregister(rdev->pdev);
1525	if (rdev->rio_mem)
1526		pci_iounmap(rdev->pdev, rdev->rio_mem);
1527	rdev->rio_mem = NULL;
1528	iounmap(rdev->rmmio);
1529	rdev->rmmio = NULL;
1530	if (rdev->family >= CHIP_BONAIRE)
1531		radeon_doorbell_fini(rdev);
1532}
1533
1534
1535/*
1536 * Suspend & resume.
1537 */
1538/*
1539 * radeon_suspend_kms - initiate device suspend
1540 *
1541 * Puts the hw in the suspend state (all asics).
1542 * Returns 0 for success or an error on failure.
1543 * Called at driver suspend.
1544 */
1545int radeon_suspend_kms(struct drm_device *dev, bool suspend,
1546		       bool notify_clients, bool freeze)
1547{
1548	struct radeon_device *rdev;
1549	struct pci_dev *pdev;
1550	struct drm_crtc *crtc;
1551	struct drm_connector *connector;
1552	int i, r;
1553
1554	if (dev == NULL || dev->dev_private == NULL) {
1555		return -ENODEV;
1556	}
1557
1558	rdev = dev->dev_private;
1559	pdev = to_pci_dev(dev->dev);
1560
1561	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1562		return 0;
1563
1564	drm_kms_helper_poll_disable(dev);
1565
1566	drm_modeset_lock_all(dev);
1567	/* turn off display hw */
1568	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1569		drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
1570	}
1571	drm_modeset_unlock_all(dev);
1572
1573	/* unpin the front buffers and cursors */
1574	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
1575		struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
1576		struct drm_framebuffer *fb = crtc->primary->fb;
1577		struct radeon_bo *robj;
1578
1579		if (radeon_crtc->cursor_bo) {
1580			struct radeon_bo *robj = gem_to_radeon_bo(radeon_crtc->cursor_bo);
1581			r = radeon_bo_reserve(robj, false);
1582			if (r == 0) {
1583				radeon_bo_unpin(robj);
1584				radeon_bo_unreserve(robj);
1585			}
1586		}
1587
1588		if (fb == NULL || fb->obj[0] == NULL) {
1589			continue;
1590		}
1591		robj = gem_to_radeon_bo(fb->obj[0]);
1592		/* don't unpin kernel fb objects */
1593		if (!radeon_fbdev_robj_is_fb(rdev, robj)) {
1594			r = radeon_bo_reserve(robj, false);
1595			if (r == 0) {
1596				radeon_bo_unpin(robj);
1597				radeon_bo_unreserve(robj);
1598			}
1599		}
1600	}
1601	/* evict vram memory */
1602	radeon_bo_evict_vram(rdev);
1603
1604	/* wait for gpu to finish processing current batch */
1605	for (i = 0; i < RADEON_NUM_RINGS; i++) {
1606		r = radeon_fence_wait_empty(rdev, i);
1607		if (r) {
1608			/* delay GPU reset to resume */
1609			radeon_fence_driver_force_completion(rdev, i);
1610		} else {
1611			/* finish executing delayed work */
1612			flush_delayed_work(&rdev->fence_drv[i].lockup_work);
1613		}
1614	}
1615
1616	radeon_save_bios_scratch_regs(rdev);
1617
1618	radeon_suspend(rdev);
1619	radeon_hpd_fini(rdev);
1620	/* evict remaining vram memory
1621	 * This second call to evict vram is to evict the gart page table
1622	 * using the CPU.
1623	 */
1624	radeon_bo_evict_vram(rdev);
1625
1626	radeon_agp_suspend(rdev);
1627
1628	pci_save_state(pdev);
1629	if (freeze && rdev->family >= CHIP_CEDAR && !(rdev->flags & RADEON_IS_IGP)) {
1630		rdev->asic->asic_reset(rdev, true);
1631		pci_restore_state(pdev);
1632	} else if (suspend) {
1633		/* Shut down the device */
1634		pci_disable_device(pdev);
1635		pci_set_power_state(pdev, PCI_D3hot);
1636	}
1637
1638	if (notify_clients) {
1639		console_lock();
1640		drm_client_dev_suspend(dev, true);
1641		console_unlock();
1642	}
1643	return 0;
1644}
1645
1646/*
1647 * radeon_resume_kms - initiate device resume
1648 *
1649 * Bring the hw back to operating state (all asics).
1650 * Returns 0 for success or an error on failure.
1651 * Called at driver resume.
1652 */
1653int radeon_resume_kms(struct drm_device *dev, bool resume, bool notify_clients)
1654{
1655	struct drm_connector *connector;
1656	struct radeon_device *rdev = dev->dev_private;
1657	struct pci_dev *pdev = to_pci_dev(dev->dev);
1658	struct drm_crtc *crtc;
1659	int r;
1660
1661	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1662		return 0;
1663
1664	if (notify_clients) {
1665		console_lock();
1666	}
1667	if (resume) {
1668		pci_set_power_state(pdev, PCI_D0);
1669		pci_restore_state(pdev);
1670		if (pci_enable_device(pdev)) {
1671			if (notify_clients)
1672				console_unlock();
1673			return -1;
1674		}
1675	}
1676	/* resume AGP if in use */
1677	radeon_agp_resume(rdev);
1678	radeon_resume(rdev);
1679
1680	r = radeon_ib_ring_tests(rdev);
1681	if (r)
1682		DRM_ERROR("ib ring test failed (%d).\n", r);
1683
1684	if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
1685		/* do dpm late init */
1686		r = radeon_pm_late_init(rdev);
1687		if (r) {
1688			rdev->pm.dpm_enabled = false;
1689			DRM_ERROR("radeon_pm_late_init failed, disabling dpm\n");
1690		}
1691	} else {
1692		/* resume old pm late */
1693		radeon_pm_resume(rdev);
1694	}
1695
1696	radeon_restore_bios_scratch_regs(rdev);
1697
1698	/* pin cursors */
1699	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
1700		struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
1701
1702		if (radeon_crtc->cursor_bo) {
1703			struct radeon_bo *robj = gem_to_radeon_bo(radeon_crtc->cursor_bo);
1704			r = radeon_bo_reserve(robj, false);
1705			if (r == 0) {
1706				/* Only 27 bit offset for legacy cursor */
1707				r = radeon_bo_pin_restricted(robj,
1708							     RADEON_GEM_DOMAIN_VRAM,
1709							     ASIC_IS_AVIVO(rdev) ?
1710							     0 : 1 << 27,
1711							     &radeon_crtc->cursor_addr);
1712				if (r != 0)
1713					DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
1714				radeon_bo_unreserve(robj);
1715			}
1716		}
1717	}
1718
1719	/* init dig PHYs, disp eng pll */
1720	if (rdev->is_atom_bios) {
1721		radeon_atom_encoder_init(rdev);
1722		radeon_atom_disp_eng_pll_init(rdev);
1723		/* turn on the BL */
1724		if (rdev->mode_info.bl_encoder) {
1725			u8 bl_level = radeon_get_backlight_level(rdev,
1726								 rdev->mode_info.bl_encoder);
1727			radeon_set_backlight_level(rdev, rdev->mode_info.bl_encoder,
1728						   bl_level);
1729		}
1730	}
1731	/* reset hpd state */
1732	radeon_hpd_init(rdev);
1733	/* blat the mode back in */
1734	if (notify_clients) {
1735		drm_helper_resume_force_mode(dev);
1736		/* turn on display hw */
1737		drm_modeset_lock_all(dev);
1738		list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1739			drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
1740		}
1741		drm_modeset_unlock_all(dev);
1742	}
1743
1744	drm_kms_helper_poll_enable(dev);
1745
1746	/* set the power state here in case we are a PX system or headless */
1747	if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled)
1748		radeon_pm_compute_clocks(rdev);
1749
1750	if (notify_clients) {
1751		drm_client_dev_resume(dev, true);
1752		console_unlock();
1753	}
1754
1755	return 0;
1756}
1757
1758/**
1759 * radeon_gpu_reset - reset the asic
1760 *
1761 * @rdev: radeon device pointer
1762 *
1763 * Attempt the reset the GPU if it has hung (all asics).
1764 * Returns 0 for success or an error on failure.
1765 */
1766int radeon_gpu_reset(struct radeon_device *rdev)
1767{
1768	unsigned ring_sizes[RADEON_NUM_RINGS];
1769	uint32_t *ring_data[RADEON_NUM_RINGS];
1770
1771	bool saved = false;
1772
1773	int i, r;
 
1774
1775	down_write(&rdev->exclusive_lock);
1776
1777	if (!rdev->needs_reset) {
1778		up_write(&rdev->exclusive_lock);
1779		return 0;
1780	}
1781
1782	atomic_inc(&rdev->gpu_reset_counter);
1783
1784	radeon_save_bios_scratch_regs(rdev);
 
 
1785	radeon_suspend(rdev);
1786	radeon_hpd_fini(rdev);
1787
1788	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
1789		ring_sizes[i] = radeon_ring_backup(rdev, &rdev->ring[i],
1790						   &ring_data[i]);
1791		if (ring_sizes[i]) {
1792			saved = true;
1793			dev_info(rdev->dev, "Saved %d dwords of commands "
1794				 "on ring %d.\n", ring_sizes[i], i);
1795		}
1796	}
1797
1798	r = radeon_asic_reset(rdev);
1799	if (!r) {
1800		dev_info(rdev->dev, "GPU reset succeeded, trying to resume\n");
1801		radeon_resume(rdev);
1802	}
1803
1804	radeon_restore_bios_scratch_regs(rdev);
1805
1806	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
1807		if (!r && ring_data[i]) {
1808			radeon_ring_restore(rdev, &rdev->ring[i],
1809					    ring_sizes[i], ring_data[i]);
1810		} else {
1811			radeon_fence_driver_force_completion(rdev, i);
1812			kfree(ring_data[i]);
1813		}
1814	}
1815
1816	if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
1817		/* do dpm late init */
1818		r = radeon_pm_late_init(rdev);
1819		if (r) {
1820			rdev->pm.dpm_enabled = false;
1821			DRM_ERROR("radeon_pm_late_init failed, disabling dpm\n");
1822		}
1823	} else {
1824		/* resume old pm late */
1825		radeon_pm_resume(rdev);
1826	}
1827
1828	/* init dig PHYs, disp eng pll */
1829	if (rdev->is_atom_bios) {
1830		radeon_atom_encoder_init(rdev);
1831		radeon_atom_disp_eng_pll_init(rdev);
1832		/* turn on the BL */
1833		if (rdev->mode_info.bl_encoder) {
1834			u8 bl_level = radeon_get_backlight_level(rdev,
1835								 rdev->mode_info.bl_encoder);
1836			radeon_set_backlight_level(rdev, rdev->mode_info.bl_encoder,
1837						   bl_level);
1838		}
1839	}
1840	/* reset hpd state */
1841	radeon_hpd_init(rdev);
1842
 
 
1843	rdev->in_reset = true;
1844	rdev->needs_reset = false;
1845
1846	downgrade_write(&rdev->exclusive_lock);
1847
1848	drm_helper_resume_force_mode(rdev_to_drm(rdev));
1849
1850	/* set the power state here in case we are a PX system or headless */
1851	if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled)
1852		radeon_pm_compute_clocks(rdev);
1853
1854	if (!r) {
1855		r = radeon_ib_ring_tests(rdev);
1856		if (r && saved)
1857			r = -EAGAIN;
1858	} else {
1859		/* bad news, how to tell it to userspace ? */
1860		dev_info(rdev->dev, "GPU reset failed\n");
1861	}
1862
1863	rdev->needs_reset = r == -EAGAIN;
1864	rdev->in_reset = false;
1865
1866	up_read(&rdev->exclusive_lock);
1867	return r;
1868}