1/*
   2 * Copyright 2008 Advanced Micro Devices, Inc.
   3 * Copyright 2008 Red Hat Inc.
   4 * Copyright 2009 Jerome Glisse.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 *
  24 * Authors: Dave Airlie
  25 *          Alex Deucher
  26 *          Jerome Glisse
  27 */
  28#ifndef __AMDGPU_H__
  29#define __AMDGPU_H__
  30
  31#include <linux/atomic.h>
  32#include <linux/wait.h>
  33#include <linux/list.h>
  34#include <linux/kref.h>
  35#include <linux/interval_tree.h>
  36#include <linux/hashtable.h>
  37#include <linux/dma-fence.h>
  38
  39#include <ttm/ttm_bo_api.h>
  40#include <ttm/ttm_bo_driver.h>
  41#include <ttm/ttm_placement.h>
  42#include <ttm/ttm_module.h>
  43#include <ttm/ttm_execbuf_util.h>
  44
  45#include <drm/drmP.h>
  46#include <drm/drm_gem.h>
  47#include <drm/amdgpu_drm.h>
 
 
 
 
 
  48
  49#include "amd_shared.h"
  50#include "amdgpu_mode.h"
  51#include "amdgpu_ih.h"
  52#include "amdgpu_irq.h"
  53#include "amdgpu_ucode.h"
  54#include "amdgpu_ttm.h"
 
  55#include "amdgpu_gds.h"
  56#include "amdgpu_sync.h"
  57#include "amdgpu_ring.h"
  58#include "amdgpu_vm.h"
  59#include "amd_powerplay.h"
  60#include "amdgpu_dpm.h"
  61#include "amdgpu_acp.h"
  62
  63#include "gpu_scheduler.h"
 
 
 
 
  64#include "amdgpu_virt.h"
 
 
  65
  66/*
  67 * Modules parameters.
  68 */
  69extern int amdgpu_modeset;
  70extern int amdgpu_vram_limit;
 
  71extern int amdgpu_gart_size;
 
  72extern int amdgpu_moverate;
  73extern int amdgpu_benchmarking;
  74extern int amdgpu_testing;
  75extern int amdgpu_audio;
  76extern int amdgpu_disp_priority;
  77extern int amdgpu_hw_i2c;
  78extern int amdgpu_pcie_gen2;
  79extern int amdgpu_msi;
  80extern int amdgpu_lockup_timeout;
  81extern int amdgpu_dpm;
  82extern int amdgpu_smc_load_fw;
  83extern int amdgpu_aspm;
  84extern int amdgpu_runtime_pm;
  85extern unsigned amdgpu_ip_block_mask;
  86extern int amdgpu_bapm;
  87extern int amdgpu_deep_color;
  88extern int amdgpu_vm_size;
  89extern int amdgpu_vm_block_size;
 
  90extern int amdgpu_vm_fault_stop;
  91extern int amdgpu_vm_debug;
 
 
 
  92extern int amdgpu_sched_jobs;
  93extern int amdgpu_sched_hw_submission;
  94extern int amdgpu_powerplay;
  95extern int amdgpu_no_evict;
  96extern int amdgpu_direct_gma_size;
  97extern unsigned amdgpu_pcie_gen_cap;
  98extern unsigned amdgpu_pcie_lane_cap;
  99extern unsigned amdgpu_cg_mask;
 100extern unsigned amdgpu_pg_mask;
 
 101extern char *amdgpu_disable_cu;
 102extern char *amdgpu_virtual_display;
 103extern unsigned amdgpu_pp_feature_mask;
 104extern int amdgpu_vram_page_split;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 105
 
 106#define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS	        3000
 107#define AMDGPU_MAX_USEC_TIMEOUT			100000	/* 100 ms */
 108#define AMDGPU_FENCE_JIFFIES_TIMEOUT		(HZ / 2)
 109/* AMDGPU_IB_POOL_SIZE must be a power of 2 */
 110#define AMDGPU_IB_POOL_SIZE			16
 111#define AMDGPU_DEBUGFS_MAX_COMPONENTS		32
 112#define AMDGPUFB_CONN_LIMIT			4
 113#define AMDGPU_BIOS_NUM_SCRATCH			8
 114
 115/* max number of IP instances */
 116#define AMDGPU_MAX_SDMA_INSTANCES		2
 117
 118/* hardcode that limit for now */
 119#define AMDGPU_VA_RESERVED_SIZE			(8 << 20)
 120
 121/* hard reset data */
 122#define AMDGPU_ASIC_RESET_DATA                  0x39d5e86b
 123
 124/* reset flags */
 125#define AMDGPU_RESET_GFX			(1 << 0)
 126#define AMDGPU_RESET_COMPUTE			(1 << 1)
 127#define AMDGPU_RESET_DMA			(1 << 2)
 128#define AMDGPU_RESET_CP				(1 << 3)
 129#define AMDGPU_RESET_GRBM			(1 << 4)
 130#define AMDGPU_RESET_DMA1			(1 << 5)
 131#define AMDGPU_RESET_RLC			(1 << 6)
 132#define AMDGPU_RESET_SEM			(1 << 7)
 133#define AMDGPU_RESET_IH				(1 << 8)
 134#define AMDGPU_RESET_VMC			(1 << 9)
 135#define AMDGPU_RESET_MC				(1 << 10)
 136#define AMDGPU_RESET_DISPLAY			(1 << 11)
 137#define AMDGPU_RESET_UVD			(1 << 12)
 138#define AMDGPU_RESET_VCE			(1 << 13)
 139#define AMDGPU_RESET_VCE1			(1 << 14)
 140
 141/* GFX current status */
 142#define AMDGPU_GFX_NORMAL_MODE			0x00000000L
 143#define AMDGPU_GFX_SAFE_MODE			0x00000001L
 144#define AMDGPU_GFX_PG_DISABLED_MODE		0x00000002L
 145#define AMDGPU_GFX_CG_DISABLED_MODE		0x00000004L
 146#define AMDGPU_GFX_LBPW_DISABLED_MODE		0x00000008L
 147
 148/* max cursor sizes (in pixels) */
 149#define CIK_CURSOR_WIDTH 128
 150#define CIK_CURSOR_HEIGHT 128
 151
 152struct amdgpu_device;
 153struct amdgpu_ib;
 154struct amdgpu_cs_parser;
 155struct amdgpu_job;
 156struct amdgpu_irq_src;
 157struct amdgpu_fpriv;
 
 158
 159enum amdgpu_cp_irq {
 160	AMDGPU_CP_IRQ_GFX_EOP = 0,
 161	AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP,
 162	AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP,
 163	AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP,
 164	AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP,
 165	AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP,
 166	AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP,
 167	AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP,
 168	AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP,
 169
 170	AMDGPU_CP_IRQ_LAST
 171};
 172
 173enum amdgpu_sdma_irq {
 174	AMDGPU_SDMA_IRQ_TRAP0 = 0,
 175	AMDGPU_SDMA_IRQ_TRAP1,
 176
 177	AMDGPU_SDMA_IRQ_LAST
 178};
 179
 180enum amdgpu_thermal_irq {
 181	AMDGPU_THERMAL_IRQ_LOW_TO_HIGH = 0,
 182	AMDGPU_THERMAL_IRQ_HIGH_TO_LOW,
 183
 184	AMDGPU_THERMAL_IRQ_LAST
 185};
 186
 187int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
 188				  enum amd_ip_block_type block_type,
 189				  enum amd_clockgating_state state);
 190int amdgpu_set_powergating_state(struct amdgpu_device *adev,
 191				  enum amd_ip_block_type block_type,
 192				  enum amd_powergating_state state);
 193int amdgpu_wait_for_idle(struct amdgpu_device *adev,
 194			 enum amd_ip_block_type block_type);
 195bool amdgpu_is_idle(struct amdgpu_device *adev,
 196		    enum amd_ip_block_type block_type);
 
 
 
 
 
 
 
 197
 198#define AMDGPU_MAX_IP_NUM 16
 199
 200struct amdgpu_ip_block_status {
 201	bool valid;
 202	bool sw;
 203	bool hw;
 204	bool late_initialized;
 205	bool hang;
 206};
 207
 208struct amdgpu_ip_block_version {
 209	const enum amd_ip_block_type type;
 210	const u32 major;
 211	const u32 minor;
 212	const u32 rev;
 213	const struct amd_ip_funcs *funcs;
 214};
 215
 216struct amdgpu_ip_block {
 217	struct amdgpu_ip_block_status status;
 218	const struct amdgpu_ip_block_version *version;
 219};
 220
 221int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
 222				enum amd_ip_block_type type,
 223				u32 major, u32 minor);
 224
 225struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
 226					     enum amd_ip_block_type type);
 
 227
 228int amdgpu_ip_block_add(struct amdgpu_device *adev,
 229			const struct amdgpu_ip_block_version *ip_block_version);
 230
 231/* provided by hw blocks that can move/clear data.  e.g., gfx or sdma */
 232struct amdgpu_buffer_funcs {
 233	/* maximum bytes in a single operation */
 234	uint32_t	copy_max_bytes;
 235
 236	/* number of dw to reserve per operation */
 237	unsigned	copy_num_dw;
 238
 239	/* used for buffer migration */
 240	void (*emit_copy_buffer)(struct amdgpu_ib *ib,
 241				 /* src addr in bytes */
 242				 uint64_t src_offset,
 243				 /* dst addr in bytes */
 244				 uint64_t dst_offset,
 245				 /* number of byte to transfer */
 246				 uint32_t byte_count);
 247
 248	/* maximum bytes in a single operation */
 249	uint32_t	fill_max_bytes;
 250
 251	/* number of dw to reserve per operation */
 252	unsigned	fill_num_dw;
 253
 254	/* used for buffer clearing */
 255	void (*emit_fill_buffer)(struct amdgpu_ib *ib,
 256				 /* value to write to memory */
 257				 uint32_t src_data,
 258				 /* dst addr in bytes */
 259				 uint64_t dst_offset,
 260				 /* number of byte to fill */
 261				 uint32_t byte_count);
 262};
 263
 264/* provided by hw blocks that can write ptes, e.g., sdma */
 265struct amdgpu_vm_pte_funcs {
 
 
 
 266	/* copy pte entries from GART */
 267	void (*copy_pte)(struct amdgpu_ib *ib,
 268			 uint64_t pe, uint64_t src,
 269			 unsigned count);
 
 270	/* write pte one entry at a time with addr mapping */
 271	void (*write_pte)(struct amdgpu_ib *ib, uint64_t pe,
 272			  uint64_t value, unsigned count,
 273			  uint32_t incr);
 274	/* for linear pte/pde updates without addr mapping */
 275	void (*set_pte_pde)(struct amdgpu_ib *ib,
 276			    uint64_t pe,
 277			    uint64_t addr, unsigned count,
 278			    uint32_t incr, uint32_t flags);
 279};
 280
 281/* provided by the gmc block */
 282struct amdgpu_gart_funcs {
 283	/* flush the vm tlb via mmio */
 284	void (*flush_gpu_tlb)(struct amdgpu_device *adev,
 285			      uint32_t vmid);
 286	/* write pte/pde updates using the cpu */
 287	int (*set_pte_pde)(struct amdgpu_device *adev,
 288			   void *cpu_pt_addr, /* cpu addr of page table */
 289			   uint32_t gpu_page_idx, /* pte/pde to update */
 290			   uint64_t addr, /* addr to write into pte/pde */
 291			   uint32_t flags); /* access flags */
 292};
 293
 294/* provided by the ih block */
 295struct amdgpu_ih_funcs {
 296	/* ring read/write ptr handling, called from interrupt context */
 297	u32 (*get_wptr)(struct amdgpu_device *adev);
 
 298	void (*decode_iv)(struct amdgpu_device *adev,
 299			  struct amdgpu_iv_entry *entry);
 300	void (*set_rptr)(struct amdgpu_device *adev);
 301};
 302
 303/*
 304 * BIOS.
 305 */
 306bool amdgpu_get_bios(struct amdgpu_device *adev);
 307bool amdgpu_read_bios(struct amdgpu_device *adev);
 308
 309/*
 310 * Dummy page
 311 */
 312struct amdgpu_dummy_page {
 313	struct page	*page;
 314	dma_addr_t	addr;
 315};
 316int amdgpu_dummy_page_init(struct amdgpu_device *adev);
 317void amdgpu_dummy_page_fini(struct amdgpu_device *adev);
 318
 319
 320/*
 321 * Clocks
 322 */
 323
 324#define AMDGPU_MAX_PPLL 3
 325
 326struct amdgpu_clock {
 327	struct amdgpu_pll ppll[AMDGPU_MAX_PPLL];
 328	struct amdgpu_pll spll;
 329	struct amdgpu_pll mpll;
 330	/* 10 Khz units */
 331	uint32_t default_mclk;
 332	uint32_t default_sclk;
 333	uint32_t default_dispclk;
 334	uint32_t current_dispclk;
 335	uint32_t dp_extclk;
 336	uint32_t max_pixel_clock;
 337};
 338
 339/*
 340 * BO.
 341 */
 342struct amdgpu_bo_list_entry {
 343	struct amdgpu_bo		*robj;
 344	struct ttm_validate_buffer	tv;
 345	struct amdgpu_bo_va		*bo_va;
 346	uint32_t			priority;
 347	struct page			**user_pages;
 348	int				user_invalidated;
 349};
 350
 351struct amdgpu_bo_va_mapping {
 352	struct list_head		list;
 353	struct interval_tree_node	it;
 354	uint64_t			offset;
 355	uint32_t			flags;
 356};
 357
 358/* bo virtual addresses in a specific vm */
 359struct amdgpu_bo_va {
 360	/* protected by bo being reserved */
 361	struct list_head		bo_list;
 362	struct dma_fence	        *last_pt_update;
 363	unsigned			ref_count;
 364
 365	/* protected by vm mutex and spinlock */
 366	struct list_head		vm_status;
 367
 368	/* mappings for this bo_va */
 369	struct list_head		invalids;
 370	struct list_head		valids;
 371
 372	/* constant after initialization */
 373	struct amdgpu_vm		*vm;
 374	struct amdgpu_bo		*bo;
 375};
 376
 377#define AMDGPU_GEM_DOMAIN_MAX		0x3
 378
 379struct amdgpu_bo {
 380	/* Protected by tbo.reserved */
 381	u32				prefered_domains;
 382	u32				allowed_domains;
 383	struct ttm_place		placements[AMDGPU_GEM_DOMAIN_MAX + 1];
 384	struct ttm_placement		placement;
 385	struct ttm_buffer_object	tbo;
 386	struct ttm_bo_kmap_obj		kmap;
 387	u64				flags;
 388	unsigned			pin_count;
 389	void				*kptr;
 390	u64				tiling_flags;
 391	u64				metadata_flags;
 392	void				*metadata;
 393	u32				metadata_size;
 394	unsigned			prime_shared_count;
 395	/* list of all virtual address to which this bo
 396	 * is associated to
 397	 */
 398	struct list_head		va;
 399	/* Constant after initialization */
 400	struct drm_gem_object		gem_base;
 401	struct amdgpu_bo		*parent;
 402	struct amdgpu_bo		*shadow;
 403
 404	struct ttm_bo_kmap_obj		dma_buf_vmap;
 405	struct amdgpu_mn		*mn;
 406	struct list_head		mn_list;
 407	struct list_head		shadow_list;
 408};
 409#define gem_to_amdgpu_bo(gobj) container_of((gobj), struct amdgpu_bo, gem_base)
 410
 411void amdgpu_gem_object_free(struct drm_gem_object *obj);
 412int amdgpu_gem_object_open(struct drm_gem_object *obj,
 413				struct drm_file *file_priv);
 414void amdgpu_gem_object_close(struct drm_gem_object *obj,
 415				struct drm_file *file_priv);
 416unsigned long amdgpu_gem_timeout(uint64_t timeout_ns);
 417struct sg_table *amdgpu_gem_prime_get_sg_table(struct drm_gem_object *obj);
 418struct drm_gem_object *
 419amdgpu_gem_prime_import_sg_table(struct drm_device *dev,
 420				 struct dma_buf_attachment *attach,
 421				 struct sg_table *sg);
 422struct dma_buf *amdgpu_gem_prime_export(struct drm_device *dev,
 423					struct drm_gem_object *gobj,
 424					int flags);
 425int amdgpu_gem_prime_pin(struct drm_gem_object *obj);
 426void amdgpu_gem_prime_unpin(struct drm_gem_object *obj);
 427struct reservation_object *amdgpu_gem_prime_res_obj(struct drm_gem_object *);
 428void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj);
 429void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
 430int amdgpu_gem_debugfs_init(struct amdgpu_device *adev);
 431
 432/* sub-allocation manager, it has to be protected by another lock.
 433 * By conception this is an helper for other part of the driver
 434 * like the indirect buffer or semaphore, which both have their
 435 * locking.
 436 *
 437 * Principe is simple, we keep a list of sub allocation in offset
 438 * order (first entry has offset == 0, last entry has the highest
 439 * offset).
 440 *
 441 * When allocating new object we first check if there is room at
 442 * the end total_size - (last_object_offset + last_object_size) >=
 443 * alloc_size. If so we allocate new object there.
 444 *
 445 * When there is not enough room at the end, we start waiting for
 446 * each sub object until we reach object_offset+object_size >=
 447 * alloc_size, this object then become the sub object we return.
 448 *
 449 * Alignment can't be bigger than page size.
 450 *
 451 * Hole are not considered for allocation to keep things simple.
 452 * Assumption is that there won't be hole (all object on same
 453 * alignment).
 454 */
 455
 456#define AMDGPU_SA_NUM_FENCE_LISTS	32
 457
 458struct amdgpu_sa_manager {
 459	wait_queue_head_t	wq;
 460	struct amdgpu_bo	*bo;
 461	struct list_head	*hole;
 462	struct list_head	flist[AMDGPU_SA_NUM_FENCE_LISTS];
 463	struct list_head	olist;
 464	unsigned		size;
 465	uint64_t		gpu_addr;
 466	void			*cpu_ptr;
 467	uint32_t		domain;
 468	uint32_t		align;
 469};
 470
 471/* sub-allocation buffer */
 472struct amdgpu_sa_bo {
 473	struct list_head		olist;
 474	struct list_head		flist;
 475	struct amdgpu_sa_manager	*manager;
 476	unsigned			soffset;
 477	unsigned			eoffset;
 478	struct dma_fence	        *fence;
 479};
 480
 481/*
 482 * GEM objects.
 483 */
 484void amdgpu_gem_force_release(struct amdgpu_device *adev);
 485int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 486				int alignment, u32 initial_domain,
 487				u64 flags, bool kernel,
 488				struct drm_gem_object **obj);
 
 489
 490int amdgpu_mode_dumb_create(struct drm_file *file_priv,
 491			    struct drm_device *dev,
 492			    struct drm_mode_create_dumb *args);
 493int amdgpu_mode_dumb_mmap(struct drm_file *filp,
 494			  struct drm_device *dev,
 495			  uint32_t handle, uint64_t *offset_p);
 496int amdgpu_fence_slab_init(void);
 497void amdgpu_fence_slab_fini(void);
 498
 499/*
 500 * GART structures, functions & helpers
 501 */
 502struct amdgpu_mc;
 503
 504#define AMDGPU_GPU_PAGE_SIZE 4096
 505#define AMDGPU_GPU_PAGE_MASK (AMDGPU_GPU_PAGE_SIZE - 1)
 506#define AMDGPU_GPU_PAGE_SHIFT 12
 507#define AMDGPU_GPU_PAGE_ALIGN(a) (((a) + AMDGPU_GPU_PAGE_MASK) & ~AMDGPU_GPU_PAGE_MASK)
 508
 509struct amdgpu_gart {
 510	dma_addr_t			table_addr;
 511	struct amdgpu_bo		*robj;
 512	void				*ptr;
 513	unsigned			num_gpu_pages;
 514	unsigned			num_cpu_pages;
 515	unsigned			table_size;
 516#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
 517	struct page			**pages;
 518#endif
 519	bool				ready;
 520	const struct amdgpu_gart_funcs *gart_funcs;
 521};
 522
 523int amdgpu_gart_table_ram_alloc(struct amdgpu_device *adev);
 524void amdgpu_gart_table_ram_free(struct amdgpu_device *adev);
 525int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev);
 526void amdgpu_gart_table_vram_free(struct amdgpu_device *adev);
 527int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev);
 528void amdgpu_gart_table_vram_unpin(struct amdgpu_device *adev);
 529int amdgpu_gart_init(struct amdgpu_device *adev);
 530void amdgpu_gart_fini(struct amdgpu_device *adev);
 531void amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset,
 532			int pages);
 533int amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset,
 534		     int pages, struct page **pagelist,
 535		     dma_addr_t *dma_addr, uint32_t flags);
 536int amdgpu_ttm_recover_gart(struct amdgpu_device *adev);
 537
 538/*
 539 * GPU MC structures, functions & helpers
 540 */
 541struct amdgpu_mc {
 542	resource_size_t		aper_size;
 543	resource_size_t		aper_base;
 544	resource_size_t		agp_base;
 545	/* for some chips with <= 32MB we need to lie
 546	 * about vram size near mc fb location */
 547	u64			mc_vram_size;
 548	u64			visible_vram_size;
 549	u64			gtt_size;
 550	u64			gtt_start;
 551	u64			gtt_end;
 552	u64			vram_start;
 553	u64			vram_end;
 554	unsigned		vram_width;
 555	u64			real_vram_size;
 556	int			vram_mtrr;
 557	u64                     gtt_base_align;
 558	u64                     mc_mask;
 559	const struct firmware   *fw;	/* MC firmware */
 560	uint32_t                fw_version;
 561	struct amdgpu_irq_src	vm_fault;
 562	uint32_t		vram_type;
 563	uint32_t                srbm_soft_reset;
 564	struct amdgpu_mode_mc_save save;
 565};
 566
 567/*
 568 * GPU doorbell structures, functions & helpers
 569 */
 570typedef enum _AMDGPU_DOORBELL_ASSIGNMENT
 571{
 572	AMDGPU_DOORBELL_KIQ                     = 0x000,
 573	AMDGPU_DOORBELL_HIQ                     = 0x001,
 574	AMDGPU_DOORBELL_DIQ                     = 0x002,
 575	AMDGPU_DOORBELL_MEC_RING0               = 0x010,
 576	AMDGPU_DOORBELL_MEC_RING1               = 0x011,
 577	AMDGPU_DOORBELL_MEC_RING2               = 0x012,
 578	AMDGPU_DOORBELL_MEC_RING3               = 0x013,
 579	AMDGPU_DOORBELL_MEC_RING4               = 0x014,
 580	AMDGPU_DOORBELL_MEC_RING5               = 0x015,
 581	AMDGPU_DOORBELL_MEC_RING6               = 0x016,
 582	AMDGPU_DOORBELL_MEC_RING7               = 0x017,
 583	AMDGPU_DOORBELL_GFX_RING0               = 0x020,
 584	AMDGPU_DOORBELL_sDMA_ENGINE0            = 0x1E0,
 585	AMDGPU_DOORBELL_sDMA_ENGINE1            = 0x1E1,
 586	AMDGPU_DOORBELL_IH                      = 0x1E8,
 587	AMDGPU_DOORBELL_MAX_ASSIGNMENT          = 0x3FF,
 588	AMDGPU_DOORBELL_INVALID                 = 0xFFFF
 589} AMDGPU_DOORBELL_ASSIGNMENT;
 590
 591struct amdgpu_doorbell {
 592	/* doorbell mmio */
 593	resource_size_t		base;
 594	resource_size_t		size;
 595	u32 __iomem		*ptr;
 596	u32			num_doorbells;	/* Number of doorbells actually reserved for amdgpu. */
 597};
 598
 599void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
 600				phys_addr_t *aperture_base,
 601				size_t *aperture_size,
 602				size_t *start_offset);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 603
 604/*
 605 * IRQS.
 606 */
 607
 608struct amdgpu_flip_work {
 609	struct delayed_work		flip_work;
 610	struct work_struct		unpin_work;
 611	struct amdgpu_device		*adev;
 612	int				crtc_id;
 613	u32				target_vblank;
 614	uint64_t			base;
 615	struct drm_pending_vblank_event *event;
 616	struct amdgpu_bo		*old_abo;
 617	struct dma_fence		*excl;
 618	unsigned			shared_count;
 619	struct dma_fence		**shared;
 620	struct dma_fence_cb		cb;
 621	bool				async;
 622};
 623
 624
 625/*
 626 * CP & rings.
 627 */
 628
 629struct amdgpu_ib {
 630	struct amdgpu_sa_bo		*sa_bo;
 631	uint32_t			length_dw;
 632	uint64_t			gpu_addr;
 633	uint32_t			*ptr;
 634	uint32_t			flags;
 635};
 636
 637extern const struct amd_sched_backend_ops amdgpu_sched_ops;
 638
 639int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
 640		     struct amdgpu_job **job, struct amdgpu_vm *vm);
 641int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size,
 642			     struct amdgpu_job **job);
 643
 644void amdgpu_job_free_resources(struct amdgpu_job *job);
 645void amdgpu_job_free(struct amdgpu_job *job);
 646int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
 647		      struct amd_sched_entity *entity, void *owner,
 648		      struct dma_fence **f);
 649
 650/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 651 * context related structures
 652 */
 653
 654struct amdgpu_ctx_ring {
 655	uint64_t		sequence;
 656	struct dma_fence	**fences;
 657	struct amd_sched_entity	entity;
 658};
 659
 660struct amdgpu_ctx {
 661	struct kref		refcount;
 662	struct amdgpu_device    *adev;
 
 663	unsigned		reset_counter;
 
 
 664	spinlock_t		ring_lock;
 665	struct dma_fence	**fences;
 666	struct amdgpu_ctx_ring	rings[AMDGPU_MAX_RINGS];
 667	bool preamble_presented;
 
 
 
 
 668};
 669
 670struct amdgpu_ctx_mgr {
 671	struct amdgpu_device	*adev;
 672	struct mutex		lock;
 673	/* protected by lock */
 674	struct idr		ctx_handles;
 675};
 676
 677struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id);
 678int amdgpu_ctx_put(struct amdgpu_ctx *ctx);
 679
 680uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
 681			      struct dma_fence *fence);
 682struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
 683				   struct amdgpu_ring *ring, uint64_t seq);
 
 
 684
 685int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
 686		     struct drm_file *filp);
 687
 
 
 688void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
 689void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr);
 690
 
 691/*
 692 * file private structure
 693 */
 694
 695struct amdgpu_fpriv {
 696	struct amdgpu_vm	vm;
 
 
 697	struct mutex		bo_list_lock;
 698	struct idr		bo_list_handles;
 699	struct amdgpu_ctx_mgr	ctx_mgr;
 700};
 701
 702/*
 703 * residency list
 704 */
 
 
 
 
 
 
 
 
 705
 706struct amdgpu_bo_list {
 707	struct mutex lock;
 
 
 708	struct amdgpu_bo *gds_obj;
 709	struct amdgpu_bo *gws_obj;
 710	struct amdgpu_bo *oa_obj;
 711	unsigned first_userptr;
 712	unsigned num_entries;
 713	struct amdgpu_bo_list_entry *array;
 714};
 715
 716struct amdgpu_bo_list *
 717amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id);
 718void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list,
 719			     struct list_head *validated);
 720void amdgpu_bo_list_put(struct amdgpu_bo_list *list);
 721void amdgpu_bo_list_free(struct amdgpu_bo_list *list);
 722
 723/*
 724 * GFX stuff
 725 */
 726#include "clearstate_defs.h"
 727
 728struct amdgpu_rlc_funcs {
 729	void (*enter_safe_mode)(struct amdgpu_device *adev);
 730	void (*exit_safe_mode)(struct amdgpu_device *adev);
 731};
 732
 733struct amdgpu_rlc {
 734	/* for power gating */
 735	struct amdgpu_bo	*save_restore_obj;
 736	uint64_t		save_restore_gpu_addr;
 737	volatile uint32_t	*sr_ptr;
 738	const u32               *reg_list;
 739	u32                     reg_list_size;
 740	/* for clear state */
 741	struct amdgpu_bo	*clear_state_obj;
 742	uint64_t		clear_state_gpu_addr;
 743	volatile uint32_t	*cs_ptr;
 744	const struct cs_section_def   *cs_data;
 745	u32                     clear_state_size;
 746	/* for cp tables */
 747	struct amdgpu_bo	*cp_table_obj;
 748	uint64_t		cp_table_gpu_addr;
 749	volatile uint32_t	*cp_table_ptr;
 750	u32                     cp_table_size;
 751
 752	/* safe mode for updating CG/PG state */
 753	bool in_safe_mode;
 754	const struct amdgpu_rlc_funcs *funcs;
 755
 756	/* for firmware data */
 757	u32 save_and_restore_offset;
 758	u32 clear_state_descriptor_offset;
 759	u32 avail_scratch_ram_locations;
 760	u32 reg_restore_list_size;
 761	u32 reg_list_format_start;
 762	u32 reg_list_format_separate_start;
 763	u32 starting_offsets_start;
 764	u32 reg_list_format_size_bytes;
 765	u32 reg_list_size_bytes;
 766
 767	u32 *register_list_format;
 768	u32 *register_restore;
 769};
 770
 
 
 771struct amdgpu_mec {
 772	struct amdgpu_bo	*hpd_eop_obj;
 773	u64			hpd_eop_gpu_addr;
 774	u32 num_pipe;
 
 775	u32 num_mec;
 776	u32 num_queue;
 
 
 
 
 
 
 
 
 
 
 
 
 
 777};
 778
 779/*
 780 * GPU scratch registers structures, functions & helpers
 781 */
 782struct amdgpu_scratch {
 783	unsigned		num_reg;
 784	uint32_t                reg_base;
 785	bool			free[32];
 786	uint32_t		reg[32];
 787};
 788
 789/*
 790 * GFX configurations
 791 */
 792#define AMDGPU_GFX_MAX_SE 4
 793#define AMDGPU_GFX_MAX_SH_PER_SE 2
 794
 795struct amdgpu_rb_config {
 796	uint32_t rb_backend_disable;
 797	uint32_t user_rb_backend_disable;
 798	uint32_t raster_config;
 799	uint32_t raster_config_1;
 800};
 801
 802struct amdgpu_gca_config {
 
 
 
 
 
 
 
 
 
 803	unsigned max_shader_engines;
 804	unsigned max_tile_pipes;
 805	unsigned max_cu_per_sh;
 806	unsigned max_sh_per_se;
 807	unsigned max_backends_per_se;
 808	unsigned max_texture_channel_caches;
 809	unsigned max_gprs;
 810	unsigned max_gs_threads;
 811	unsigned max_hw_contexts;
 812	unsigned sc_prim_fifo_size_frontend;
 813	unsigned sc_prim_fifo_size_backend;
 814	unsigned sc_hiz_tile_fifo_size;
 815	unsigned sc_earlyz_tile_fifo_size;
 816
 817	unsigned num_tile_pipes;
 818	unsigned backend_enable_mask;
 819	unsigned mem_max_burst_length_bytes;
 820	unsigned mem_row_size_in_kb;
 821	unsigned shader_engine_tile_size;
 822	unsigned num_gpus;
 823	unsigned multi_gpu_tile_size;
 824	unsigned mc_arb_ramcfg;
 825	unsigned gb_addr_config;
 826	unsigned num_rbs;
 
 
 827
 828	uint32_t tile_mode_array[32];
 829	uint32_t macrotile_mode_array[16];
 830
 
 831	struct amdgpu_rb_config rb_config[AMDGPU_GFX_MAX_SE][AMDGPU_GFX_MAX_SH_PER_SE];
 
 
 
 832};
 833
 834struct amdgpu_cu_info {
 835	uint32_t number; /* total active CU number */
 
 
 
 
 
 
 
 836	uint32_t ao_cu_mask;
 
 837	uint32_t bitmap[4][4];
 838};
 839
 840struct amdgpu_gfx_funcs {
 841	/* get the gpu clock counter */
 842	uint64_t (*get_gpu_clock_counter)(struct amdgpu_device *adev);
 843	void (*select_se_sh)(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance);
 844	void (*read_wave_data)(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields);
 845	void (*read_wave_vgprs)(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t thread, uint32_t start, uint32_t size, uint32_t *dst);
 846	void (*read_wave_sgprs)(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t start, uint32_t size, uint32_t *dst);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 847};
 848
 849struct amdgpu_gfx {
 850	struct mutex			gpu_clock_mutex;
 851	struct amdgpu_gca_config	config;
 852	struct amdgpu_rlc		rlc;
 853	struct amdgpu_mec		mec;
 
 854	struct amdgpu_scratch		scratch;
 855	const struct firmware		*me_fw;	/* ME firmware */
 856	uint32_t			me_fw_version;
 857	const struct firmware		*pfp_fw; /* PFP firmware */
 858	uint32_t			pfp_fw_version;
 859	const struct firmware		*ce_fw;	/* CE firmware */
 860	uint32_t			ce_fw_version;
 861	const struct firmware		*rlc_fw; /* RLC firmware */
 862	uint32_t			rlc_fw_version;
 863	const struct firmware		*mec_fw; /* MEC firmware */
 864	uint32_t			mec_fw_version;
 865	const struct firmware		*mec2_fw; /* MEC2 firmware */
 866	uint32_t			mec2_fw_version;
 867	uint32_t			me_feature_version;
 868	uint32_t			ce_feature_version;
 869	uint32_t			pfp_feature_version;
 870	uint32_t			rlc_feature_version;
 871	uint32_t			mec_feature_version;
 872	uint32_t			mec2_feature_version;
 873	struct amdgpu_ring		gfx_ring[AMDGPU_MAX_GFX_RINGS];
 874	unsigned			num_gfx_rings;
 875	struct amdgpu_ring		compute_ring[AMDGPU_MAX_COMPUTE_RINGS];
 876	unsigned			num_compute_rings;
 877	struct amdgpu_irq_src		eop_irq;
 878	struct amdgpu_irq_src		priv_reg_irq;
 879	struct amdgpu_irq_src		priv_inst_irq;
 880	/* gfx status */
 881	uint32_t			gfx_current_status;
 882	/* ce ram size*/
 883	unsigned			ce_ram_size;
 884	struct amdgpu_cu_info		cu_info;
 885	const struct amdgpu_gfx_funcs	*funcs;
 886
 887	/* reset mask */
 888	uint32_t                        grbm_soft_reset;
 889	uint32_t                        srbm_soft_reset;
 
 
 
 
 
 
 
 
 890};
 891
 892int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 893		  unsigned size, struct amdgpu_ib *ib);
 894void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib,
 895		    struct dma_fence *f);
 896int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 897		       struct amdgpu_ib *ib, struct dma_fence *last_vm_update,
 898		       struct amdgpu_job *job, struct dma_fence **f);
 899int amdgpu_ib_pool_init(struct amdgpu_device *adev);
 900void amdgpu_ib_pool_fini(struct amdgpu_device *adev);
 901int amdgpu_ib_ring_tests(struct amdgpu_device *adev);
 902
 903/*
 904 * CS.
 905 */
 906struct amdgpu_cs_chunk {
 907	uint32_t		chunk_id;
 908	uint32_t		length_dw;
 909	void			*kdata;
 910};
 911
 912struct amdgpu_cs_parser {
 913	struct amdgpu_device	*adev;
 914	struct drm_file		*filp;
 915	struct amdgpu_ctx	*ctx;
 916
 917	/* chunks */
 918	unsigned		nchunks;
 919	struct amdgpu_cs_chunk	*chunks;
 920
 921	/* scheduler job object */
 922	struct amdgpu_job	*job;
 923
 924	/* buffer objects */
 925	struct ww_acquire_ctx		ticket;
 926	struct amdgpu_bo_list		*bo_list;
 
 927	struct amdgpu_bo_list_entry	vm_pd;
 928	struct list_head		validated;
 929	struct dma_fence		*fence;
 930	uint64_t			bytes_moved_threshold;
 
 931	uint64_t			bytes_moved;
 
 932	struct amdgpu_bo_list_entry	*evictable;
 933
 934	/* user fence */
 935	struct amdgpu_bo_list_entry	uf_entry;
 
 
 
 936};
 937
 938#define AMDGPU_PREAMBLE_IB_PRESENT          (1 << 0) /* bit set means command submit involves a preamble IB */
 939#define AMDGPU_PREAMBLE_IB_PRESENT_FIRST    (1 << 1) /* bit set means preamble IB is first presented in belonging context */
 940#define AMDGPU_HAVE_CTX_SWITCH              (1 << 2) /* bit set means context switch occured */
 941
 942struct amdgpu_job {
 943	struct amd_sched_job    base;
 944	struct amdgpu_device	*adev;
 945	struct amdgpu_vm	*vm;
 946	struct amdgpu_ring	*ring;
 947	struct amdgpu_sync	sync;
 
 948	struct amdgpu_ib	*ibs;
 949	struct dma_fence	*fence; /* the hw fence */
 950	uint32_t		preamble_status;
 951	uint32_t		num_ibs;
 952	void			*owner;
 953	uint64_t		fence_ctx; /* the fence_context this job uses */
 954	bool                    vm_needs_flush;
 955	unsigned		vm_id;
 956	uint64_t		vm_pd_addr;
 
 
 957	uint32_t		gds_base, gds_size;
 958	uint32_t		gws_base, gws_size;
 959	uint32_t		oa_base, oa_size;
 
 960
 961	/* user fence handling */
 962	uint64_t		uf_addr;
 963	uint64_t		uf_sequence;
 964
 965};
 966#define to_amdgpu_job(sched_job)		\
 967		container_of((sched_job), struct amdgpu_job, base)
 968
 969static inline u32 amdgpu_get_ib_value(struct amdgpu_cs_parser *p,
 970				      uint32_t ib_idx, int idx)
 971{
 972	return p->job->ibs[ib_idx].ptr[idx];
 973}
 974
 975static inline void amdgpu_set_ib_value(struct amdgpu_cs_parser *p,
 976				       uint32_t ib_idx, int idx,
 977				       uint32_t value)
 978{
 979	p->job->ibs[ib_idx].ptr[idx] = value;
 980}
 981
 982/*
 983 * Writeback
 984 */
 985#define AMDGPU_MAX_WB 1024	/* Reserve at most 1024 WB slots for amdgpu-owned rings. */
 986
 987struct amdgpu_wb {
 988	struct amdgpu_bo	*wb_obj;
 989	volatile uint32_t	*wb;
 990	uint64_t		gpu_addr;
 991	u32			num_wb;	/* Number of wb slots actually reserved for amdgpu. */
 992	unsigned long		used[DIV_ROUND_UP(AMDGPU_MAX_WB, BITS_PER_LONG)];
 993};
 994
 995int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb);
 996void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb);
 997
 998void amdgpu_get_pcie_info(struct amdgpu_device *adev);
 999
1000/*
1001 * UVD
1002 */
1003#define AMDGPU_DEFAULT_UVD_HANDLES	10
1004#define AMDGPU_MAX_UVD_HANDLES		40
1005#define AMDGPU_UVD_STACK_SIZE		(200*1024)
1006#define AMDGPU_UVD_HEAP_SIZE		(256*1024)
1007#define AMDGPU_UVD_SESSION_SIZE		(50*1024)
1008#define AMDGPU_UVD_FIRMWARE_OFFSET	256
1009
1010struct amdgpu_uvd {
1011	struct amdgpu_bo	*vcpu_bo;
1012	void			*cpu_addr;
1013	uint64_t		gpu_addr;
1014	unsigned		fw_version;
1015	void			*saved_bo;
1016	unsigned		max_handles;
1017	atomic_t		handles[AMDGPU_MAX_UVD_HANDLES];
1018	struct drm_file		*filp[AMDGPU_MAX_UVD_HANDLES];
1019	struct delayed_work	idle_work;
1020	const struct firmware	*fw;	/* UVD firmware */
1021	struct amdgpu_ring	ring;
1022	struct amdgpu_irq_src	irq;
1023	bool			address_64_bit;
1024	bool			use_ctx_buf;
1025	struct amd_sched_entity entity;
1026	uint32_t                srbm_soft_reset;
1027};
1028
1029/*
1030 * VCE
1031 */
1032#define AMDGPU_MAX_VCE_HANDLES	16
1033#define AMDGPU_VCE_FIRMWARE_OFFSET 256
1034
1035#define AMDGPU_VCE_HARVEST_VCE0 (1 << 0)
1036#define AMDGPU_VCE_HARVEST_VCE1 (1 << 1)
1037
1038struct amdgpu_vce {
1039	struct amdgpu_bo	*vcpu_bo;
1040	uint64_t		gpu_addr;
1041	unsigned		fw_version;
1042	unsigned		fb_version;
1043	atomic_t		handles[AMDGPU_MAX_VCE_HANDLES];
1044	struct drm_file		*filp[AMDGPU_MAX_VCE_HANDLES];
1045	uint32_t		img_size[AMDGPU_MAX_VCE_HANDLES];
1046	struct delayed_work	idle_work;
1047	struct mutex		idle_mutex;
1048	const struct firmware	*fw;	/* VCE firmware */
1049	struct amdgpu_ring	ring[AMDGPU_MAX_VCE_RINGS];
1050	struct amdgpu_irq_src	irq;
1051	unsigned		harvest_config;
1052	struct amd_sched_entity	entity;
1053	uint32_t                srbm_soft_reset;
1054	unsigned		num_rings;
1055};
1056
1057/*
1058 * SDMA
1059 */
1060struct amdgpu_sdma_instance {
1061	/* SDMA firmware */
1062	const struct firmware	*fw;
1063	uint32_t		fw_version;
1064	uint32_t		feature_version;
1065
1066	struct amdgpu_ring	ring;
1067	bool			burst_nop;
1068};
1069
1070struct amdgpu_sdma {
1071	struct amdgpu_sdma_instance instance[AMDGPU_MAX_SDMA_INSTANCES];
1072#ifdef CONFIG_DRM_AMDGPU_SI
1073	//SI DMA has a difference trap irq number for the second engine
1074	struct amdgpu_irq_src	trap_irq_1;
1075#endif
1076	struct amdgpu_irq_src	trap_irq;
1077	struct amdgpu_irq_src	illegal_inst_irq;
1078	int			num_instances;
1079	uint32_t                    srbm_soft_reset;
1080};
1081
1082/*
1083 * Firmware
1084 */
 
 
 
 
 
 
1085struct amdgpu_firmware {
1086	struct amdgpu_firmware_info ucode[AMDGPU_UCODE_ID_MAXIMUM];
1087	bool smu_load;
1088	struct amdgpu_bo *fw_buf;
1089	unsigned int fw_size;
 
 
 
 
 
 
 
 
 
 
 
1090};
1091
1092/*
1093 * Benchmarking
1094 */
1095void amdgpu_benchmark(struct amdgpu_device *adev, int test_number);
1096
1097
1098/*
1099 * Testing
1100 */
1101void amdgpu_test_moves(struct amdgpu_device *adev);
1102void amdgpu_test_ring_sync(struct amdgpu_device *adev,
1103			   struct amdgpu_ring *cpA,
1104			   struct amdgpu_ring *cpB);
1105void amdgpu_test_syncing(struct amdgpu_device *adev);
1106
1107/*
1108 * MMU Notifier
1109 */
1110#if defined(CONFIG_MMU_NOTIFIER)
1111int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
1112void amdgpu_mn_unregister(struct amdgpu_bo *bo);
1113#else
1114static inline int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
1115{
1116	return -ENODEV;
1117}
1118static inline void amdgpu_mn_unregister(struct amdgpu_bo *bo) {}
1119#endif
1120
1121/*
1122 * Debugfs
1123 */
1124struct amdgpu_debugfs {
1125	const struct drm_info_list	*files;
1126	unsigned		num_files;
1127};
1128
1129int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
1130			     const struct drm_info_list *files,
1131			     unsigned nfiles);
1132int amdgpu_debugfs_fence_init(struct amdgpu_device *adev);
1133
1134#if defined(CONFIG_DEBUG_FS)
1135int amdgpu_debugfs_init(struct drm_minor *minor);
1136void amdgpu_debugfs_cleanup(struct drm_minor *minor);
1137#endif
1138
1139int amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
1140
1141/*
1142 * amdgpu smumgr functions
1143 */
1144struct amdgpu_smumgr_funcs {
1145	int (*check_fw_load_finish)(struct amdgpu_device *adev, uint32_t fwtype);
1146	int (*request_smu_load_fw)(struct amdgpu_device *adev);
1147	int (*request_smu_specific_fw)(struct amdgpu_device *adev, uint32_t fwtype);
1148};
1149
1150/*
1151 * amdgpu smumgr
1152 */
1153struct amdgpu_smumgr {
1154	struct amdgpu_bo *toc_buf;
1155	struct amdgpu_bo *smu_buf;
1156	/* asic priv smu data */
1157	void *priv;
1158	spinlock_t smu_lock;
1159	/* smumgr functions */
1160	const struct amdgpu_smumgr_funcs *smumgr_funcs;
1161	/* ucode loading complete flag */
1162	uint32_t fw_flags;
1163};
1164
1165/*
1166 * ASIC specific register table accessible by UMD
1167 */
1168struct amdgpu_allowed_register_entry {
1169	uint32_t reg_offset;
1170	bool untouched;
1171	bool grbm_indexed;
1172};
1173
1174/*
1175 * ASIC specific functions.
1176 */
1177struct amdgpu_asic_funcs {
1178	bool (*read_disabled_bios)(struct amdgpu_device *adev);
1179	bool (*read_bios_from_rom)(struct amdgpu_device *adev,
1180				   u8 *bios, u32 length_bytes);
1181	void (*detect_hw_virtualization) (struct amdgpu_device *adev);
1182	int (*read_register)(struct amdgpu_device *adev, u32 se_num,
1183			     u32 sh_num, u32 reg_offset, u32 *value);
1184	void (*set_vga_state)(struct amdgpu_device *adev, bool state);
1185	int (*reset)(struct amdgpu_device *adev);
1186	/* get the reference clock */
1187	u32 (*get_xclk)(struct amdgpu_device *adev);
1188	/* MM block clocks */
1189	int (*set_uvd_clocks)(struct amdgpu_device *adev, u32 vclk, u32 dclk);
1190	int (*set_vce_clocks)(struct amdgpu_device *adev, u32 evclk, u32 ecclk);
1191	/* static power management */
1192	int (*get_pcie_lanes)(struct amdgpu_device *adev);
1193	void (*set_pcie_lanes)(struct amdgpu_device *adev, int lanes);
 
 
 
 
 
 
 
1194};
1195
1196/*
1197 * IOCTL.
1198 */
1199int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
1200			    struct drm_file *filp);
1201int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data,
1202				struct drm_file *filp);
1203
1204int amdgpu_gem_info_ioctl(struct drm_device *dev, void *data,
1205			  struct drm_file *filp);
1206int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
1207			struct drm_file *filp);
1208int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data,
1209			  struct drm_file *filp);
1210int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
1211			      struct drm_file *filp);
1212int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
1213			  struct drm_file *filp);
1214int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
1215			struct drm_file *filp);
1216int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
 
 
1217int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
1218int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
1219				struct drm_file *filp);
1220
1221int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
1222				struct drm_file *filp);
1223
1224/* VRAM scratch page for HDP bug, default vram page */
1225struct amdgpu_vram_scratch {
1226	struct amdgpu_bo		*robj;
1227	volatile uint32_t		*ptr;
1228	u64				gpu_addr;
1229};
1230
1231/*
1232 * ACPI
1233 */
1234struct amdgpu_atif_notification_cfg {
1235	bool enabled;
1236	int command_code;
1237};
1238
1239struct amdgpu_atif_notifications {
1240	bool display_switch;
1241	bool expansion_mode_change;
1242	bool thermal_state;
1243	bool forced_power_state;
1244	bool system_power_state;
1245	bool display_conf_change;
1246	bool px_gfx_switch;
1247	bool brightness_change;
1248	bool dgpu_display_event;
1249};
1250
1251struct amdgpu_atif_functions {
1252	bool system_params;
1253	bool sbios_requests;
1254	bool select_active_disp;
1255	bool lid_state;
1256	bool get_tv_standard;
1257	bool set_tv_standard;
1258	bool get_panel_expansion_mode;
1259	bool set_panel_expansion_mode;
1260	bool temperature_change;
1261	bool graphics_device_types;
1262};
1263
1264struct amdgpu_atif {
1265	struct amdgpu_atif_notifications notifications;
1266	struct amdgpu_atif_functions functions;
1267	struct amdgpu_atif_notification_cfg notification_cfg;
1268	struct amdgpu_encoder *encoder_for_bl;
1269};
1270
1271struct amdgpu_atcs_functions {
1272	bool get_ext_state;
1273	bool pcie_perf_req;
1274	bool pcie_dev_rdy;
1275	bool pcie_bus_width;
1276};
1277
1278struct amdgpu_atcs {
1279	struct amdgpu_atcs_functions functions;
1280};
1281
1282/*
 
 
 
 
 
 
 
 
 
 
1283 * CGS
1284 */
1285struct cgs_device *amdgpu_cgs_create_device(struct amdgpu_device *adev);
1286void amdgpu_cgs_destroy_device(struct cgs_device *cgs_device);
1287
1288/*
1289 * Core structure, functions and helpers.
1290 */
1291typedef uint32_t (*amdgpu_rreg_t)(struct amdgpu_device*, uint32_t);
1292typedef void (*amdgpu_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
1293
1294typedef uint32_t (*amdgpu_block_rreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
1295typedef void (*amdgpu_block_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t, uint32_t);
1296
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1297struct amdgpu_device {
1298	struct device			*dev;
1299	struct drm_device		*ddev;
1300	struct pci_dev			*pdev;
1301
1302#ifdef CONFIG_DRM_AMD_ACP
1303	struct amdgpu_acp		acp;
1304#endif
1305
1306	/* ASIC */
1307	enum amd_asic_type		asic_type;
1308	uint32_t			family;
1309	uint32_t			rev_id;
1310	uint32_t			external_rev_id;
1311	unsigned long			flags;
1312	int				usec_timeout;
1313	const struct amdgpu_asic_funcs	*asic_funcs;
1314	bool				shutdown;
1315	bool				need_dma32;
 
1316	bool				accel_working;
1317	struct work_struct		reset_work;
1318	struct notifier_block		acpi_nb;
1319	struct amdgpu_i2c_chan		*i2c_bus[AMDGPU_MAX_I2C_BUS];
1320	struct amdgpu_debugfs		debugfs[AMDGPU_DEBUGFS_MAX_COMPONENTS];
1321	unsigned			debugfs_count;
1322#if defined(CONFIG_DEBUG_FS)
1323	struct dentry			*debugfs_regs[AMDGPU_DEBUGFS_MAX_COMPONENTS];
1324#endif
1325	struct amdgpu_atif		atif;
1326	struct amdgpu_atcs		atcs;
1327	struct mutex			srbm_mutex;
1328	/* GRBM index mutex. Protects concurrent access to GRBM index */
1329	struct mutex                    grbm_idx_mutex;
1330	struct dev_pm_domain		vga_pm_domain;
1331	bool				have_disp_power_ref;
1332
1333	/* BIOS */
 
1334	uint8_t				*bios;
1335	uint32_t			bios_size;
1336	bool				is_atom_bios;
1337	struct amdgpu_bo		*stollen_vga_memory;
1338	uint32_t			bios_scratch[AMDGPU_BIOS_NUM_SCRATCH];
1339
1340	/* Register/doorbell mmio */
1341	resource_size_t			rmmio_base;
1342	resource_size_t			rmmio_size;
1343	void __iomem			*rmmio;
1344	/* protects concurrent MM_INDEX/DATA based register access */
1345	spinlock_t mmio_idx_lock;
1346	/* protects concurrent SMC based register access */
1347	spinlock_t smc_idx_lock;
1348	amdgpu_rreg_t			smc_rreg;
1349	amdgpu_wreg_t			smc_wreg;
1350	/* protects concurrent PCIE register access */
1351	spinlock_t pcie_idx_lock;
1352	amdgpu_rreg_t			pcie_rreg;
1353	amdgpu_wreg_t			pcie_wreg;
1354	amdgpu_rreg_t			pciep_rreg;
1355	amdgpu_wreg_t			pciep_wreg;
1356	/* protects concurrent UVD register access */
1357	spinlock_t uvd_ctx_idx_lock;
1358	amdgpu_rreg_t			uvd_ctx_rreg;
1359	amdgpu_wreg_t			uvd_ctx_wreg;
1360	/* protects concurrent DIDT register access */
1361	spinlock_t didt_idx_lock;
1362	amdgpu_rreg_t			didt_rreg;
1363	amdgpu_wreg_t			didt_wreg;
1364	/* protects concurrent gc_cac register access */
1365	spinlock_t gc_cac_idx_lock;
1366	amdgpu_rreg_t			gc_cac_rreg;
1367	amdgpu_wreg_t			gc_cac_wreg;
 
 
 
 
1368	/* protects concurrent ENDPOINT (audio) register access */
1369	spinlock_t audio_endpt_idx_lock;
1370	amdgpu_block_rreg_t		audio_endpt_rreg;
1371	amdgpu_block_wreg_t		audio_endpt_wreg;
1372	void __iomem                    *rio_mem;
1373	resource_size_t			rio_mem_size;
1374	struct amdgpu_doorbell		doorbell;
1375
1376	/* clock/pll info */
1377	struct amdgpu_clock            clock;
1378
1379	/* MC */
1380	struct amdgpu_mc		mc;
1381	struct amdgpu_gart		gart;
1382	struct amdgpu_dummy_page	dummy_page;
1383	struct amdgpu_vm_manager	vm_manager;
 
1384
1385	/* memory management */
1386	struct amdgpu_mman		mman;
1387	struct amdgpu_vram_scratch	vram_scratch;
1388	struct amdgpu_wb		wb;
1389	atomic64_t			vram_usage;
1390	atomic64_t			vram_vis_usage;
1391	atomic64_t			gtt_usage;
1392	atomic64_t			num_bytes_moved;
1393	atomic64_t			num_evictions;
 
1394	atomic_t			gpu_reset_counter;
 
1395
1396	/* data for buffer migration throttling */
1397	struct {
1398		spinlock_t		lock;
1399		s64			last_update_us;
1400		s64			accum_us; /* accumulated microseconds */
 
1401		u32			log2_max_MBps;
1402	} mm_stats;
1403
1404	/* display */
1405	bool				enable_virtual_display;
1406	struct amdgpu_mode_info		mode_info;
 
1407	struct work_struct		hotplug_work;
1408	struct amdgpu_irq_src		crtc_irq;
1409	struct amdgpu_irq_src		pageflip_irq;
1410	struct amdgpu_irq_src		hpd_irq;
1411
1412	/* rings */
1413	u64				fence_context;
1414	unsigned			num_rings;
1415	struct amdgpu_ring		*rings[AMDGPU_MAX_RINGS];
1416	bool				ib_pool_ready;
1417	struct amdgpu_sa_manager	ring_tmp_bo;
1418
1419	/* interrupts */
1420	struct amdgpu_irq		irq;
1421
1422	/* powerplay */
1423	struct amd_powerplay		powerplay;
1424	bool				pp_enabled;
1425	bool				pp_force_state_enabled;
1426
1427	/* dpm */
1428	struct amdgpu_pm		pm;
1429	u32				cg_flags;
1430	u32				pg_flags;
1431
1432	/* amdgpu smumgr */
1433	struct amdgpu_smumgr smu;
1434
1435	/* gfx */
1436	struct amdgpu_gfx		gfx;
1437
1438	/* sdma */
1439	struct amdgpu_sdma		sdma;
1440
1441	/* uvd */
1442	struct amdgpu_uvd		uvd;
1443
1444	/* vce */
1445	struct amdgpu_vce		vce;
1446
 
 
 
1447	/* firmwares */
1448	struct amdgpu_firmware		firmware;
1449
 
 
 
1450	/* GDS */
1451	struct amdgpu_gds		gds;
1452
 
 
 
1453	struct amdgpu_ip_block          ip_blocks[AMDGPU_MAX_IP_NUM];
1454	int				num_ip_blocks;
1455	struct mutex	mn_lock;
1456	DECLARE_HASHTABLE(mn_hash, 7);
1457
1458	/* tracking pinned memory */
1459	u64 vram_pin_size;
1460	u64 invisible_pin_size;
1461	u64 gart_pin_size;
1462
1463	/* amdkfd interface */
1464	struct kfd_dev          *kfd;
1465
1466	struct amdgpu_virtualization virtualization;
 
 
 
 
 
 
 
 
 
 
1467
1468	/* link all shadow bo */
1469	struct list_head                shadow_list;
1470	struct mutex                    shadow_list_lock;
1471	/* link all gtt */
1472	spinlock_t			gtt_list_lock;
1473	struct list_head                gtt_list;
1474
 
 
 
 
 
 
 
 
1475};
1476
1477static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
1478{
1479	return container_of(bdev, struct amdgpu_device, mman.bdev);
1480}
1481
1482bool amdgpu_device_is_px(struct drm_device *dev);
1483int amdgpu_device_init(struct amdgpu_device *adev,
1484		       struct drm_device *ddev,
1485		       struct pci_dev *pdev,
1486		       uint32_t flags);
1487void amdgpu_device_fini(struct amdgpu_device *adev);
1488int amdgpu_gpu_wait_for_idle(struct amdgpu_device *adev);
1489
1490uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
1491			bool always_indirect);
1492void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
1493		    bool always_indirect);
 
 
 
1494u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg);
1495void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v);
1496
1497u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index);
1498void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v);
 
 
 
 
 
 
 
1499
1500/*
1501 * Registers read & write functions.
1502 */
1503#define RREG32(reg) amdgpu_mm_rreg(adev, (reg), false)
1504#define RREG32_IDX(reg) amdgpu_mm_rreg(adev, (reg), true)
1505#define DREG32(reg) printk(KERN_INFO "REGISTER: " #reg " : 0x%08X\n", amdgpu_mm_rreg(adev, (reg), false))
1506#define WREG32(reg, v) amdgpu_mm_wreg(adev, (reg), (v), false)
1507#define WREG32_IDX(reg, v) amdgpu_mm_wreg(adev, (reg), (v), true)
 
 
 
 
 
 
 
 
 
 
1508#define REG_SET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
1509#define REG_GET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
1510#define RREG32_PCIE(reg) adev->pcie_rreg(adev, (reg))
1511#define WREG32_PCIE(reg, v) adev->pcie_wreg(adev, (reg), (v))
1512#define RREG32_PCIE_PORT(reg) adev->pciep_rreg(adev, (reg))
1513#define WREG32_PCIE_PORT(reg, v) adev->pciep_wreg(adev, (reg), (v))
1514#define RREG32_SMC(reg) adev->smc_rreg(adev, (reg))
1515#define WREG32_SMC(reg, v) adev->smc_wreg(adev, (reg), (v))
1516#define RREG32_UVD_CTX(reg) adev->uvd_ctx_rreg(adev, (reg))
1517#define WREG32_UVD_CTX(reg, v) adev->uvd_ctx_wreg(adev, (reg), (v))
1518#define RREG32_DIDT(reg) adev->didt_rreg(adev, (reg))
1519#define WREG32_DIDT(reg, v) adev->didt_wreg(adev, (reg), (v))
1520#define RREG32_GC_CAC(reg) adev->gc_cac_rreg(adev, (reg))
1521#define WREG32_GC_CAC(reg, v) adev->gc_cac_wreg(adev, (reg), (v))
 
 
1522#define RREG32_AUDIO_ENDPT(block, reg) adev->audio_endpt_rreg(adev, (block), (reg))
1523#define WREG32_AUDIO_ENDPT(block, reg, v) adev->audio_endpt_wreg(adev, (block), (reg), (v))
1524#define WREG32_P(reg, val, mask)				\
1525	do {							\
1526		uint32_t tmp_ = RREG32(reg);			\
1527		tmp_ &= (mask);					\
1528		tmp_ |= ((val) & ~(mask));			\
1529		WREG32(reg, tmp_);				\
1530	} while (0)
1531#define WREG32_AND(reg, and) WREG32_P(reg, 0, and)
1532#define WREG32_OR(reg, or) WREG32_P(reg, or, ~(or))
1533#define WREG32_PLL_P(reg, val, mask)				\
1534	do {							\
1535		uint32_t tmp_ = RREG32_PLL(reg);		\
1536		tmp_ &= (mask);					\
1537		tmp_ |= ((val) & ~(mask));			\
1538		WREG32_PLL(reg, tmp_);				\
1539	} while (0)
1540#define DREG32_SYS(sqf, adev, reg) seq_printf((sqf), #reg " : 0x%08X\n", amdgpu_mm_rreg((adev), (reg), false))
1541#define RREG32_IO(reg) amdgpu_io_rreg(adev, (reg))
1542#define WREG32_IO(reg, v) amdgpu_io_wreg(adev, (reg), (v))
1543
1544#define RDOORBELL32(index) amdgpu_mm_rdoorbell(adev, (index))
1545#define WDOORBELL32(index, v) amdgpu_mm_wdoorbell(adev, (index), (v))
 
 
1546
1547#define REG_FIELD_SHIFT(reg, field) reg##__##field##__SHIFT
1548#define REG_FIELD_MASK(reg, field) reg##__##field##_MASK
1549
1550#define REG_SET_FIELD(orig_val, reg, field, field_val)			\
1551	(((orig_val) & ~REG_FIELD_MASK(reg, field)) |			\
1552	 (REG_FIELD_MASK(reg, field) & ((field_val) << REG_FIELD_SHIFT(reg, field))))
1553
1554#define REG_GET_FIELD(value, reg, field)				\
1555	(((value) & REG_FIELD_MASK(reg, field)) >> REG_FIELD_SHIFT(reg, field))
1556
1557#define WREG32_FIELD(reg, field, val)	\
1558	WREG32(mm##reg, (RREG32(mm##reg) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
1559
 
 
 
1560/*
1561 * BIOS helpers.
1562 */
1563#define RBIOS8(i) (adev->bios[i])
1564#define RBIOS16(i) (RBIOS8(i) | (RBIOS8((i)+1) << 8))
1565#define RBIOS32(i) ((RBIOS16(i)) | (RBIOS16((i)+2) << 16))
1566
1567/*
1568 * RING helpers.
1569 */
1570static inline void amdgpu_ring_write(struct amdgpu_ring *ring, uint32_t v)
1571{
1572	if (ring->count_dw <= 0)
1573		DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n");
1574	ring->ring[ring->wptr++] = v;
1575	ring->wptr &= ring->ptr_mask;
1576	ring->count_dw--;
1577}
1578
1579static inline struct amdgpu_sdma_instance *
1580amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
1581{
1582	struct amdgpu_device *adev = ring->adev;
1583	int i;
1584
1585	for (i = 0; i < adev->sdma.num_instances; i++)
1586		if (&adev->sdma.instance[i].ring == ring)
1587			break;
1588
1589	if (i < AMDGPU_MAX_SDMA_INSTANCES)
1590		return &adev->sdma.instance[i];
1591	else
1592		return NULL;
1593}
1594
1595/*
1596 * ASICs macro.
1597 */
1598#define amdgpu_asic_set_vga_state(adev, state) (adev)->asic_funcs->set_vga_state((adev), (state))
1599#define amdgpu_asic_reset(adev) (adev)->asic_funcs->reset((adev))
1600#define amdgpu_asic_get_xclk(adev) (adev)->asic_funcs->get_xclk((adev))
1601#define amdgpu_asic_set_uvd_clocks(adev, v, d) (adev)->asic_funcs->set_uvd_clocks((adev), (v), (d))
1602#define amdgpu_asic_set_vce_clocks(adev, ev, ec) (adev)->asic_funcs->set_vce_clocks((adev), (ev), (ec))
1603#define amdgpu_get_pcie_lanes(adev) (adev)->asic_funcs->get_pcie_lanes((adev))
1604#define amdgpu_set_pcie_lanes(adev, l) (adev)->asic_funcs->set_pcie_lanes((adev), (l))
1605#define amdgpu_asic_get_gpu_clock_counter(adev) (adev)->asic_funcs->get_gpu_clock_counter((adev))
1606#define amdgpu_asic_read_disabled_bios(adev) (adev)->asic_funcs->read_disabled_bios((adev))
1607#define amdgpu_asic_read_bios_from_rom(adev, b, l) (adev)->asic_funcs->read_bios_from_rom((adev), (b), (l))
1608#define amdgpu_asic_detect_hw_virtualization(adev) (adev)->asic_funcs->detect_hw_virtualization((adev))
1609#define amdgpu_asic_read_register(adev, se, sh, offset, v)((adev)->asic_funcs->read_register((adev), (se), (sh), (offset), (v)))
1610#define amdgpu_gart_flush_gpu_tlb(adev, vmid) (adev)->gart.gart_funcs->flush_gpu_tlb((adev), (vmid))
1611#define amdgpu_gart_set_pte_pde(adev, pt, idx, addr, flags) (adev)->gart.gart_funcs->set_pte_pde((adev), (pt), (idx), (addr), (flags))
 
 
 
 
 
 
 
1612#define amdgpu_vm_copy_pte(adev, ib, pe, src, count) ((adev)->vm_manager.vm_pte_funcs->copy_pte((ib), (pe), (src), (count)))
1613#define amdgpu_vm_write_pte(adev, ib, pe, value, count, incr) ((adev)->vm_manager.vm_pte_funcs->write_pte((ib), (pe), (value), (count), (incr)))
1614#define amdgpu_vm_set_pte_pde(adev, ib, pe, addr, count, incr, flags) ((adev)->vm_manager.vm_pte_funcs->set_pte_pde((ib), (pe), (addr), (count), (incr), (flags)))
1615#define amdgpu_ring_parse_cs(r, p, ib) ((r)->funcs->parse_cs((p), (ib)))
1616#define amdgpu_ring_test_ring(r) (r)->funcs->test_ring((r))
1617#define amdgpu_ring_test_ib(r, t) (r)->funcs->test_ib((r), (t))
1618#define amdgpu_ring_get_rptr(r) (r)->funcs->get_rptr((r))
1619#define amdgpu_ring_get_wptr(r) (r)->funcs->get_wptr((r))
1620#define amdgpu_ring_set_wptr(r) (r)->funcs->set_wptr((r))
1621#define amdgpu_ring_emit_ib(r, ib, vm_id, c) (r)->funcs->emit_ib((r), (ib), (vm_id), (c))
1622#define amdgpu_ring_emit_pipeline_sync(r) (r)->funcs->emit_pipeline_sync((r))
1623#define amdgpu_ring_emit_vm_flush(r, vmid, addr) (r)->funcs->emit_vm_flush((r), (vmid), (addr))
1624#define amdgpu_ring_emit_fence(r, addr, seq, flags) (r)->funcs->emit_fence((r), (addr), (seq), (flags))
1625#define amdgpu_ring_emit_gds_switch(r, v, db, ds, wb, ws, ab, as) (r)->funcs->emit_gds_switch((r), (v), (db), (ds), (wb), (ws), (ab), (as))
1626#define amdgpu_ring_emit_hdp_flush(r) (r)->funcs->emit_hdp_flush((r))
1627#define amdgpu_ring_emit_hdp_invalidate(r) (r)->funcs->emit_hdp_invalidate((r))
1628#define amdgpu_ring_emit_switch_buffer(r) (r)->funcs->emit_switch_buffer((r))
1629#define amdgpu_ring_emit_cntxcntl(r, d) (r)->funcs->emit_cntxcntl((r), (d))
 
 
 
 
1630#define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib)))
1631#define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r))
1632#define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o))
1633#define amdgpu_ih_get_wptr(adev) (adev)->irq.ih_funcs->get_wptr((adev))
 
1634#define amdgpu_ih_decode_iv(adev, iv) (adev)->irq.ih_funcs->decode_iv((adev), (iv))
1635#define amdgpu_ih_set_rptr(adev) (adev)->irq.ih_funcs->set_rptr((adev))
1636#define amdgpu_display_set_vga_render_state(adev, r) (adev)->mode_info.funcs->set_vga_render_state((adev), (r))
1637#define amdgpu_display_vblank_get_counter(adev, crtc) (adev)->mode_info.funcs->vblank_get_counter((adev), (crtc))
1638#define amdgpu_display_vblank_wait(adev, crtc) (adev)->mode_info.funcs->vblank_wait((adev), (crtc))
1639#define amdgpu_display_backlight_set_level(adev, e, l) (adev)->mode_info.funcs->backlight_set_level((e), (l))
1640#define amdgpu_display_backlight_get_level(adev, e) (adev)->mode_info.funcs->backlight_get_level((e))
1641#define amdgpu_display_hpd_sense(adev, h) (adev)->mode_info.funcs->hpd_sense((adev), (h))
1642#define amdgpu_display_hpd_set_polarity(adev, h) (adev)->mode_info.funcs->hpd_set_polarity((adev), (h))
1643#define amdgpu_display_hpd_get_gpio_reg(adev) (adev)->mode_info.funcs->hpd_get_gpio_reg((adev))
1644#define amdgpu_display_bandwidth_update(adev) (adev)->mode_info.funcs->bandwidth_update((adev))
1645#define amdgpu_display_page_flip(adev, crtc, base, async) (adev)->mode_info.funcs->page_flip((adev), (crtc), (base), (async))
1646#define amdgpu_display_page_flip_get_scanoutpos(adev, crtc, vbl, pos) (adev)->mode_info.funcs->page_flip_get_scanoutpos((adev), (crtc), (vbl), (pos))
1647#define amdgpu_display_add_encoder(adev, e, s, c) (adev)->mode_info.funcs->add_encoder((adev), (e), (s), (c))
1648#define amdgpu_display_add_connector(adev, ci, sd, ct, ib, coi, h, r) (adev)->mode_info.funcs->add_connector((adev), (ci), (sd), (ct), (ib), (coi), (h), (r))
1649#define amdgpu_display_stop_mc_access(adev, s) (adev)->mode_info.funcs->stop_mc_access((adev), (s))
1650#define amdgpu_display_resume_mc_access(adev, s) (adev)->mode_info.funcs->resume_mc_access((adev), (s))
1651#define amdgpu_emit_copy_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_copy_buffer((ib),  (s), (d), (b))
1652#define amdgpu_emit_fill_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b))
1653#define amdgpu_gfx_get_gpu_clock_counter(adev) (adev)->gfx.funcs->get_gpu_clock_counter((adev))
1654#define amdgpu_gfx_select_se_sh(adev, se, sh, instance) (adev)->gfx.funcs->select_se_sh((adev), (se), (sh), (instance))
1655#define amdgpu_gds_switch(adev, r, v, d, w, a) (adev)->gds.funcs->patch_gds_switch((r), (v), (d), (w), (a))
 
 
1656
1657/* Common functions */
1658int amdgpu_gpu_reset(struct amdgpu_device *adev);
1659bool amdgpu_need_backup(struct amdgpu_device *adev);
1660void amdgpu_pci_config_reset(struct amdgpu_device *adev);
1661bool amdgpu_card_posted(struct amdgpu_device *adev);
1662void amdgpu_update_display_priority(struct amdgpu_device *adev);
1663
1664int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data);
1665int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type,
1666		       u32 ip_instance, u32 ring,
1667		       struct amdgpu_ring **out_ring);
1668void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain);
1669bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);
1670int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
1671int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
1672				     uint32_t flags);
1673bool amdgpu_ttm_tt_has_userptr(struct ttm_tt *ttm);
1674struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm);
1675bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
1676				  unsigned long end);
1677bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
1678				       int *last_invalidated);
1679bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);
1680uint32_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
1681				 struct ttm_mem_reg *mem);
1682void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base);
1683void amdgpu_gtt_location(struct amdgpu_device *adev, struct amdgpu_mc *mc);
1684void amdgpu_ttm_set_active_vram_size(struct amdgpu_device *adev, u64 size);
1685int amdgpu_ttm_init(struct amdgpu_device *adev);
1686void amdgpu_ttm_fini(struct amdgpu_device *adev);
1687void amdgpu_program_register_sequence(struct amdgpu_device *adev,
1688					     const u32 *registers,
1689					     const u32 array_size);
1690
1691bool amdgpu_device_is_px(struct drm_device *dev);
1692/* atpx handler */
1693#if defined(CONFIG_VGA_SWITCHEROO)
1694void amdgpu_register_atpx_handler(void);
1695void amdgpu_unregister_atpx_handler(void);
1696bool amdgpu_has_atpx_dgpu_power_cntl(void);
1697bool amdgpu_is_atpx_hybrid(void);
1698bool amdgpu_atpx_dgpu_req_power_for_displays(void);
 
1699#else
1700static inline void amdgpu_register_atpx_handler(void) {}
1701static inline void amdgpu_unregister_atpx_handler(void) {}
1702static inline bool amdgpu_has_atpx_dgpu_power_cntl(void) { return false; }
1703static inline bool amdgpu_is_atpx_hybrid(void) { return false; }
1704static inline bool amdgpu_atpx_dgpu_req_power_for_displays(void) { return false; }
 
1705#endif
1706
1707/*
1708 * KMS
1709 */
1710extern const struct drm_ioctl_desc amdgpu_ioctls_kms[];
1711extern const int amdgpu_max_kms_ioctl;
1712
1713int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags);
1714int amdgpu_driver_unload_kms(struct drm_device *dev);
1715void amdgpu_driver_lastclose_kms(struct drm_device *dev);
1716int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv);
1717void amdgpu_driver_postclose_kms(struct drm_device *dev,
1718				 struct drm_file *file_priv);
1719void amdgpu_driver_preclose_kms(struct drm_device *dev,
1720				struct drm_file *file_priv);
1721int amdgpu_suspend(struct amdgpu_device *adev);
1722int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon);
1723int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon);
1724u32 amdgpu_get_vblank_counter_kms(struct drm_device *dev, unsigned int pipe);
1725int amdgpu_enable_vblank_kms(struct drm_device *dev, unsigned int pipe);
1726void amdgpu_disable_vblank_kms(struct drm_device *dev, unsigned int pipe);
1727int amdgpu_get_vblank_timestamp_kms(struct drm_device *dev, unsigned int pipe,
1728				    int *max_error,
1729				    struct timeval *vblank_time,
1730				    unsigned flags);
1731long amdgpu_kms_compat_ioctl(struct file *filp, unsigned int cmd,
1732			     unsigned long arg);
1733
1734/*
1735 * functions used by amdgpu_encoder.c
1736 */
1737struct amdgpu_afmt_acr {
1738	u32 clock;
1739
1740	int n_32khz;
1741	int cts_32khz;
1742
1743	int n_44_1khz;
1744	int cts_44_1khz;
1745
1746	int n_48khz;
1747	int cts_48khz;
1748
1749};
1750
1751struct amdgpu_afmt_acr amdgpu_afmt_acr(uint32_t clock);
1752
1753/* amdgpu_acpi.c */
1754#if defined(CONFIG_ACPI)
1755int amdgpu_acpi_init(struct amdgpu_device *adev);
1756void amdgpu_acpi_fini(struct amdgpu_device *adev);
1757bool amdgpu_acpi_is_pcie_performance_request_supported(struct amdgpu_device *adev);
1758int amdgpu_acpi_pcie_performance_request(struct amdgpu_device *adev,
1759						u8 perf_req, bool advertise);
1760int amdgpu_acpi_pcie_notify_device_ready(struct amdgpu_device *adev);
1761#else
1762static inline int amdgpu_acpi_init(struct amdgpu_device *adev) { return 0; }
1763static inline void amdgpu_acpi_fini(struct amdgpu_device *adev) { }
1764#endif
1765
1766struct amdgpu_bo_va_mapping *
1767amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
1768		       uint64_t addr, struct amdgpu_bo **bo);
1769int amdgpu_cs_sysvm_access_required(struct amdgpu_cs_parser *parser);
 
 
 
 
 
1770
1771#include "amdgpu_object.h"
1772#endif