1/*
   2 * Copyright 2008 Advanced Micro Devices, Inc.
   3 * Copyright 2008 Red Hat Inc.
   4 * Copyright 2009 Jerome Glisse.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 *
  24 * Authors: Dave Airlie
  25 *          Alex Deucher
  26 *          Jerome Glisse
  27 */
  28#ifndef __AMDGPU_H__
  29#define __AMDGPU_H__
  30
  31#include <linux/atomic.h>
  32#include <linux/wait.h>
  33#include <linux/list.h>
  34#include <linux/kref.h>
  35#include <linux/rbtree.h>
  36#include <linux/hashtable.h>
  37#include <linux/dma-fence.h>
  38
  39#include <drm/ttm/ttm_bo_api.h>
  40#include <drm/ttm/ttm_bo_driver.h>
  41#include <drm/ttm/ttm_placement.h>
  42#include <drm/ttm/ttm_module.h>
  43#include <drm/ttm/ttm_execbuf_util.h>
  44
  45#include <drm/drmP.h>
  46#include <drm/drm_gem.h>
  47#include <drm/amdgpu_drm.h>
  48#include <drm/gpu_scheduler.h>
  49
  50#include <kgd_kfd_interface.h>
  51#include "dm_pp_interface.h"
  52#include "kgd_pp_interface.h"
  53
  54#include "amd_shared.h"
  55#include "amdgpu_mode.h"
  56#include "amdgpu_ih.h"
  57#include "amdgpu_irq.h"
  58#include "amdgpu_ucode.h"
  59#include "amdgpu_ttm.h"
  60#include "amdgpu_psp.h"
  61#include "amdgpu_gds.h"
  62#include "amdgpu_sync.h"
  63#include "amdgpu_ring.h"
  64#include "amdgpu_vm.h"
  65#include "amdgpu_dpm.h"
  66#include "amdgpu_acp.h"
  67#include "amdgpu_uvd.h"
  68#include "amdgpu_vce.h"
  69#include "amdgpu_vcn.h"
  70#include "amdgpu_mn.h"
  71#include "amdgpu_gmc.h"
  72#include "amdgpu_dm.h"
  73#include "amdgpu_virt.h"
  74#include "amdgpu_gart.h"
  75#include "amdgpu_debugfs.h"
  76
  77/*
  78 * Modules parameters.
  79 */
  80extern int amdgpu_modeset;
  81extern int amdgpu_vram_limit;
  82extern int amdgpu_vis_vram_limit;
  83extern int amdgpu_gart_size;
  84extern int amdgpu_gtt_size;
  85extern int amdgpu_moverate;
  86extern int amdgpu_benchmarking;
  87extern int amdgpu_testing;
  88extern int amdgpu_audio;
  89extern int amdgpu_disp_priority;
  90extern int amdgpu_hw_i2c;
  91extern int amdgpu_pcie_gen2;
  92extern int amdgpu_msi;
  93extern int amdgpu_lockup_timeout;
  94extern int amdgpu_dpm;
  95extern int amdgpu_fw_load_type;
  96extern int amdgpu_aspm;
  97extern int amdgpu_runtime_pm;
  98extern uint amdgpu_ip_block_mask;
  99extern int amdgpu_bapm;
 100extern int amdgpu_deep_color;
 101extern int amdgpu_vm_size;
 102extern int amdgpu_vm_block_size;
 103extern int amdgpu_vm_fragment_size;
 104extern int amdgpu_vm_fault_stop;
 105extern int amdgpu_vm_debug;
 106extern int amdgpu_vm_update_mode;
 107extern int amdgpu_dc;
 108extern int amdgpu_dc_log;
 109extern int amdgpu_sched_jobs;
 110extern int amdgpu_sched_hw_submission;
 111extern int amdgpu_no_evict;
 112extern int amdgpu_direct_gma_size;
 113extern uint amdgpu_pcie_gen_cap;
 114extern uint amdgpu_pcie_lane_cap;
 115extern uint amdgpu_cg_mask;
 116extern uint amdgpu_pg_mask;
 117extern uint amdgpu_sdma_phase_quantum;
 118extern char *amdgpu_disable_cu;
 119extern char *amdgpu_virtual_display;
 120extern uint amdgpu_pp_feature_mask;
 121extern int amdgpu_vram_page_split;
 122extern int amdgpu_ngg;
 123extern int amdgpu_prim_buf_per_se;
 124extern int amdgpu_pos_buf_per_se;
 125extern int amdgpu_cntl_sb_buf_per_se;
 126extern int amdgpu_param_buf_per_se;
 127extern int amdgpu_job_hang_limit;
 128extern int amdgpu_lbpw;
 129extern int amdgpu_compute_multipipe;
 130extern int amdgpu_gpu_recovery;
 131extern int amdgpu_emu_mode;
 132
 133#ifdef CONFIG_DRM_AMDGPU_SI
 134extern int amdgpu_si_support;
 135#endif
 136#ifdef CONFIG_DRM_AMDGPU_CIK
 137extern int amdgpu_cik_support;
 138#endif
 139
 140#define AMDGPU_DEFAULT_GTT_SIZE_MB		3072ULL /* 3GB by default */
 141#define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS	        3000
 142#define AMDGPU_MAX_USEC_TIMEOUT			100000	/* 100 ms */
 143#define AMDGPU_FENCE_JIFFIES_TIMEOUT		(HZ / 2)
 144/* AMDGPU_IB_POOL_SIZE must be a power of 2 */
 145#define AMDGPU_IB_POOL_SIZE			16
 146#define AMDGPU_DEBUGFS_MAX_COMPONENTS		32
 147#define AMDGPUFB_CONN_LIMIT			4
 148#define AMDGPU_BIOS_NUM_SCRATCH			16
 
 
 
 
 
 
 149
 150/* max number of IP instances */
 151#define AMDGPU_MAX_SDMA_INSTANCES		2
 152
 
 
 
 153/* hard reset data */
 154#define AMDGPU_ASIC_RESET_DATA                  0x39d5e86b
 155
 156/* reset flags */
 157#define AMDGPU_RESET_GFX			(1 << 0)
 158#define AMDGPU_RESET_COMPUTE			(1 << 1)
 159#define AMDGPU_RESET_DMA			(1 << 2)
 160#define AMDGPU_RESET_CP				(1 << 3)
 161#define AMDGPU_RESET_GRBM			(1 << 4)
 162#define AMDGPU_RESET_DMA1			(1 << 5)
 163#define AMDGPU_RESET_RLC			(1 << 6)
 164#define AMDGPU_RESET_SEM			(1 << 7)
 165#define AMDGPU_RESET_IH				(1 << 8)
 166#define AMDGPU_RESET_VMC			(1 << 9)
 167#define AMDGPU_RESET_MC				(1 << 10)
 168#define AMDGPU_RESET_DISPLAY			(1 << 11)
 169#define AMDGPU_RESET_UVD			(1 << 12)
 170#define AMDGPU_RESET_VCE			(1 << 13)
 171#define AMDGPU_RESET_VCE1			(1 << 14)
 172
 173/* GFX current status */
 174#define AMDGPU_GFX_NORMAL_MODE			0x00000000L
 175#define AMDGPU_GFX_SAFE_MODE			0x00000001L
 176#define AMDGPU_GFX_PG_DISABLED_MODE		0x00000002L
 177#define AMDGPU_GFX_CG_DISABLED_MODE		0x00000004L
 178#define AMDGPU_GFX_LBPW_DISABLED_MODE		0x00000008L
 179
 180/* max cursor sizes (in pixels) */
 181#define CIK_CURSOR_WIDTH 128
 182#define CIK_CURSOR_HEIGHT 128
 183
 184struct amdgpu_device;
 185struct amdgpu_ib;
 
 
 186struct amdgpu_cs_parser;
 187struct amdgpu_job;
 188struct amdgpu_irq_src;
 189struct amdgpu_fpriv;
 190struct amdgpu_bo_va_mapping;
 191
 192enum amdgpu_cp_irq {
 193	AMDGPU_CP_IRQ_GFX_EOP = 0,
 194	AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP,
 195	AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP,
 196	AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP,
 197	AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP,
 198	AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP,
 199	AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP,
 200	AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP,
 201	AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP,
 202
 203	AMDGPU_CP_IRQ_LAST
 204};
 205
 206enum amdgpu_sdma_irq {
 207	AMDGPU_SDMA_IRQ_TRAP0 = 0,
 208	AMDGPU_SDMA_IRQ_TRAP1,
 209
 210	AMDGPU_SDMA_IRQ_LAST
 211};
 212
 213enum amdgpu_thermal_irq {
 214	AMDGPU_THERMAL_IRQ_LOW_TO_HIGH = 0,
 215	AMDGPU_THERMAL_IRQ_HIGH_TO_LOW,
 216
 217	AMDGPU_THERMAL_IRQ_LAST
 218};
 219
 220enum amdgpu_kiq_irq {
 221	AMDGPU_CP_KIQ_IRQ_DRIVER0 = 0,
 222	AMDGPU_CP_KIQ_IRQ_LAST
 223};
 224
 225int amdgpu_device_ip_set_clockgating_state(struct amdgpu_device *adev,
 226					   enum amd_ip_block_type block_type,
 227					   enum amd_clockgating_state state);
 228int amdgpu_device_ip_set_powergating_state(struct amdgpu_device *adev,
 229					   enum amd_ip_block_type block_type,
 230					   enum amd_powergating_state state);
 231void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
 232					    u32 *flags);
 233int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
 234				   enum amd_ip_block_type block_type);
 235bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
 236			      enum amd_ip_block_type block_type);
 237
 238#define AMDGPU_MAX_IP_NUM 16
 239
 240struct amdgpu_ip_block_status {
 241	bool valid;
 242	bool sw;
 243	bool hw;
 244	bool late_initialized;
 245	bool hang;
 246};
 247
 248struct amdgpu_ip_block_version {
 249	const enum amd_ip_block_type type;
 250	const u32 major;
 251	const u32 minor;
 252	const u32 rev;
 253	const struct amd_ip_funcs *funcs;
 254};
 255
 256struct amdgpu_ip_block {
 257	struct amdgpu_ip_block_status status;
 258	const struct amdgpu_ip_block_version *version;
 259};
 260
 261int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
 262				       enum amd_ip_block_type type,
 263				       u32 major, u32 minor);
 264
 265struct amdgpu_ip_block *
 266amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
 267			      enum amd_ip_block_type type);
 268
 269int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
 270			       const struct amdgpu_ip_block_version *ip_block_version);
 271
 272/* provided by hw blocks that can move/clear data.  e.g., gfx or sdma */
 273struct amdgpu_buffer_funcs {
 274	/* maximum bytes in a single operation */
 275	uint32_t	copy_max_bytes;
 276
 277	/* number of dw to reserve per operation */
 278	unsigned	copy_num_dw;
 279
 280	/* used for buffer migration */
 281	void (*emit_copy_buffer)(struct amdgpu_ib *ib,
 282				 /* src addr in bytes */
 283				 uint64_t src_offset,
 284				 /* dst addr in bytes */
 285				 uint64_t dst_offset,
 286				 /* number of byte to transfer */
 287				 uint32_t byte_count);
 288
 289	/* maximum bytes in a single operation */
 290	uint32_t	fill_max_bytes;
 291
 292	/* number of dw to reserve per operation */
 293	unsigned	fill_num_dw;
 294
 295	/* used for buffer clearing */
 296	void (*emit_fill_buffer)(struct amdgpu_ib *ib,
 297				 /* value to write to memory */
 298				 uint32_t src_data,
 299				 /* dst addr in bytes */
 300				 uint64_t dst_offset,
 301				 /* number of byte to fill */
 302				 uint32_t byte_count);
 303};
 304
 305/* provided by hw blocks that can write ptes, e.g., sdma */
 306struct amdgpu_vm_pte_funcs {
 307	/* number of dw to reserve per operation */
 308	unsigned	copy_pte_num_dw;
 309
 310	/* copy pte entries from GART */
 311	void (*copy_pte)(struct amdgpu_ib *ib,
 312			 uint64_t pe, uint64_t src,
 313			 unsigned count);
 314
 315	/* write pte one entry at a time with addr mapping */
 316	void (*write_pte)(struct amdgpu_ib *ib, uint64_t pe,
 317			  uint64_t value, unsigned count,
 318			  uint32_t incr);
 
 319	/* for linear pte/pde updates without addr mapping */
 320	void (*set_pte_pde)(struct amdgpu_ib *ib,
 321			    uint64_t pe,
 322			    uint64_t addr, unsigned count,
 323			    uint32_t incr, uint64_t flags);
 
 
 
 
 
 
 
 
 
 
 
 
 
 324};
 325
 326/* provided by the ih block */
 327struct amdgpu_ih_funcs {
 328	/* ring read/write ptr handling, called from interrupt context */
 329	u32 (*get_wptr)(struct amdgpu_device *adev);
 330	bool (*prescreen_iv)(struct amdgpu_device *adev);
 331	void (*decode_iv)(struct amdgpu_device *adev,
 332			  struct amdgpu_iv_entry *entry);
 333	void (*set_rptr)(struct amdgpu_device *adev);
 334};
 335
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 336/*
 337 * BIOS.
 338 */
 339bool amdgpu_get_bios(struct amdgpu_device *adev);
 340bool amdgpu_read_bios(struct amdgpu_device *adev);
 341
 342/*
 
 
 
 
 
 
 
 
 
 
 
 343 * Clocks
 344 */
 345
 346#define AMDGPU_MAX_PPLL 3
 347
 348struct amdgpu_clock {
 349	struct amdgpu_pll ppll[AMDGPU_MAX_PPLL];
 350	struct amdgpu_pll spll;
 351	struct amdgpu_pll mpll;
 352	/* 10 Khz units */
 353	uint32_t default_mclk;
 354	uint32_t default_sclk;
 355	uint32_t default_dispclk;
 356	uint32_t current_dispclk;
 357	uint32_t dp_extclk;
 358	uint32_t max_pixel_clock;
 359};
 360
 361/*
 362 * GEM.
 363 */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 364
 365#define AMDGPU_GEM_DOMAIN_MAX		0x3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 366#define gem_to_amdgpu_bo(gobj) container_of((gobj), struct amdgpu_bo, gem_base)
 367
 368void amdgpu_gem_object_free(struct drm_gem_object *obj);
 369int amdgpu_gem_object_open(struct drm_gem_object *obj,
 370				struct drm_file *file_priv);
 371void amdgpu_gem_object_close(struct drm_gem_object *obj,
 372				struct drm_file *file_priv);
 373unsigned long amdgpu_gem_timeout(uint64_t timeout_ns);
 374struct sg_table *amdgpu_gem_prime_get_sg_table(struct drm_gem_object *obj);
 375struct drm_gem_object *
 376amdgpu_gem_prime_import_sg_table(struct drm_device *dev,
 377				 struct dma_buf_attachment *attach,
 378				 struct sg_table *sg);
 379struct dma_buf *amdgpu_gem_prime_export(struct drm_device *dev,
 380					struct drm_gem_object *gobj,
 381					int flags);
 382struct drm_gem_object *amdgpu_gem_prime_import(struct drm_device *dev,
 383					    struct dma_buf *dma_buf);
 384struct reservation_object *amdgpu_gem_prime_res_obj(struct drm_gem_object *);
 385void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj);
 386void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
 387int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
 388
 389/* sub-allocation manager, it has to be protected by another lock.
 390 * By conception this is an helper for other part of the driver
 391 * like the indirect buffer or semaphore, which both have their
 392 * locking.
 393 *
 394 * Principe is simple, we keep a list of sub allocation in offset
 395 * order (first entry has offset == 0, last entry has the highest
 396 * offset).
 397 *
 398 * When allocating new object we first check if there is room at
 399 * the end total_size - (last_object_offset + last_object_size) >=
 400 * alloc_size. If so we allocate new object there.
 401 *
 402 * When there is not enough room at the end, we start waiting for
 403 * each sub object until we reach object_offset+object_size >=
 404 * alloc_size, this object then become the sub object we return.
 405 *
 406 * Alignment can't be bigger than page size.
 407 *
 408 * Hole are not considered for allocation to keep things simple.
 409 * Assumption is that there won't be hole (all object on same
 410 * alignment).
 411 */
 412
 413#define AMDGPU_SA_NUM_FENCE_LISTS	32
 414
 415struct amdgpu_sa_manager {
 416	wait_queue_head_t	wq;
 417	struct amdgpu_bo	*bo;
 418	struct list_head	*hole;
 419	struct list_head	flist[AMDGPU_SA_NUM_FENCE_LISTS];
 420	struct list_head	olist;
 421	unsigned		size;
 422	uint64_t		gpu_addr;
 423	void			*cpu_ptr;
 424	uint32_t		domain;
 425	uint32_t		align;
 426};
 427
 428/* sub-allocation buffer */
 429struct amdgpu_sa_bo {
 430	struct list_head		olist;
 431	struct list_head		flist;
 432	struct amdgpu_sa_manager	*manager;
 433	unsigned			soffset;
 434	unsigned			eoffset;
 435	struct dma_fence	        *fence;
 436};
 437
 438/*
 439 * GEM objects.
 440 */
 441void amdgpu_gem_force_release(struct amdgpu_device *adev);
 442int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 443			     int alignment, u32 initial_domain,
 444			     u64 flags, enum ttm_bo_type type,
 445			     struct reservation_object *resv,
 446			     struct drm_gem_object **obj);
 447
 448int amdgpu_mode_dumb_create(struct drm_file *file_priv,
 449			    struct drm_device *dev,
 450			    struct drm_mode_create_dumb *args);
 451int amdgpu_mode_dumb_mmap(struct drm_file *filp,
 452			  struct drm_device *dev,
 453			  uint32_t handle, uint64_t *offset_p);
 454int amdgpu_fence_slab_init(void);
 455void amdgpu_fence_slab_fini(void);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 456
 457/*
 458 * GPU doorbell structures, functions & helpers
 459 */
 460typedef enum _AMDGPU_DOORBELL_ASSIGNMENT
 461{
 462	AMDGPU_DOORBELL_KIQ                     = 0x000,
 463	AMDGPU_DOORBELL_HIQ                     = 0x001,
 464	AMDGPU_DOORBELL_DIQ                     = 0x002,
 465	AMDGPU_DOORBELL_MEC_RING0               = 0x010,
 466	AMDGPU_DOORBELL_MEC_RING1               = 0x011,
 467	AMDGPU_DOORBELL_MEC_RING2               = 0x012,
 468	AMDGPU_DOORBELL_MEC_RING3               = 0x013,
 469	AMDGPU_DOORBELL_MEC_RING4               = 0x014,
 470	AMDGPU_DOORBELL_MEC_RING5               = 0x015,
 471	AMDGPU_DOORBELL_MEC_RING6               = 0x016,
 472	AMDGPU_DOORBELL_MEC_RING7               = 0x017,
 473	AMDGPU_DOORBELL_GFX_RING0               = 0x020,
 474	AMDGPU_DOORBELL_sDMA_ENGINE0            = 0x1E0,
 475	AMDGPU_DOORBELL_sDMA_ENGINE1            = 0x1E1,
 476	AMDGPU_DOORBELL_IH                      = 0x1E8,
 477	AMDGPU_DOORBELL_MAX_ASSIGNMENT          = 0x3FF,
 478	AMDGPU_DOORBELL_INVALID                 = 0xFFFF
 479} AMDGPU_DOORBELL_ASSIGNMENT;
 480
 481struct amdgpu_doorbell {
 482	/* doorbell mmio */
 483	resource_size_t		base;
 484	resource_size_t		size;
 485	u32 __iomem		*ptr;
 486	u32			num_doorbells;	/* Number of doorbells actually reserved for amdgpu. */
 487};
 488
 489/*
 490 * 64bit doorbell, offset are in QWORD, occupy 2KB doorbell space
 491 */
 492typedef enum _AMDGPU_DOORBELL64_ASSIGNMENT
 493{
 494	/*
 495	 * All compute related doorbells: kiq, hiq, diq, traditional compute queue, user queue, should locate in
 496	 * a continues range so that programming CP_MEC_DOORBELL_RANGE_LOWER/UPPER can cover this range.
 497	 *  Compute related doorbells are allocated from 0x00 to 0x8a
 498	 */
 499
 500
 501	/* kernel scheduling */
 502	AMDGPU_DOORBELL64_KIQ                     = 0x00,
 503
 504	/* HSA interface queue and debug queue */
 505	AMDGPU_DOORBELL64_HIQ                     = 0x01,
 506	AMDGPU_DOORBELL64_DIQ                     = 0x02,
 507
 508	/* Compute engines */
 509	AMDGPU_DOORBELL64_MEC_RING0               = 0x03,
 510	AMDGPU_DOORBELL64_MEC_RING1               = 0x04,
 511	AMDGPU_DOORBELL64_MEC_RING2               = 0x05,
 512	AMDGPU_DOORBELL64_MEC_RING3               = 0x06,
 513	AMDGPU_DOORBELL64_MEC_RING4               = 0x07,
 514	AMDGPU_DOORBELL64_MEC_RING5               = 0x08,
 515	AMDGPU_DOORBELL64_MEC_RING6               = 0x09,
 516	AMDGPU_DOORBELL64_MEC_RING7               = 0x0a,
 517
 518	/* User queue doorbell range (128 doorbells) */
 519	AMDGPU_DOORBELL64_USERQUEUE_START         = 0x0b,
 520	AMDGPU_DOORBELL64_USERQUEUE_END           = 0x8a,
 521
 522	/* Graphics engine */
 523	AMDGPU_DOORBELL64_GFX_RING0               = 0x8b,
 524
 525	/*
 526	 * Other graphics doorbells can be allocated here: from 0x8c to 0xef
 527	 * Graphics voltage island aperture 1
 528	 * default non-graphics QWORD index is 0xF0 - 0xFF inclusive
 529	 */
 530
 531	/* sDMA engines */
 532	AMDGPU_DOORBELL64_sDMA_ENGINE0            = 0xF0,
 533	AMDGPU_DOORBELL64_sDMA_HI_PRI_ENGINE0     = 0xF1,
 534	AMDGPU_DOORBELL64_sDMA_ENGINE1            = 0xF2,
 535	AMDGPU_DOORBELL64_sDMA_HI_PRI_ENGINE1     = 0xF3,
 536
 537	/* Interrupt handler */
 538	AMDGPU_DOORBELL64_IH                      = 0xF4,  /* For legacy interrupt ring buffer */
 539	AMDGPU_DOORBELL64_IH_RING1                = 0xF5,  /* For page migration request log */
 540	AMDGPU_DOORBELL64_IH_RING2                = 0xF6,  /* For page migration translation/invalidation log */
 541
 542	/* VCN engine use 32 bits doorbell  */
 543	AMDGPU_DOORBELL64_VCN0_1                  = 0xF8, /* lower 32 bits for VNC0 and upper 32 bits for VNC1 */
 544	AMDGPU_DOORBELL64_VCN2_3                  = 0xF9,
 545	AMDGPU_DOORBELL64_VCN4_5                  = 0xFA,
 546	AMDGPU_DOORBELL64_VCN6_7                  = 0xFB,
 547
 548	/* overlap the doorbell assignment with VCN as they are  mutually exclusive
 549	 * VCE engine's doorbell is 32 bit and two VCE ring share one QWORD
 550	 */
 551	AMDGPU_DOORBELL64_UVD_RING0_1             = 0xF8,
 552	AMDGPU_DOORBELL64_UVD_RING2_3             = 0xF9,
 553	AMDGPU_DOORBELL64_UVD_RING4_5             = 0xFA,
 554	AMDGPU_DOORBELL64_UVD_RING6_7             = 0xFB,
 555
 556	AMDGPU_DOORBELL64_VCE_RING0_1             = 0xFC,
 557	AMDGPU_DOORBELL64_VCE_RING2_3             = 0xFD,
 558	AMDGPU_DOORBELL64_VCE_RING4_5             = 0xFE,
 559	AMDGPU_DOORBELL64_VCE_RING6_7             = 0xFF,
 560
 561	AMDGPU_DOORBELL64_MAX_ASSIGNMENT          = 0xFF,
 562	AMDGPU_DOORBELL64_INVALID                 = 0xFFFF
 563} AMDGPU_DOORBELL64_ASSIGNMENT;
 564
 565/*
 566 * IRQS.
 567 */
 568
 569struct amdgpu_flip_work {
 570	struct delayed_work		flip_work;
 571	struct work_struct		unpin_work;
 572	struct amdgpu_device		*adev;
 573	int				crtc_id;
 574	u32				target_vblank;
 575	uint64_t			base;
 576	struct drm_pending_vblank_event *event;
 577	struct amdgpu_bo		*old_abo;
 578	struct dma_fence		*excl;
 579	unsigned			shared_count;
 580	struct dma_fence		**shared;
 581	struct dma_fence_cb		cb;
 582	bool				async;
 583};
 584
 585
 586/*
 587 * CP & rings.
 588 */
 589
 590struct amdgpu_ib {
 591	struct amdgpu_sa_bo		*sa_bo;
 592	uint32_t			length_dw;
 593	uint64_t			gpu_addr;
 594	uint32_t			*ptr;
 
 
 
 
 
 
 
 
 595	uint32_t			flags;
 
 
 596};
 597
 598extern const struct drm_sched_backend_ops amdgpu_sched_ops;
 
 
 
 
 
 
 
 
 599
 600int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
 601		     struct amdgpu_job **job, struct amdgpu_vm *vm);
 602int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size,
 603			     struct amdgpu_job **job);
 604
 605void amdgpu_job_free_resources(struct amdgpu_job *job);
 606void amdgpu_job_free(struct amdgpu_job *job);
 607int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
 608		      struct drm_sched_entity *entity, void *owner,
 609		      struct dma_fence **f);
 610
 611/*
 612 * Queue manager
 613 */
 614struct amdgpu_queue_mapper {
 615	int 		hw_ip;
 616	struct mutex	lock;
 617	/* protected by lock */
 618	struct amdgpu_ring *queue_map[AMDGPU_MAX_RINGS];
 619};
 620
 621struct amdgpu_queue_mgr {
 622	struct amdgpu_queue_mapper mapper[AMDGPU_MAX_IP_NUM];
 623};
 624
 625int amdgpu_queue_mgr_init(struct amdgpu_device *adev,
 626			  struct amdgpu_queue_mgr *mgr);
 627int amdgpu_queue_mgr_fini(struct amdgpu_device *adev,
 628			  struct amdgpu_queue_mgr *mgr);
 629int amdgpu_queue_mgr_map(struct amdgpu_device *adev,
 630			 struct amdgpu_queue_mgr *mgr,
 631			 u32 hw_ip, u32 instance, u32 ring,
 632			 struct amdgpu_ring **out_ring);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 633
 634/*
 635 * context related structures
 636 */
 637
 638struct amdgpu_ctx_ring {
 639	uint64_t		sequence;
 640	struct dma_fence	**fences;
 641	struct drm_sched_entity	entity;
 642};
 643
 644struct amdgpu_ctx {
 645	struct kref		refcount;
 646	struct amdgpu_device    *adev;
 647	struct amdgpu_queue_mgr queue_mgr;
 648	unsigned		reset_counter;
 649	unsigned        reset_counter_query;
 650	uint32_t		vram_lost_counter;
 651	spinlock_t		ring_lock;
 652	struct dma_fence	**fences;
 653	struct amdgpu_ctx_ring	rings[AMDGPU_MAX_RINGS];
 654	bool			preamble_presented;
 655	enum drm_sched_priority init_priority;
 656	enum drm_sched_priority override_priority;
 657	struct mutex            lock;
 658	atomic_t	guilty;
 659};
 660
 661struct amdgpu_ctx_mgr {
 662	struct amdgpu_device	*adev;
 663	struct mutex		lock;
 664	/* protected by lock */
 665	struct idr		ctx_handles;
 666};
 667
 668struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id);
 669int amdgpu_ctx_put(struct amdgpu_ctx *ctx);
 670
 671int amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
 672			      struct dma_fence *fence, uint64_t *seq);
 673struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
 674				   struct amdgpu_ring *ring, uint64_t seq);
 675void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx,
 676				  enum drm_sched_priority priority);
 677
 678int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
 679		     struct drm_file *filp);
 680
 681int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx, unsigned ring_id);
 682
 683void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
 684void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr);
 685
 686
 687/*
 688 * file private structure
 689 */
 690
 691struct amdgpu_fpriv {
 692	struct amdgpu_vm	vm;
 693	struct amdgpu_bo_va	*prt_va;
 694	struct amdgpu_bo_va	*csa_va;
 695	struct mutex		bo_list_lock;
 696	struct idr		bo_list_handles;
 697	struct amdgpu_ctx_mgr	ctx_mgr;
 698};
 699
 700/*
 701 * residency list
 702 */
 703struct amdgpu_bo_list_entry {
 704	struct amdgpu_bo		*robj;
 705	struct ttm_validate_buffer	tv;
 706	struct amdgpu_bo_va		*bo_va;
 707	uint32_t			priority;
 708	struct page			**user_pages;
 709	int				user_invalidated;
 710};
 711
 712struct amdgpu_bo_list {
 713	struct mutex lock;
 714	struct rcu_head rhead;
 715	struct kref refcount;
 716	struct amdgpu_bo *gds_obj;
 717	struct amdgpu_bo *gws_obj;
 718	struct amdgpu_bo *oa_obj;
 719	unsigned first_userptr;
 720	unsigned num_entries;
 721	struct amdgpu_bo_list_entry *array;
 722};
 723
 724struct amdgpu_bo_list *
 725amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id);
 726void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list,
 727			     struct list_head *validated);
 728void amdgpu_bo_list_put(struct amdgpu_bo_list *list);
 729void amdgpu_bo_list_free(struct amdgpu_bo_list *list);
 730
 731/*
 732 * GFX stuff
 733 */
 734#include "clearstate_defs.h"
 735
 736struct amdgpu_rlc_funcs {
 737	void (*enter_safe_mode)(struct amdgpu_device *adev);
 738	void (*exit_safe_mode)(struct amdgpu_device *adev);
 739};
 740
 741struct amdgpu_rlc {
 742	/* for power gating */
 743	struct amdgpu_bo	*save_restore_obj;
 744	uint64_t		save_restore_gpu_addr;
 745	volatile uint32_t	*sr_ptr;
 746	const u32               *reg_list;
 747	u32                     reg_list_size;
 748	/* for clear state */
 749	struct amdgpu_bo	*clear_state_obj;
 750	uint64_t		clear_state_gpu_addr;
 751	volatile uint32_t	*cs_ptr;
 752	const struct cs_section_def   *cs_data;
 753	u32                     clear_state_size;
 754	/* for cp tables */
 755	struct amdgpu_bo	*cp_table_obj;
 756	uint64_t		cp_table_gpu_addr;
 757	volatile uint32_t	*cp_table_ptr;
 758	u32                     cp_table_size;
 759
 760	/* safe mode for updating CG/PG state */
 761	bool in_safe_mode;
 762	const struct amdgpu_rlc_funcs *funcs;
 763
 764	/* for firmware data */
 765	u32 save_and_restore_offset;
 766	u32 clear_state_descriptor_offset;
 767	u32 avail_scratch_ram_locations;
 768	u32 reg_restore_list_size;
 769	u32 reg_list_format_start;
 770	u32 reg_list_format_separate_start;
 771	u32 starting_offsets_start;
 772	u32 reg_list_format_size_bytes;
 773	u32 reg_list_size_bytes;
 774
 775	u32 *register_list_format;
 776	u32 *register_restore;
 777};
 778
 779#define AMDGPU_MAX_COMPUTE_QUEUES KGD_MAX_QUEUES
 780
 781struct amdgpu_mec {
 782	struct amdgpu_bo	*hpd_eop_obj;
 783	u64			hpd_eop_gpu_addr;
 784	struct amdgpu_bo	*mec_fw_obj;
 785	u64			mec_fw_gpu_addr;
 786	u32 num_mec;
 787	u32 num_pipe_per_mec;
 788	u32 num_queue_per_pipe;
 789	void			*mqd_backup[AMDGPU_MAX_COMPUTE_RINGS + 1];
 790
 791	/* These are the resources for which amdgpu takes ownership */
 792	DECLARE_BITMAP(queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
 793};
 794
 795struct amdgpu_kiq {
 796	u64			eop_gpu_addr;
 797	struct amdgpu_bo	*eop_obj;
 798	spinlock_t              ring_lock;
 799	struct amdgpu_ring	ring;
 800	struct amdgpu_irq_src	irq;
 801};
 802
 803/*
 804 * GPU scratch registers structures, functions & helpers
 805 */
 806struct amdgpu_scratch {
 807	unsigned		num_reg;
 808	uint32_t                reg_base;
 809	uint32_t		free_mask;
 
 810};
 811
 812/*
 813 * GFX configurations
 814 */
 815#define AMDGPU_GFX_MAX_SE 4
 816#define AMDGPU_GFX_MAX_SH_PER_SE 2
 817
 818struct amdgpu_rb_config {
 819	uint32_t rb_backend_disable;
 820	uint32_t user_rb_backend_disable;
 821	uint32_t raster_config;
 822	uint32_t raster_config_1;
 823};
 824
 825struct gb_addr_config {
 826	uint16_t pipe_interleave_size;
 827	uint8_t num_pipes;
 828	uint8_t max_compress_frags;
 829	uint8_t num_banks;
 830	uint8_t num_se;
 831	uint8_t num_rb_per_se;
 832};
 833
 834struct amdgpu_gfx_config {
 835	unsigned max_shader_engines;
 836	unsigned max_tile_pipes;
 837	unsigned max_cu_per_sh;
 838	unsigned max_sh_per_se;
 839	unsigned max_backends_per_se;
 840	unsigned max_texture_channel_caches;
 841	unsigned max_gprs;
 842	unsigned max_gs_threads;
 843	unsigned max_hw_contexts;
 844	unsigned sc_prim_fifo_size_frontend;
 845	unsigned sc_prim_fifo_size_backend;
 846	unsigned sc_hiz_tile_fifo_size;
 847	unsigned sc_earlyz_tile_fifo_size;
 848
 849	unsigned num_tile_pipes;
 850	unsigned backend_enable_mask;
 851	unsigned mem_max_burst_length_bytes;
 852	unsigned mem_row_size_in_kb;
 853	unsigned shader_engine_tile_size;
 854	unsigned num_gpus;
 855	unsigned multi_gpu_tile_size;
 856	unsigned mc_arb_ramcfg;
 857	unsigned gb_addr_config;
 858	unsigned num_rbs;
 859	unsigned gs_vgt_table_depth;
 860	unsigned gs_prim_buffer_depth;
 861
 862	uint32_t tile_mode_array[32];
 863	uint32_t macrotile_mode_array[16];
 864
 865	struct gb_addr_config gb_addr_config_fields;
 866	struct amdgpu_rb_config rb_config[AMDGPU_GFX_MAX_SE][AMDGPU_GFX_MAX_SH_PER_SE];
 867
 868	/* gfx configure feature */
 869	uint32_t double_offchip_lds_buf;
 870};
 871
 872struct amdgpu_cu_info {
 873	uint32_t simd_per_cu;
 874	uint32_t max_waves_per_simd;
 875	uint32_t wave_front_size;
 876	uint32_t max_scratch_slots_per_cu;
 877	uint32_t lds_size;
 878
 879	/* total active CU number */
 880	uint32_t number;
 881	uint32_t ao_cu_mask;
 882	uint32_t ao_cu_bitmap[4][4];
 883	uint32_t bitmap[4][4];
 884};
 885
 886struct amdgpu_gfx_funcs {
 887	/* get the gpu clock counter */
 888	uint64_t (*get_gpu_clock_counter)(struct amdgpu_device *adev);
 889	void (*select_se_sh)(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance);
 890	void (*read_wave_data)(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields);
 891	void (*read_wave_vgprs)(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t thread, uint32_t start, uint32_t size, uint32_t *dst);
 892	void (*read_wave_sgprs)(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t start, uint32_t size, uint32_t *dst);
 893	void (*select_me_pipe_q)(struct amdgpu_device *adev, u32 me, u32 pipe, u32 queue);
 894};
 895
 896struct amdgpu_ngg_buf {
 897	struct amdgpu_bo	*bo;
 898	uint64_t		gpu_addr;
 899	uint32_t		size;
 900	uint32_t		bo_size;
 901};
 902
 903enum {
 904	NGG_PRIM = 0,
 905	NGG_POS,
 906	NGG_CNTL,
 907	NGG_PARAM,
 908	NGG_BUF_MAX
 909};
 910
 911struct amdgpu_ngg {
 912	struct amdgpu_ngg_buf	buf[NGG_BUF_MAX];
 913	uint32_t		gds_reserve_addr;
 914	uint32_t		gds_reserve_size;
 915	bool			init;
 916};
 917
 918struct amdgpu_gfx {
 919	struct mutex			gpu_clock_mutex;
 920	struct amdgpu_gfx_config	config;
 921	struct amdgpu_rlc		rlc;
 922	struct amdgpu_mec		mec;
 923	struct amdgpu_kiq		kiq;
 924	struct amdgpu_scratch		scratch;
 925	const struct firmware		*me_fw;	/* ME firmware */
 926	uint32_t			me_fw_version;
 927	const struct firmware		*pfp_fw; /* PFP firmware */
 928	uint32_t			pfp_fw_version;
 929	const struct firmware		*ce_fw;	/* CE firmware */
 930	uint32_t			ce_fw_version;
 931	const struct firmware		*rlc_fw; /* RLC firmware */
 932	uint32_t			rlc_fw_version;
 933	const struct firmware		*mec_fw; /* MEC firmware */
 934	uint32_t			mec_fw_version;
 935	const struct firmware		*mec2_fw; /* MEC2 firmware */
 936	uint32_t			mec2_fw_version;
 937	uint32_t			me_feature_version;
 938	uint32_t			ce_feature_version;
 939	uint32_t			pfp_feature_version;
 940	uint32_t			rlc_feature_version;
 941	uint32_t			mec_feature_version;
 942	uint32_t			mec2_feature_version;
 943	struct amdgpu_ring		gfx_ring[AMDGPU_MAX_GFX_RINGS];
 944	unsigned			num_gfx_rings;
 945	struct amdgpu_ring		compute_ring[AMDGPU_MAX_COMPUTE_RINGS];
 946	unsigned			num_compute_rings;
 947	struct amdgpu_irq_src		eop_irq;
 948	struct amdgpu_irq_src		priv_reg_irq;
 949	struct amdgpu_irq_src		priv_inst_irq;
 950	/* gfx status */
 951	uint32_t			gfx_current_status;
 952	/* ce ram size*/
 953	unsigned			ce_ram_size;
 954	struct amdgpu_cu_info		cu_info;
 955	const struct amdgpu_gfx_funcs	*funcs;
 956
 957	/* reset mask */
 958	uint32_t                        grbm_soft_reset;
 959	uint32_t                        srbm_soft_reset;
 960	/* s3/s4 mask */
 961	bool                            in_suspend;
 962	/* NGG */
 963	struct amdgpu_ngg		ngg;
 964
 965	/* pipe reservation */
 966	struct mutex			pipe_reserve_mutex;
 967	DECLARE_BITMAP			(pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
 968};
 969
 970int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 971		  unsigned size, struct amdgpu_ib *ib);
 972void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib,
 973		    struct dma_fence *f);
 974int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 975		       struct amdgpu_ib *ibs, struct amdgpu_job *job,
 976		       struct dma_fence **f);
 977int amdgpu_ib_pool_init(struct amdgpu_device *adev);
 978void amdgpu_ib_pool_fini(struct amdgpu_device *adev);
 979int amdgpu_ib_ring_tests(struct amdgpu_device *adev);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 980
 981/*
 982 * CS.
 983 */
 984struct amdgpu_cs_chunk {
 985	uint32_t		chunk_id;
 986	uint32_t		length_dw;
 987	void			*kdata;
 988};
 989
 990struct amdgpu_cs_parser {
 991	struct amdgpu_device	*adev;
 992	struct drm_file		*filp;
 993	struct amdgpu_ctx	*ctx;
 994
 995	/* chunks */
 996	unsigned		nchunks;
 997	struct amdgpu_cs_chunk	*chunks;
 998
 999	/* scheduler job object */
1000	struct amdgpu_job	*job;
1001
1002	/* buffer objects */
1003	struct ww_acquire_ctx		ticket;
1004	struct amdgpu_bo_list		*bo_list;
1005	struct amdgpu_mn		*mn;
1006	struct amdgpu_bo_list_entry	vm_pd;
1007	struct list_head		validated;
1008	struct dma_fence		*fence;
1009	uint64_t			bytes_moved_threshold;
1010	uint64_t			bytes_moved_vis_threshold;
1011	uint64_t			bytes_moved;
1012	uint64_t			bytes_moved_vis;
1013	struct amdgpu_bo_list_entry	*evictable;
1014
1015	/* user fence */
1016	struct amdgpu_bo_list_entry	uf_entry;
1017
1018	unsigned num_post_dep_syncobjs;
1019	struct drm_syncobj **post_dep_syncobjs;
1020};
1021
1022#define AMDGPU_PREAMBLE_IB_PRESENT          (1 << 0) /* bit set means command submit involves a preamble IB */
1023#define AMDGPU_PREAMBLE_IB_PRESENT_FIRST    (1 << 1) /* bit set means preamble IB is first presented in belonging context */
1024#define AMDGPU_HAVE_CTX_SWITCH              (1 << 2) /* bit set means context switch occured */
1025
1026struct amdgpu_job {
1027	struct drm_sched_job    base;
1028	struct amdgpu_device	*adev;
1029	struct amdgpu_vm	*vm;
1030	struct amdgpu_ring	*ring;
1031	struct amdgpu_sync	sync;
1032	struct amdgpu_sync	sched_sync;
1033	struct amdgpu_ib	*ibs;
1034	struct dma_fence	*fence; /* the hw fence */
1035	uint32_t		preamble_status;
1036	uint32_t		num_ibs;
1037	void			*owner;
1038	uint64_t		fence_ctx; /* the fence_context this job uses */
1039	bool                    vm_needs_flush;
1040	uint64_t		vm_pd_addr;
1041	unsigned		vmid;
1042	unsigned		pasid;
1043	uint32_t		gds_base, gds_size;
1044	uint32_t		gws_base, gws_size;
1045	uint32_t		oa_base, oa_size;
1046	uint32_t		vram_lost_counter;
1047
1048	/* user fence handling */
1049	uint64_t		uf_addr;
1050	uint64_t		uf_sequence;
1051
1052};
1053#define to_amdgpu_job(sched_job)		\
1054		container_of((sched_job), struct amdgpu_job, base)
1055
1056static inline u32 amdgpu_get_ib_value(struct amdgpu_cs_parser *p,
1057				      uint32_t ib_idx, int idx)
1058{
1059	return p->job->ibs[ib_idx].ptr[idx];
1060}
1061
1062static inline void amdgpu_set_ib_value(struct amdgpu_cs_parser *p,
1063				       uint32_t ib_idx, int idx,
1064				       uint32_t value)
1065{
1066	p->job->ibs[ib_idx].ptr[idx] = value;
1067}
1068
1069/*
1070 * Writeback
1071 */
1072#define AMDGPU_MAX_WB 128	/* Reserve at most 128 WB slots for amdgpu-owned rings. */
1073
1074struct amdgpu_wb {
1075	struct amdgpu_bo	*wb_obj;
1076	volatile uint32_t	*wb;
1077	uint64_t		gpu_addr;
1078	u32			num_wb;	/* Number of wb slots actually reserved for amdgpu. */
1079	unsigned long		used[DIV_ROUND_UP(AMDGPU_MAX_WB, BITS_PER_LONG)];
1080};
1081
1082int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb);
1083void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1084
1085/*
1086 * SDMA
1087 */
1088struct amdgpu_sdma_instance {
1089	/* SDMA firmware */
1090	const struct firmware	*fw;
1091	uint32_t		fw_version;
1092	uint32_t		feature_version;
1093
1094	struct amdgpu_ring	ring;
1095	bool			burst_nop;
1096};
1097
1098struct amdgpu_sdma {
1099	struct amdgpu_sdma_instance instance[AMDGPU_MAX_SDMA_INSTANCES];
1100#ifdef CONFIG_DRM_AMDGPU_SI
1101	//SI DMA has a difference trap irq number for the second engine
1102	struct amdgpu_irq_src	trap_irq_1;
1103#endif
1104	struct amdgpu_irq_src	trap_irq;
1105	struct amdgpu_irq_src	illegal_inst_irq;
1106	int			num_instances;
1107	uint32_t                    srbm_soft_reset;
1108};
1109
1110/*
1111 * Firmware
1112 */
1113enum amdgpu_firmware_load_type {
1114	AMDGPU_FW_LOAD_DIRECT = 0,
1115	AMDGPU_FW_LOAD_SMU,
1116	AMDGPU_FW_LOAD_PSP,
1117};
1118
1119struct amdgpu_firmware {
1120	struct amdgpu_firmware_info ucode[AMDGPU_UCODE_ID_MAXIMUM];
1121	enum amdgpu_firmware_load_type load_type;
1122	struct amdgpu_bo *fw_buf;
1123	unsigned int fw_size;
1124	unsigned int max_ucodes;
1125	/* firmwares are loaded by psp instead of smu from vega10 */
1126	const struct amdgpu_psp_funcs *funcs;
1127	struct amdgpu_bo *rbuf;
1128	struct mutex mutex;
1129
1130	/* gpu info firmware data pointer */
1131	const struct firmware *gpu_info_fw;
1132
1133	void *fw_buf_ptr;
1134	uint64_t fw_buf_mc;
1135};
1136
1137/*
1138 * Benchmarking
1139 */
1140void amdgpu_benchmark(struct amdgpu_device *adev, int test_number);
1141
1142
1143/*
1144 * Testing
1145 */
1146void amdgpu_test_moves(struct amdgpu_device *adev);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1147
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1148
1149/*
1150 * amdgpu smumgr functions
1151 */
1152struct amdgpu_smumgr_funcs {
1153	int (*check_fw_load_finish)(struct amdgpu_device *adev, uint32_t fwtype);
1154	int (*request_smu_load_fw)(struct amdgpu_device *adev);
1155	int (*request_smu_specific_fw)(struct amdgpu_device *adev, uint32_t fwtype);
1156};
1157
1158/*
1159 * amdgpu smumgr
1160 */
1161struct amdgpu_smumgr {
1162	struct amdgpu_bo *toc_buf;
1163	struct amdgpu_bo *smu_buf;
1164	/* asic priv smu data */
1165	void *priv;
1166	spinlock_t smu_lock;
1167	/* smumgr functions */
1168	const struct amdgpu_smumgr_funcs *smumgr_funcs;
1169	/* ucode loading complete flag */
1170	uint32_t fw_flags;
1171};
1172
1173/*
1174 * ASIC specific register table accessible by UMD
1175 */
1176struct amdgpu_allowed_register_entry {
1177	uint32_t reg_offset;
 
1178	bool grbm_indexed;
1179};
1180
 
 
 
 
 
 
 
1181/*
1182 * ASIC specific functions.
1183 */
1184struct amdgpu_asic_funcs {
1185	bool (*read_disabled_bios)(struct amdgpu_device *adev);
1186	bool (*read_bios_from_rom)(struct amdgpu_device *adev,
1187				   u8 *bios, u32 length_bytes);
1188	int (*read_register)(struct amdgpu_device *adev, u32 se_num,
1189			     u32 sh_num, u32 reg_offset, u32 *value);
1190	void (*set_vga_state)(struct amdgpu_device *adev, bool state);
1191	int (*reset)(struct amdgpu_device *adev);
 
 
1192	/* get the reference clock */
1193	u32 (*get_xclk)(struct amdgpu_device *adev);
 
 
 
1194	/* MM block clocks */
1195	int (*set_uvd_clocks)(struct amdgpu_device *adev, u32 vclk, u32 dclk);
1196	int (*set_vce_clocks)(struct amdgpu_device *adev, u32 evclk, u32 ecclk);
1197	/* static power management */
1198	int (*get_pcie_lanes)(struct amdgpu_device *adev);
1199	void (*set_pcie_lanes)(struct amdgpu_device *adev, int lanes);
1200	/* get config memsize register */
1201	u32 (*get_config_memsize)(struct amdgpu_device *adev);
1202	/* flush hdp write queue */
1203	void (*flush_hdp)(struct amdgpu_device *adev, struct amdgpu_ring *ring);
1204	/* invalidate hdp read cache */
1205	void (*invalidate_hdp)(struct amdgpu_device *adev,
1206			       struct amdgpu_ring *ring);
1207};
1208
1209/*
1210 * IOCTL.
1211 */
1212int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
1213			    struct drm_file *filp);
1214int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data,
1215				struct drm_file *filp);
1216
1217int amdgpu_gem_info_ioctl(struct drm_device *dev, void *data,
1218			  struct drm_file *filp);
1219int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
1220			struct drm_file *filp);
1221int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data,
1222			  struct drm_file *filp);
1223int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
1224			      struct drm_file *filp);
1225int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
1226			  struct drm_file *filp);
1227int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
1228			struct drm_file *filp);
1229int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
1230int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
1231				    struct drm_file *filp);
1232int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
1233int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
1234				struct drm_file *filp);
1235
1236int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
1237				struct drm_file *filp);
1238
1239/* VRAM scratch page for HDP bug, default vram page */
1240struct amdgpu_vram_scratch {
1241	struct amdgpu_bo		*robj;
1242	volatile uint32_t		*ptr;
1243	u64				gpu_addr;
1244};
1245
1246/*
1247 * ACPI
1248 */
1249struct amdgpu_atif_notification_cfg {
1250	bool enabled;
1251	int command_code;
1252};
1253
1254struct amdgpu_atif_notifications {
1255	bool display_switch;
1256	bool expansion_mode_change;
1257	bool thermal_state;
1258	bool forced_power_state;
1259	bool system_power_state;
1260	bool display_conf_change;
1261	bool px_gfx_switch;
1262	bool brightness_change;
1263	bool dgpu_display_event;
1264};
1265
1266struct amdgpu_atif_functions {
1267	bool system_params;
1268	bool sbios_requests;
1269	bool select_active_disp;
1270	bool lid_state;
1271	bool get_tv_standard;
1272	bool set_tv_standard;
1273	bool get_panel_expansion_mode;
1274	bool set_panel_expansion_mode;
1275	bool temperature_change;
1276	bool graphics_device_types;
1277};
1278
1279struct amdgpu_atif {
1280	struct amdgpu_atif_notifications notifications;
1281	struct amdgpu_atif_functions functions;
1282	struct amdgpu_atif_notification_cfg notification_cfg;
1283	struct amdgpu_encoder *encoder_for_bl;
1284};
1285
1286struct amdgpu_atcs_functions {
1287	bool get_ext_state;
1288	bool pcie_perf_req;
1289	bool pcie_dev_rdy;
1290	bool pcie_bus_width;
1291};
1292
1293struct amdgpu_atcs {
1294	struct amdgpu_atcs_functions functions;
1295};
1296
1297/*
1298 * Firmware VRAM reservation
1299 */
1300struct amdgpu_fw_vram_usage {
1301	u64 start_offset;
1302	u64 size;
1303	struct amdgpu_bo *reserved_bo;
1304	void *va;
1305};
1306
1307/*
1308 * CGS
1309 */
1310struct cgs_device *amdgpu_cgs_create_device(struct amdgpu_device *adev);
1311void amdgpu_cgs_destroy_device(struct cgs_device *cgs_device);
 
 
 
 
 
 
1312
1313/*
1314 * Core structure, functions and helpers.
1315 */
1316typedef uint32_t (*amdgpu_rreg_t)(struct amdgpu_device*, uint32_t);
1317typedef void (*amdgpu_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
1318
1319typedef uint32_t (*amdgpu_block_rreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
1320typedef void (*amdgpu_block_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t, uint32_t);
1321
1322
1323/*
1324 * amdgpu nbio functions
1325 *
1326 */
1327struct nbio_hdp_flush_reg {
1328	u32 ref_and_mask_cp0;
1329	u32 ref_and_mask_cp1;
1330	u32 ref_and_mask_cp2;
1331	u32 ref_and_mask_cp3;
1332	u32 ref_and_mask_cp4;
1333	u32 ref_and_mask_cp5;
1334	u32 ref_and_mask_cp6;
1335	u32 ref_and_mask_cp7;
1336	u32 ref_and_mask_cp8;
1337	u32 ref_and_mask_cp9;
1338	u32 ref_and_mask_sdma0;
1339	u32 ref_and_mask_sdma1;
1340};
1341
1342struct amdgpu_nbio_funcs {
1343	const struct nbio_hdp_flush_reg *hdp_flush_reg;
1344	u32 (*get_hdp_flush_req_offset)(struct amdgpu_device *adev);
1345	u32 (*get_hdp_flush_done_offset)(struct amdgpu_device *adev);
1346	u32 (*get_pcie_index_offset)(struct amdgpu_device *adev);
1347	u32 (*get_pcie_data_offset)(struct amdgpu_device *adev);
1348	u32 (*get_rev_id)(struct amdgpu_device *adev);
1349	void (*mc_access_enable)(struct amdgpu_device *adev, bool enable);
1350	void (*hdp_flush)(struct amdgpu_device *adev, struct amdgpu_ring *ring);
1351	u32 (*get_memsize)(struct amdgpu_device *adev);
1352	void (*sdma_doorbell_range)(struct amdgpu_device *adev, int instance,
1353				    bool use_doorbell, int doorbell_index);
1354	void (*enable_doorbell_aperture)(struct amdgpu_device *adev,
1355					 bool enable);
1356	void (*enable_doorbell_selfring_aperture)(struct amdgpu_device *adev,
1357						  bool enable);
1358	void (*ih_doorbell_range)(struct amdgpu_device *adev,
1359				  bool use_doorbell, int doorbell_index);
1360	void (*update_medium_grain_clock_gating)(struct amdgpu_device *adev,
1361						 bool enable);
1362	void (*update_medium_grain_light_sleep)(struct amdgpu_device *adev,
1363						bool enable);
1364	void (*get_clockgating_state)(struct amdgpu_device *adev,
1365				      u32 *flags);
1366	void (*ih_control)(struct amdgpu_device *adev);
1367	void (*init_registers)(struct amdgpu_device *adev);
1368	void (*detect_hw_virt)(struct amdgpu_device *adev);
1369};
1370
1371
1372/* Define the HW IP blocks will be used in driver , add more if necessary */
1373enum amd_hw_ip_block_type {
1374	GC_HWIP = 1,
1375	HDP_HWIP,
1376	SDMA0_HWIP,
1377	SDMA1_HWIP,
1378	MMHUB_HWIP,
1379	ATHUB_HWIP,
1380	NBIO_HWIP,
1381	MP0_HWIP,
1382	MP1_HWIP,
1383	UVD_HWIP,
1384	VCN_HWIP = UVD_HWIP,
1385	VCE_HWIP,
1386	DF_HWIP,
1387	DCE_HWIP,
1388	OSSSYS_HWIP,
1389	SMUIO_HWIP,
1390	PWR_HWIP,
1391	NBIF_HWIP,
1392	THM_HWIP,
1393	MAX_HWIP
1394};
1395
1396#define HWIP_MAX_INSTANCE	6
1397
1398struct amd_powerplay {
1399	void *pp_handle;
1400	const struct amd_pm_funcs *pp_funcs;
1401};
1402
1403#define AMDGPU_RESET_MAGIC_NUM 64
1404struct amdgpu_device {
1405	struct device			*dev;
1406	struct drm_device		*ddev;
1407	struct pci_dev			*pdev;
1408
1409#ifdef CONFIG_DRM_AMD_ACP
1410	struct amdgpu_acp		acp;
1411#endif
1412
1413	/* ASIC */
1414	enum amd_asic_type		asic_type;
1415	uint32_t			family;
1416	uint32_t			rev_id;
1417	uint32_t			external_rev_id;
1418	unsigned long			flags;
1419	int				usec_timeout;
1420	const struct amdgpu_asic_funcs	*asic_funcs;
1421	bool				shutdown;
 
1422	bool				need_dma32;
1423	bool				need_swiotlb;
1424	bool				accel_working;
1425	struct work_struct		reset_work;
1426	struct notifier_block		acpi_nb;
1427	struct amdgpu_i2c_chan		*i2c_bus[AMDGPU_MAX_I2C_BUS];
1428	struct amdgpu_debugfs		debugfs[AMDGPU_DEBUGFS_MAX_COMPONENTS];
1429	unsigned			debugfs_count;
1430#if defined(CONFIG_DEBUG_FS)
1431	struct dentry			*debugfs_regs[AMDGPU_DEBUGFS_MAX_COMPONENTS];
1432#endif
1433	struct amdgpu_atif		atif;
1434	struct amdgpu_atcs		atcs;
1435	struct mutex			srbm_mutex;
1436	/* GRBM index mutex. Protects concurrent access to GRBM index */
1437	struct mutex                    grbm_idx_mutex;
1438	struct dev_pm_domain		vga_pm_domain;
1439	bool				have_disp_power_ref;
1440
1441	/* BIOS */
1442	bool				is_atom_fw;
1443	uint8_t				*bios;
1444	uint32_t			bios_size;
1445	struct amdgpu_bo		*stolen_vga_memory;
1446	uint32_t			bios_scratch_reg_offset;
1447	uint32_t			bios_scratch[AMDGPU_BIOS_NUM_SCRATCH];
1448
1449	/* Register/doorbell mmio */
1450	resource_size_t			rmmio_base;
1451	resource_size_t			rmmio_size;
1452	void __iomem			*rmmio;
1453	/* protects concurrent MM_INDEX/DATA based register access */
1454	spinlock_t mmio_idx_lock;
1455	/* protects concurrent SMC based register access */
1456	spinlock_t smc_idx_lock;
1457	amdgpu_rreg_t			smc_rreg;
1458	amdgpu_wreg_t			smc_wreg;
1459	/* protects concurrent PCIE register access */
1460	spinlock_t pcie_idx_lock;
1461	amdgpu_rreg_t			pcie_rreg;
1462	amdgpu_wreg_t			pcie_wreg;
1463	amdgpu_rreg_t			pciep_rreg;
1464	amdgpu_wreg_t			pciep_wreg;
1465	/* protects concurrent UVD register access */
1466	spinlock_t uvd_ctx_idx_lock;
1467	amdgpu_rreg_t			uvd_ctx_rreg;
1468	amdgpu_wreg_t			uvd_ctx_wreg;
1469	/* protects concurrent DIDT register access */
1470	spinlock_t didt_idx_lock;
1471	amdgpu_rreg_t			didt_rreg;
1472	amdgpu_wreg_t			didt_wreg;
1473	/* protects concurrent gc_cac register access */
1474	spinlock_t gc_cac_idx_lock;
1475	amdgpu_rreg_t			gc_cac_rreg;
1476	amdgpu_wreg_t			gc_cac_wreg;
1477	/* protects concurrent se_cac register access */
1478	spinlock_t se_cac_idx_lock;
1479	amdgpu_rreg_t			se_cac_rreg;
1480	amdgpu_wreg_t			se_cac_wreg;
1481	/* protects concurrent ENDPOINT (audio) register access */
1482	spinlock_t audio_endpt_idx_lock;
1483	amdgpu_block_rreg_t		audio_endpt_rreg;
1484	amdgpu_block_wreg_t		audio_endpt_wreg;
1485	void __iomem                    *rio_mem;
1486	resource_size_t			rio_mem_size;
1487	struct amdgpu_doorbell		doorbell;
1488
1489	/* clock/pll info */
1490	struct amdgpu_clock            clock;
1491
1492	/* MC */
1493	struct amdgpu_gmc		gmc;
1494	struct amdgpu_gart		gart;
1495	dma_addr_t			dummy_page_addr;
1496	struct amdgpu_vm_manager	vm_manager;
1497	struct amdgpu_vmhub             vmhub[AMDGPU_MAX_VMHUBS];
1498
1499	/* memory management */
1500	struct amdgpu_mman		mman;
1501	struct amdgpu_vram_scratch	vram_scratch;
1502	struct amdgpu_wb		wb;
 
 
 
1503	atomic64_t			num_bytes_moved;
1504	atomic64_t			num_evictions;
1505	atomic64_t			num_vram_cpu_page_faults;
1506	atomic_t			gpu_reset_counter;
1507	atomic_t			vram_lost_counter;
1508
1509	/* data for buffer migration throttling */
1510	struct {
1511		spinlock_t		lock;
1512		s64			last_update_us;
1513		s64			accum_us; /* accumulated microseconds */
1514		s64			accum_us_vis; /* for visible VRAM */
1515		u32			log2_max_MBps;
1516	} mm_stats;
1517
1518	/* display */
1519	bool				enable_virtual_display;
1520	struct amdgpu_mode_info		mode_info;
1521	/* For pre-DCE11. DCE11 and later are in "struct amdgpu_device->dm" */
1522	struct work_struct		hotplug_work;
1523	struct amdgpu_irq_src		crtc_irq;
1524	struct amdgpu_irq_src		pageflip_irq;
1525	struct amdgpu_irq_src		hpd_irq;
1526
1527	/* rings */
1528	u64				fence_context;
1529	unsigned			num_rings;
1530	struct amdgpu_ring		*rings[AMDGPU_MAX_RINGS];
1531	bool				ib_pool_ready;
1532	struct amdgpu_sa_manager	ring_tmp_bo;
1533
1534	/* interrupts */
1535	struct amdgpu_irq		irq;
1536
1537	/* powerplay */
1538	struct amd_powerplay		powerplay;
 
1539	bool				pp_force_state_enabled;
1540
1541	/* dpm */
1542	struct amdgpu_pm		pm;
1543	u32				cg_flags;
1544	u32				pg_flags;
1545
1546	/* amdgpu smumgr */
1547	struct amdgpu_smumgr smu;
1548
1549	/* gfx */
1550	struct amdgpu_gfx		gfx;
1551
1552	/* sdma */
1553	struct amdgpu_sdma		sdma;
1554
1555	/* uvd */
1556	struct amdgpu_uvd		uvd;
1557
1558	/* vce */
1559	struct amdgpu_vce		vce;
1560
1561	/* vcn */
1562	struct amdgpu_vcn		vcn;
1563
1564	/* firmwares */
1565	struct amdgpu_firmware		firmware;
1566
1567	/* PSP */
1568	struct psp_context		psp;
1569
1570	/* GDS */
1571	struct amdgpu_gds		gds;
1572
1573	/* display related functionality */
1574	struct amdgpu_display_manager dm;
1575
1576	struct amdgpu_ip_block          ip_blocks[AMDGPU_MAX_IP_NUM];
1577	int				num_ip_blocks;
 
1578	struct mutex	mn_lock;
1579	DECLARE_HASHTABLE(mn_hash, 7);
1580
1581	/* tracking pinned memory */
1582	u64 vram_pin_size;
1583	u64 invisible_pin_size;
1584	u64 gart_pin_size;
1585
1586	/* amdkfd interface */
1587	struct kfd_dev          *kfd;
1588
1589	/* soc15 register offset based on ip, instance and  segment */
1590	uint32_t 		*reg_offset[MAX_HWIP][HWIP_MAX_INSTANCE];
1591
1592	const struct amdgpu_nbio_funcs	*nbio_funcs;
1593
1594	/* delayed work_func for deferring clockgating during resume */
1595	struct delayed_work     late_init_work;
1596
1597	struct amdgpu_virt	virt;
1598	/* firmware VRAM reservation */
1599	struct amdgpu_fw_vram_usage fw_vram_usage;
1600
1601	/* link all shadow bo */
1602	struct list_head                shadow_list;
1603	struct mutex                    shadow_list_lock;
1604	/* keep an lru list of rings by HW IP */
1605	struct list_head		ring_lru_list;
1606	spinlock_t			ring_lru_list_lock;
1607
1608	/* record hw reset is performed */
1609	bool has_hw_reset;
1610	u8				reset_magic[AMDGPU_RESET_MAGIC_NUM];
1611
1612	/* record last mm index being written through WREG32*/
1613	unsigned long last_mm_index;
1614	bool                            in_gpu_reset;
1615	struct mutex  lock_reset;
1616};
1617
1618static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
1619{
1620	return container_of(bdev, struct amdgpu_device, mman.bdev);
1621}
1622
1623int amdgpu_device_init(struct amdgpu_device *adev,
1624		       struct drm_device *ddev,
1625		       struct pci_dev *pdev,
1626		       uint32_t flags);
1627void amdgpu_device_fini(struct amdgpu_device *adev);
1628int amdgpu_gpu_wait_for_idle(struct amdgpu_device *adev);
1629
1630uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
1631			uint32_t acc_flags);
1632void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
1633		    uint32_t acc_flags);
1634void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value);
1635uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset);
1636
1637u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg);
1638void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v);
1639
1640u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index);
1641void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v);
1642u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index);
1643void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v);
1644
1645bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type);
1646bool amdgpu_device_has_dc_support(struct amdgpu_device *adev);
1647
1648int emu_soc_asic_init(struct amdgpu_device *adev);
1649
1650/*
1651 * Registers read & write functions.
1652 */
1653
1654#define AMDGPU_REGS_IDX       (1<<0)
1655#define AMDGPU_REGS_NO_KIQ    (1<<1)
1656
1657#define RREG32_NO_KIQ(reg) amdgpu_mm_rreg(adev, (reg), AMDGPU_REGS_NO_KIQ)
1658#define WREG32_NO_KIQ(reg, v) amdgpu_mm_wreg(adev, (reg), (v), AMDGPU_REGS_NO_KIQ)
1659
1660#define RREG8(reg) amdgpu_mm_rreg8(adev, (reg))
1661#define WREG8(reg, v) amdgpu_mm_wreg8(adev, (reg), (v))
1662
1663#define RREG32(reg) amdgpu_mm_rreg(adev, (reg), 0)
1664#define RREG32_IDX(reg) amdgpu_mm_rreg(adev, (reg), AMDGPU_REGS_IDX)
1665#define DREG32(reg) printk(KERN_INFO "REGISTER: " #reg " : 0x%08X\n", amdgpu_mm_rreg(adev, (reg), 0))
1666#define WREG32(reg, v) amdgpu_mm_wreg(adev, (reg), (v), 0)
1667#define WREG32_IDX(reg, v) amdgpu_mm_wreg(adev, (reg), (v), AMDGPU_REGS_IDX)
1668#define REG_SET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
1669#define REG_GET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
1670#define RREG32_PCIE(reg) adev->pcie_rreg(adev, (reg))
1671#define WREG32_PCIE(reg, v) adev->pcie_wreg(adev, (reg), (v))
1672#define RREG32_PCIE_PORT(reg) adev->pciep_rreg(adev, (reg))
1673#define WREG32_PCIE_PORT(reg, v) adev->pciep_wreg(adev, (reg), (v))
1674#define RREG32_SMC(reg) adev->smc_rreg(adev, (reg))
1675#define WREG32_SMC(reg, v) adev->smc_wreg(adev, (reg), (v))
1676#define RREG32_UVD_CTX(reg) adev->uvd_ctx_rreg(adev, (reg))
1677#define WREG32_UVD_CTX(reg, v) adev->uvd_ctx_wreg(adev, (reg), (v))
1678#define RREG32_DIDT(reg) adev->didt_rreg(adev, (reg))
1679#define WREG32_DIDT(reg, v) adev->didt_wreg(adev, (reg), (v))
1680#define RREG32_GC_CAC(reg) adev->gc_cac_rreg(adev, (reg))
1681#define WREG32_GC_CAC(reg, v) adev->gc_cac_wreg(adev, (reg), (v))
1682#define RREG32_SE_CAC(reg) adev->se_cac_rreg(adev, (reg))
1683#define WREG32_SE_CAC(reg, v) adev->se_cac_wreg(adev, (reg), (v))
1684#define RREG32_AUDIO_ENDPT(block, reg) adev->audio_endpt_rreg(adev, (block), (reg))
1685#define WREG32_AUDIO_ENDPT(block, reg, v) adev->audio_endpt_wreg(adev, (block), (reg), (v))
1686#define WREG32_P(reg, val, mask)				\
1687	do {							\
1688		uint32_t tmp_ = RREG32(reg);			\
1689		tmp_ &= (mask);					\
1690		tmp_ |= ((val) & ~(mask));			\
1691		WREG32(reg, tmp_);				\
1692	} while (0)
1693#define WREG32_AND(reg, and) WREG32_P(reg, 0, and)
1694#define WREG32_OR(reg, or) WREG32_P(reg, or, ~(or))
1695#define WREG32_PLL_P(reg, val, mask)				\
1696	do {							\
1697		uint32_t tmp_ = RREG32_PLL(reg);		\
1698		tmp_ &= (mask);					\
1699		tmp_ |= ((val) & ~(mask));			\
1700		WREG32_PLL(reg, tmp_);				\
1701	} while (0)
1702#define DREG32_SYS(sqf, adev, reg) seq_printf((sqf), #reg " : 0x%08X\n", amdgpu_mm_rreg((adev), (reg), false))
1703#define RREG32_IO(reg) amdgpu_io_rreg(adev, (reg))
1704#define WREG32_IO(reg, v) amdgpu_io_wreg(adev, (reg), (v))
1705
1706#define RDOORBELL32(index) amdgpu_mm_rdoorbell(adev, (index))
1707#define WDOORBELL32(index, v) amdgpu_mm_wdoorbell(adev, (index), (v))
1708#define RDOORBELL64(index) amdgpu_mm_rdoorbell64(adev, (index))
1709#define WDOORBELL64(index, v) amdgpu_mm_wdoorbell64(adev, (index), (v))
1710
1711#define REG_FIELD_SHIFT(reg, field) reg##__##field##__SHIFT
1712#define REG_FIELD_MASK(reg, field) reg##__##field##_MASK
1713
1714#define REG_SET_FIELD(orig_val, reg, field, field_val)			\
1715	(((orig_val) & ~REG_FIELD_MASK(reg, field)) |			\
1716	 (REG_FIELD_MASK(reg, field) & ((field_val) << REG_FIELD_SHIFT(reg, field))))
1717
1718#define REG_GET_FIELD(value, reg, field)				\
1719	(((value) & REG_FIELD_MASK(reg, field)) >> REG_FIELD_SHIFT(reg, field))
1720
1721#define WREG32_FIELD(reg, field, val)	\
1722	WREG32(mm##reg, (RREG32(mm##reg) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
1723
1724#define WREG32_FIELD_OFFSET(reg, offset, field, val)	\
1725	WREG32(mm##reg + offset, (RREG32(mm##reg + offset) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
1726
1727/*
1728 * BIOS helpers.
1729 */
1730#define RBIOS8(i) (adev->bios[i])
1731#define RBIOS16(i) (RBIOS8(i) | (RBIOS8((i)+1) << 8))
1732#define RBIOS32(i) ((RBIOS16(i)) | (RBIOS16((i)+2) << 16))
1733
 
 
 
 
 
 
 
 
 
 
 
 
1734static inline struct amdgpu_sdma_instance *
1735amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
1736{
1737	struct amdgpu_device *adev = ring->adev;
1738	int i;
1739
1740	for (i = 0; i < adev->sdma.num_instances; i++)
1741		if (&adev->sdma.instance[i].ring == ring)
1742			break;
1743
1744	if (i < AMDGPU_MAX_SDMA_INSTANCES)
1745		return &adev->sdma.instance[i];
1746	else
1747		return NULL;
1748}
1749
1750/*
1751 * ASICs macro.
1752 */
1753#define amdgpu_asic_set_vga_state(adev, state) (adev)->asic_funcs->set_vga_state((adev), (state))
1754#define amdgpu_asic_reset(adev) (adev)->asic_funcs->reset((adev))
 
1755#define amdgpu_asic_get_xclk(adev) (adev)->asic_funcs->get_xclk((adev))
1756#define amdgpu_asic_set_uvd_clocks(adev, v, d) (adev)->asic_funcs->set_uvd_clocks((adev), (v), (d))
1757#define amdgpu_asic_set_vce_clocks(adev, ev, ec) (adev)->asic_funcs->set_vce_clocks((adev), (ev), (ec))
1758#define amdgpu_get_pcie_lanes(adev) (adev)->asic_funcs->get_pcie_lanes((adev))
1759#define amdgpu_set_pcie_lanes(adev, l) (adev)->asic_funcs->set_pcie_lanes((adev), (l))
1760#define amdgpu_asic_get_gpu_clock_counter(adev) (adev)->asic_funcs->get_gpu_clock_counter((adev))
1761#define amdgpu_asic_read_disabled_bios(adev) (adev)->asic_funcs->read_disabled_bios((adev))
1762#define amdgpu_asic_read_bios_from_rom(adev, b, l) (adev)->asic_funcs->read_bios_from_rom((adev), (b), (l))
1763#define amdgpu_asic_read_register(adev, se, sh, offset, v)((adev)->asic_funcs->read_register((adev), (se), (sh), (offset), (v)))
1764#define amdgpu_asic_get_config_memsize(adev) (adev)->asic_funcs->get_config_memsize((adev))
1765#define amdgpu_asic_flush_hdp(adev, r) (adev)->asic_funcs->flush_hdp((adev), (r))
1766#define amdgpu_asic_invalidate_hdp(adev, r) (adev)->asic_funcs->invalidate_hdp((adev), (r))
1767#define amdgpu_gmc_flush_gpu_tlb(adev, vmid) (adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid))
1768#define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr))
1769#define amdgpu_gmc_emit_pasid_mapping(r, vmid, pasid) (r)->adev->gmc.gmc_funcs->emit_pasid_mapping((r), (vmid), (pasid))
1770#define amdgpu_gmc_set_pte_pde(adev, pt, idx, addr, flags) (adev)->gmc.gmc_funcs->set_pte_pde((adev), (pt), (idx), (addr), (flags))
1771#define amdgpu_gmc_get_vm_pde(adev, level, dst, flags) (adev)->gmc.gmc_funcs->get_vm_pde((adev), (level), (dst), (flags))
1772#define amdgpu_gmc_get_pte_flags(adev, flags) (adev)->gmc.gmc_funcs->get_vm_pte_flags((adev),(flags))
1773#define amdgpu_vm_copy_pte(adev, ib, pe, src, count) ((adev)->vm_manager.vm_pte_funcs->copy_pte((ib), (pe), (src), (count)))
1774#define amdgpu_vm_write_pte(adev, ib, pe, value, count, incr) ((adev)->vm_manager.vm_pte_funcs->write_pte((ib), (pe), (value), (count), (incr)))
1775#define amdgpu_vm_set_pte_pde(adev, ib, pe, addr, count, incr, flags) ((adev)->vm_manager.vm_pte_funcs->set_pte_pde((ib), (pe), (addr), (count), (incr), (flags)))
1776#define amdgpu_ring_parse_cs(r, p, ib) ((r)->funcs->parse_cs((p), (ib)))
1777#define amdgpu_ring_test_ring(r) (r)->funcs->test_ring((r))
1778#define amdgpu_ring_test_ib(r, t) (r)->funcs->test_ib((r), (t))
1779#define amdgpu_ring_get_rptr(r) (r)->funcs->get_rptr((r))
1780#define amdgpu_ring_get_wptr(r) (r)->funcs->get_wptr((r))
1781#define amdgpu_ring_set_wptr(r) (r)->funcs->set_wptr((r))
1782#define amdgpu_ring_emit_ib(r, ib, vmid, c) (r)->funcs->emit_ib((r), (ib), (vmid), (c))
1783#define amdgpu_ring_emit_pipeline_sync(r) (r)->funcs->emit_pipeline_sync((r))
1784#define amdgpu_ring_emit_vm_flush(r, vmid, addr) (r)->funcs->emit_vm_flush((r), (vmid), (addr))
1785#define amdgpu_ring_emit_fence(r, addr, seq, flags) (r)->funcs->emit_fence((r), (addr), (seq), (flags))
1786#define amdgpu_ring_emit_gds_switch(r, v, db, ds, wb, ws, ab, as) (r)->funcs->emit_gds_switch((r), (v), (db), (ds), (wb), (ws), (ab), (as))
1787#define amdgpu_ring_emit_hdp_flush(r) (r)->funcs->emit_hdp_flush((r))
1788#define amdgpu_ring_emit_switch_buffer(r) (r)->funcs->emit_switch_buffer((r))
1789#define amdgpu_ring_emit_cntxcntl(r, d) (r)->funcs->emit_cntxcntl((r), (d))
1790#define amdgpu_ring_emit_rreg(r, d) (r)->funcs->emit_rreg((r), (d))
1791#define amdgpu_ring_emit_wreg(r, d, v) (r)->funcs->emit_wreg((r), (d), (v))
1792#define amdgpu_ring_emit_reg_wait(r, d, v, m) (r)->funcs->emit_reg_wait((r), (d), (v), (m))
1793#define amdgpu_ring_emit_tmz(r, b) (r)->funcs->emit_tmz((r), (b))
1794#define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib)))
1795#define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r))
1796#define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o))
1797#define amdgpu_ih_get_wptr(adev) (adev)->irq.ih_funcs->get_wptr((adev))
1798#define amdgpu_ih_prescreen_iv(adev) (adev)->irq.ih_funcs->prescreen_iv((adev))
1799#define amdgpu_ih_decode_iv(adev, iv) (adev)->irq.ih_funcs->decode_iv((adev), (iv))
1800#define amdgpu_ih_set_rptr(adev) (adev)->irq.ih_funcs->set_rptr((adev))
 
1801#define amdgpu_display_vblank_get_counter(adev, crtc) (adev)->mode_info.funcs->vblank_get_counter((adev), (crtc))
 
 
1802#define amdgpu_display_backlight_set_level(adev, e, l) (adev)->mode_info.funcs->backlight_set_level((e), (l))
1803#define amdgpu_display_backlight_get_level(adev, e) (adev)->mode_info.funcs->backlight_get_level((e))
1804#define amdgpu_display_hpd_sense(adev, h) (adev)->mode_info.funcs->hpd_sense((adev), (h))
1805#define amdgpu_display_hpd_set_polarity(adev, h) (adev)->mode_info.funcs->hpd_set_polarity((adev), (h))
1806#define amdgpu_display_hpd_get_gpio_reg(adev) (adev)->mode_info.funcs->hpd_get_gpio_reg((adev))
1807#define amdgpu_display_bandwidth_update(adev) (adev)->mode_info.funcs->bandwidth_update((adev))
1808#define amdgpu_display_page_flip(adev, crtc, base, async) (adev)->mode_info.funcs->page_flip((adev), (crtc), (base), (async))
1809#define amdgpu_display_page_flip_get_scanoutpos(adev, crtc, vbl, pos) (adev)->mode_info.funcs->page_flip_get_scanoutpos((adev), (crtc), (vbl), (pos))
1810#define amdgpu_display_add_encoder(adev, e, s, c) (adev)->mode_info.funcs->add_encoder((adev), (e), (s), (c))
1811#define amdgpu_display_add_connector(adev, ci, sd, ct, ib, coi, h, r) (adev)->mode_info.funcs->add_connector((adev), (ci), (sd), (ct), (ib), (coi), (h), (r))
 
 
1812#define amdgpu_emit_copy_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_copy_buffer((ib),  (s), (d), (b))
1813#define amdgpu_emit_fill_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b))
1814#define amdgpu_gfx_get_gpu_clock_counter(adev) (adev)->gfx.funcs->get_gpu_clock_counter((adev))
1815#define amdgpu_gfx_select_se_sh(adev, se, sh, instance) (adev)->gfx.funcs->select_se_sh((adev), (se), (sh), (instance))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1816#define amdgpu_gds_switch(adev, r, v, d, w, a) (adev)->gds.funcs->patch_gds_switch((r), (v), (d), (w), (a))
1817#define amdgpu_psp_check_fw_loading_status(adev, i) (adev)->firmware.funcs->check_fw_loading_status((adev), (i))
1818#define amdgpu_gfx_select_me_pipe_q(adev, me, pipe, q) (adev)->gfx.funcs->select_me_pipe_q((adev), (me), (pipe), (q))
1819
1820/* Common functions */
1821int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
1822			      struct amdgpu_job* job, bool force);
1823void amdgpu_device_pci_config_reset(struct amdgpu_device *adev);
1824bool amdgpu_device_need_post(struct amdgpu_device *adev);
1825void amdgpu_display_update_priority(struct amdgpu_device *adev);
1826
1827void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
1828				  u64 num_vis_bytes);
1829void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain);
 
1830bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);
1831void amdgpu_device_vram_location(struct amdgpu_device *adev,
1832				 struct amdgpu_gmc *mc, u64 base);
1833void amdgpu_device_gart_location(struct amdgpu_device *adev,
1834				 struct amdgpu_gmc *mc);
1835int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev);
1836void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
 
 
 
 
 
 
 
 
 
 
1837					     const u32 *registers,
1838					     const u32 array_size);
1839
1840bool amdgpu_device_is_px(struct drm_device *dev);
1841/* atpx handler */
1842#if defined(CONFIG_VGA_SWITCHEROO)
1843void amdgpu_register_atpx_handler(void);
1844void amdgpu_unregister_atpx_handler(void);
1845bool amdgpu_has_atpx_dgpu_power_cntl(void);
1846bool amdgpu_is_atpx_hybrid(void);
1847bool amdgpu_atpx_dgpu_req_power_for_displays(void);
1848bool amdgpu_has_atpx(void);
1849#else
1850static inline void amdgpu_register_atpx_handler(void) {}
1851static inline void amdgpu_unregister_atpx_handler(void) {}
1852static inline bool amdgpu_has_atpx_dgpu_power_cntl(void) { return false; }
1853static inline bool amdgpu_is_atpx_hybrid(void) { return false; }
1854static inline bool amdgpu_atpx_dgpu_req_power_for_displays(void) { return false; }
1855static inline bool amdgpu_has_atpx(void) { return false; }
1856#endif
1857
1858/*
1859 * KMS
1860 */
1861extern const struct drm_ioctl_desc amdgpu_ioctls_kms[];
1862extern const int amdgpu_max_kms_ioctl;
1863
1864int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags);
1865void amdgpu_driver_unload_kms(struct drm_device *dev);
1866void amdgpu_driver_lastclose_kms(struct drm_device *dev);
1867int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv);
1868void amdgpu_driver_postclose_kms(struct drm_device *dev,
1869				 struct drm_file *file_priv);
1870int amdgpu_device_ip_suspend(struct amdgpu_device *adev);
1871int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon);
1872int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon);
 
1873u32 amdgpu_get_vblank_counter_kms(struct drm_device *dev, unsigned int pipe);
1874int amdgpu_enable_vblank_kms(struct drm_device *dev, unsigned int pipe);
1875void amdgpu_disable_vblank_kms(struct drm_device *dev, unsigned int pipe);
 
 
 
 
1876long amdgpu_kms_compat_ioctl(struct file *filp, unsigned int cmd,
1877			     unsigned long arg);
1878
1879/*
1880 * functions used by amdgpu_encoder.c
1881 */
1882struct amdgpu_afmt_acr {
1883	u32 clock;
1884
1885	int n_32khz;
1886	int cts_32khz;
1887
1888	int n_44_1khz;
1889	int cts_44_1khz;
1890
1891	int n_48khz;
1892	int cts_48khz;
1893
1894};
1895
1896struct amdgpu_afmt_acr amdgpu_afmt_acr(uint32_t clock);
1897
1898/* amdgpu_acpi.c */
1899#if defined(CONFIG_ACPI)
1900int amdgpu_acpi_init(struct amdgpu_device *adev);
1901void amdgpu_acpi_fini(struct amdgpu_device *adev);
1902bool amdgpu_acpi_is_pcie_performance_request_supported(struct amdgpu_device *adev);
1903int amdgpu_acpi_pcie_performance_request(struct amdgpu_device *adev,
1904						u8 perf_req, bool advertise);
1905int amdgpu_acpi_pcie_notify_device_ready(struct amdgpu_device *adev);
1906#else
1907static inline int amdgpu_acpi_init(struct amdgpu_device *adev) { return 0; }
1908static inline void amdgpu_acpi_fini(struct amdgpu_device *adev) { }
1909#endif
1910
1911int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
1912			   uint64_t addr, struct amdgpu_bo **bo,
1913			   struct amdgpu_bo_va_mapping **mapping);
1914
1915#if defined(CONFIG_DRM_AMD_DC)
1916int amdgpu_dm_display_resume(struct amdgpu_device *adev );
1917#else
1918static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return 0; }
1919#endif
1920
1921#include "amdgpu_object.h"
 
1922#endif