1/*
   2 * Copyright 2018 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/firmware.h>
  25#include <drm/drm_drv.h>
  26
  27#include "amdgpu.h"
  28#include "amdgpu_vcn.h"
  29#include "soc15.h"
  30#include "soc15d.h"
  31#include "amdgpu_pm.h"
  32#include "amdgpu_psp.h"
  33#include "mmsch_v2_0.h"
  34#include "vcn_v2_0.h"
  35
  36#include "vcn/vcn_2_0_0_offset.h"
  37#include "vcn/vcn_2_0_0_sh_mask.h"
  38#include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
  39
  40#define VCN_VID_SOC_ADDRESS_2_0					0x1fa00
  41#define VCN1_VID_SOC_ADDRESS_3_0				0x48200
  42
  43#define mmUVD_CONTEXT_ID_INTERNAL_OFFSET			0x1fd
  44#define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET			0x503
  45#define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET			0x504
  46#define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET			0x505
  47#define mmUVD_NO_OP_INTERNAL_OFFSET				0x53f
  48#define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET			0x54a
  49#define mmUVD_SCRATCH9_INTERNAL_OFFSET				0xc01d
  50
  51#define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET			0x1e1
  52#define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET		0x5a6
  53#define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET		0x5a7
  54#define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET			0x1e2
  55
  56static void vcn_v2_0_set_dec_ring_funcs(struct amdgpu_device *adev);
  57static void vcn_v2_0_set_enc_ring_funcs(struct amdgpu_device *adev);
  58static void vcn_v2_0_set_irq_funcs(struct amdgpu_device *adev);
  59static int vcn_v2_0_set_powergating_state(void *handle,
  60				enum amd_powergating_state state);
  61static int vcn_v2_0_pause_dpg_mode(struct amdgpu_device *adev,
  62				int inst_idx, struct dpg_pause_state *new_state);
  63static int vcn_v2_0_start_sriov(struct amdgpu_device *adev);
  64/**
  65 * vcn_v2_0_early_init - set function pointers and load microcode
  66 *
  67 * @handle: amdgpu_device pointer
  68 *
  69 * Set ring and irq function pointers
  70 * Load microcode from filesystem
  71 */
  72static int vcn_v2_0_early_init(void *handle)
  73{
  74	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  75
  76	if (amdgpu_sriov_vf(adev))
  77		adev->vcn.num_enc_rings = 1;
  78	else
  79		adev->vcn.num_enc_rings = 2;
  80
  81	vcn_v2_0_set_dec_ring_funcs(adev);
  82	vcn_v2_0_set_enc_ring_funcs(adev);
  83	vcn_v2_0_set_irq_funcs(adev);
  84
  85	return amdgpu_vcn_early_init(adev);
  86}
  87
  88/**
  89 * vcn_v2_0_sw_init - sw init for VCN block
  90 *
  91 * @handle: amdgpu_device pointer
  92 *
  93 * Load firmware and sw initialization
  94 */
  95static int vcn_v2_0_sw_init(void *handle)
  96{
  97	struct amdgpu_ring *ring;
  98	int i, r;
  99	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 100	volatile struct amdgpu_fw_shared *fw_shared;
 101
 102	/* VCN DEC TRAP */
 103	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
 104			      VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT,
 105			      &adev->vcn.inst->irq);
 106	if (r)
 107		return r;
 108
 109	/* VCN ENC TRAP */
 110	for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 111		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
 112				      i + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE,
 113				      &adev->vcn.inst->irq);
 114		if (r)
 115			return r;
 116	}
 117
 118	r = amdgpu_vcn_sw_init(adev);
 119	if (r)
 120		return r;
 121
 122	amdgpu_vcn_setup_ucode(adev);
 123
 124	r = amdgpu_vcn_resume(adev);
 125	if (r)
 126		return r;
 127
 128	ring = &adev->vcn.inst->ring_dec;
 129
 130	ring->use_doorbell = true;
 131	ring->doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1 << 1;
 132	ring->vm_hub = AMDGPU_MMHUB0(0);
 133
 134	sprintf(ring->name, "vcn_dec");
 135	r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0,
 136			     AMDGPU_RING_PRIO_DEFAULT, NULL);
 137	if (r)
 138		return r;
 139
 140	adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;
 141	adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;
 142	adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;
 143	adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;
 144	adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;
 145	adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;
 146
 147	adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
 148	adev->vcn.inst->external.scratch9 = SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9);
 149	adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
 150	adev->vcn.inst->external.data0 = SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0);
 151	adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
 152	adev->vcn.inst->external.data1 = SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1);
 153	adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
 154	adev->vcn.inst->external.cmd = SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD);
 155	adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
 156	adev->vcn.inst->external.nop = SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP);
 157
 158	for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 159		enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(i);
 160
 161		ring = &adev->vcn.inst->ring_enc[i];
 162		ring->use_doorbell = true;
 163		ring->vm_hub = AMDGPU_MMHUB0(0);
 164		if (!amdgpu_sriov_vf(adev))
 165			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + i;
 166		else
 167			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1 + i;
 168		sprintf(ring->name, "vcn_enc%d", i);
 169		r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0,
 170				     hw_prio, NULL);
 171		if (r)
 172			return r;
 173	}
 174
 175	adev->vcn.pause_dpg_mode = vcn_v2_0_pause_dpg_mode;
 176
 177	r = amdgpu_virt_alloc_mm_table(adev);
 178	if (r)
 179		return r;
 180
 181	fw_shared = adev->vcn.inst->fw_shared.cpu_addr;
 182	fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG);
 183
 184	if (amdgpu_vcnfw_log)
 185		amdgpu_vcn_fwlog_init(adev->vcn.inst);
 186
 187	return 0;
 188}
 189
 190/**
 191 * vcn_v2_0_sw_fini - sw fini for VCN block
 192 *
 193 * @handle: amdgpu_device pointer
 194 *
 195 * VCN suspend and free up sw allocation
 196 */
 197static int vcn_v2_0_sw_fini(void *handle)
 198{
 199	int r, idx;
 200	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 201	volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared.cpu_addr;
 202
 203	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
 204		fw_shared->present_flag_0 = 0;
 205		drm_dev_exit(idx);
 206	}
 207
 208	amdgpu_virt_free_mm_table(adev);
 209
 210	r = amdgpu_vcn_suspend(adev);
 211	if (r)
 212		return r;
 213
 214	r = amdgpu_vcn_sw_fini(adev);
 215
 216	return r;
 217}
 218
 219/**
 220 * vcn_v2_0_hw_init - start and test VCN block
 221 *
 222 * @handle: amdgpu_device pointer
 223 *
 224 * Initialize the hardware, boot up the VCPU and do some testing
 225 */
 226static int vcn_v2_0_hw_init(void *handle)
 227{
 228	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 229	struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec;
 230	int i, r;
 231
 232	adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
 233					     ring->doorbell_index, 0);
 234
 235	if (amdgpu_sriov_vf(adev))
 236		vcn_v2_0_start_sriov(adev);
 237
 238	r = amdgpu_ring_test_helper(ring);
 239	if (r)
 240		goto done;
 241
 242	//Disable vcn decode for sriov
 243	if (amdgpu_sriov_vf(adev))
 244		ring->sched.ready = false;
 245
 246	for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 247		ring = &adev->vcn.inst->ring_enc[i];
 248		r = amdgpu_ring_test_helper(ring);
 249		if (r)
 250			goto done;
 251	}
 252
 253done:
 254	if (!r)
 255		DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
 256			(adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
 257
 258	return r;
 259}
 260
 261/**
 262 * vcn_v2_0_hw_fini - stop the hardware block
 263 *
 264 * @handle: amdgpu_device pointer
 265 *
 266 * Stop the VCN block, mark ring as not ready any more
 267 */
 268static int vcn_v2_0_hw_fini(void *handle)
 269{
 270	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 271
 272	cancel_delayed_work_sync(&adev->vcn.idle_work);
 273
 274	if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
 275	    (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
 276	      RREG32_SOC15(VCN, 0, mmUVD_STATUS)))
 277		vcn_v2_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
 278
 279	return 0;
 280}
 281
 282/**
 283 * vcn_v2_0_suspend - suspend VCN block
 284 *
 285 * @handle: amdgpu_device pointer
 286 *
 287 * HW fini and suspend VCN block
 288 */
 289static int vcn_v2_0_suspend(void *handle)
 290{
 291	int r;
 292	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 293
 294	r = vcn_v2_0_hw_fini(adev);
 295	if (r)
 296		return r;
 297
 298	r = amdgpu_vcn_suspend(adev);
 299
 300	return r;
 301}
 302
 303/**
 304 * vcn_v2_0_resume - resume VCN block
 305 *
 306 * @handle: amdgpu_device pointer
 307 *
 308 * Resume firmware and hw init VCN block
 309 */
 310static int vcn_v2_0_resume(void *handle)
 311{
 312	int r;
 313	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 314
 315	r = amdgpu_vcn_resume(adev);
 316	if (r)
 317		return r;
 318
 319	r = vcn_v2_0_hw_init(adev);
 320
 321	return r;
 322}
 323
 324/**
 325 * vcn_v2_0_mc_resume - memory controller programming
 326 *
 327 * @adev: amdgpu_device pointer
 328 *
 329 * Let the VCN memory controller know it's offsets
 330 */
 331static void vcn_v2_0_mc_resume(struct amdgpu_device *adev)
 332{
 333	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[0]->size + 4);
 334	uint32_t offset;
 335
 336	if (amdgpu_sriov_vf(adev))
 337		return;
 338
 339	/* cache window 0: fw */
 340	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 341		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 342			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo));
 343		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 344			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi));
 345		WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, 0);
 346		offset = 0;
 347	} else {
 348		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 349			lower_32_bits(adev->vcn.inst->gpu_addr));
 350		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 351			upper_32_bits(adev->vcn.inst->gpu_addr));
 352		offset = size;
 353		WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
 354			AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
 355	}
 356
 357	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size);
 358
 359	/* cache window 1: stack */
 360	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
 361		lower_32_bits(adev->vcn.inst->gpu_addr + offset));
 362	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
 363		upper_32_bits(adev->vcn.inst->gpu_addr + offset));
 364	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, 0);
 365	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
 366
 367	/* cache window 2: context */
 368	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
 369		lower_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 370	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
 371		upper_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 372	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, 0);
 373	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
 374
 375	/* non-cache window */
 376	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
 377		lower_32_bits(adev->vcn.inst->fw_shared.gpu_addr));
 378	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
 379		upper_32_bits(adev->vcn.inst->fw_shared.gpu_addr));
 380	WREG32_SOC15(UVD, 0, mmUVD_VCPU_NONCACHE_OFFSET0, 0);
 381	WREG32_SOC15(UVD, 0, mmUVD_VCPU_NONCACHE_SIZE0,
 382		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));
 383
 384	WREG32_SOC15(UVD, 0, mmUVD_GFX10_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
 385}
 386
 387static void vcn_v2_0_mc_resume_dpg_mode(struct amdgpu_device *adev, bool indirect)
 388{
 389	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[0]->size + 4);
 390	uint32_t offset;
 391
 392	/* cache window 0: fw */
 393	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 394		if (!indirect) {
 395			WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 396				UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 397				(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo), 0, indirect);
 398			WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 399				UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 400				(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi), 0, indirect);
 401			WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 402				UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
 403		} else {
 404			WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 405				UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
 406			WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 407				UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
 408			WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 409				UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
 410		}
 411		offset = 0;
 412	} else {
 413		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 414			UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 415			lower_32_bits(adev->vcn.inst->gpu_addr), 0, indirect);
 416		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 417			UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 418			upper_32_bits(adev->vcn.inst->gpu_addr), 0, indirect);
 419		offset = size;
 420		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 421			UVD, 0, mmUVD_VCPU_CACHE_OFFSET0),
 422			AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
 423	}
 424
 425	if (!indirect)
 426		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 427			UVD, 0, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
 428	else
 429		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 430			UVD, 0, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
 431
 432	/* cache window 1: stack */
 433	if (!indirect) {
 434		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 435			UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
 436			lower_32_bits(adev->vcn.inst->gpu_addr + offset), 0, indirect);
 437		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 438			UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
 439			upper_32_bits(adev->vcn.inst->gpu_addr + offset), 0, indirect);
 440		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 441			UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
 442	} else {
 443		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 444			UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
 445		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 446			UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
 447		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 448			UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
 449	}
 450	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 451		UVD, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
 452
 453	/* cache window 2: context */
 454	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 455		UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
 456		lower_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
 457	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 458		UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
 459		upper_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
 460	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 461		UVD, 0, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
 462	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 463		UVD, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
 464
 465	/* non-cache window */
 466	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 467		UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
 468		lower_32_bits(adev->vcn.inst->fw_shared.gpu_addr), 0, indirect);
 469	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 470		UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
 471		upper_32_bits(adev->vcn.inst->fw_shared.gpu_addr), 0, indirect);
 472	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 473		UVD, 0, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
 474	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 475		UVD, 0, mmUVD_VCPU_NONCACHE_SIZE0),
 476		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);
 477
 478	/* VCN global tiling registers */
 479	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 480		UVD, 0, mmUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
 481}
 482
 483/**
 484 * vcn_v2_0_disable_clock_gating - disable VCN clock gating
 485 *
 486 * @adev: amdgpu_device pointer
 487 *
 488 * Disable clock gating for VCN block
 489 */
 490static void vcn_v2_0_disable_clock_gating(struct amdgpu_device *adev)
 491{
 492	uint32_t data;
 493
 494	if (amdgpu_sriov_vf(adev))
 495		return;
 496
 497	/* UVD disable CGC */
 498	data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 499	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 500		data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 501	else
 502		data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
 503	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 504	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 505	WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 506
 507	data = RREG32_SOC15(VCN, 0, mmUVD_CGC_GATE);
 508	data &= ~(UVD_CGC_GATE__SYS_MASK
 509		| UVD_CGC_GATE__UDEC_MASK
 510		| UVD_CGC_GATE__MPEG2_MASK
 511		| UVD_CGC_GATE__REGS_MASK
 512		| UVD_CGC_GATE__RBC_MASK
 513		| UVD_CGC_GATE__LMI_MC_MASK
 514		| UVD_CGC_GATE__LMI_UMC_MASK
 515		| UVD_CGC_GATE__IDCT_MASK
 516		| UVD_CGC_GATE__MPRD_MASK
 517		| UVD_CGC_GATE__MPC_MASK
 518		| UVD_CGC_GATE__LBSI_MASK
 519		| UVD_CGC_GATE__LRBBM_MASK
 520		| UVD_CGC_GATE__UDEC_RE_MASK
 521		| UVD_CGC_GATE__UDEC_CM_MASK
 522		| UVD_CGC_GATE__UDEC_IT_MASK
 523		| UVD_CGC_GATE__UDEC_DB_MASK
 524		| UVD_CGC_GATE__UDEC_MP_MASK
 525		| UVD_CGC_GATE__WCB_MASK
 526		| UVD_CGC_GATE__VCPU_MASK
 527		| UVD_CGC_GATE__SCPU_MASK);
 528	WREG32_SOC15(VCN, 0, mmUVD_CGC_GATE, data);
 529
 530	data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 531	data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 532		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 533		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 534		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 535		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 536		| UVD_CGC_CTRL__SYS_MODE_MASK
 537		| UVD_CGC_CTRL__UDEC_MODE_MASK
 538		| UVD_CGC_CTRL__MPEG2_MODE_MASK
 539		| UVD_CGC_CTRL__REGS_MODE_MASK
 540		| UVD_CGC_CTRL__RBC_MODE_MASK
 541		| UVD_CGC_CTRL__LMI_MC_MODE_MASK
 542		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 543		| UVD_CGC_CTRL__IDCT_MODE_MASK
 544		| UVD_CGC_CTRL__MPRD_MODE_MASK
 545		| UVD_CGC_CTRL__MPC_MODE_MASK
 546		| UVD_CGC_CTRL__LBSI_MODE_MASK
 547		| UVD_CGC_CTRL__LRBBM_MODE_MASK
 548		| UVD_CGC_CTRL__WCB_MODE_MASK
 549		| UVD_CGC_CTRL__VCPU_MODE_MASK
 550		| UVD_CGC_CTRL__SCPU_MODE_MASK);
 551	WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 552
 553	/* turn on */
 554	data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE);
 555	data |= (UVD_SUVD_CGC_GATE__SRE_MASK
 556		| UVD_SUVD_CGC_GATE__SIT_MASK
 557		| UVD_SUVD_CGC_GATE__SMP_MASK
 558		| UVD_SUVD_CGC_GATE__SCM_MASK
 559		| UVD_SUVD_CGC_GATE__SDB_MASK
 560		| UVD_SUVD_CGC_GATE__SRE_H264_MASK
 561		| UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
 562		| UVD_SUVD_CGC_GATE__SIT_H264_MASK
 563		| UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
 564		| UVD_SUVD_CGC_GATE__SCM_H264_MASK
 565		| UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
 566		| UVD_SUVD_CGC_GATE__SDB_H264_MASK
 567		| UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
 568		| UVD_SUVD_CGC_GATE__SCLR_MASK
 569		| UVD_SUVD_CGC_GATE__UVD_SC_MASK
 570		| UVD_SUVD_CGC_GATE__ENT_MASK
 571		| UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
 572		| UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
 573		| UVD_SUVD_CGC_GATE__SITE_MASK
 574		| UVD_SUVD_CGC_GATE__SRE_VP9_MASK
 575		| UVD_SUVD_CGC_GATE__SCM_VP9_MASK
 576		| UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
 577		| UVD_SUVD_CGC_GATE__SDB_VP9_MASK
 578		| UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
 579	WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE, data);
 580
 581	data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
 582	data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 583		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 584		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 585		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 586		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 587		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 588		| UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 589		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 590		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 591		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 592	WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
 593}
 594
 595static void vcn_v2_0_clock_gating_dpg_mode(struct amdgpu_device *adev,
 596		uint8_t sram_sel, uint8_t indirect)
 597{
 598	uint32_t reg_data = 0;
 599
 600	/* enable sw clock gating control */
 601	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 602		reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 603	else
 604		reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 605	reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 606	reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 607	reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
 608		 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
 609		 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
 610		 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
 611		 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
 612		 UVD_CGC_CTRL__SYS_MODE_MASK |
 613		 UVD_CGC_CTRL__UDEC_MODE_MASK |
 614		 UVD_CGC_CTRL__MPEG2_MODE_MASK |
 615		 UVD_CGC_CTRL__REGS_MODE_MASK |
 616		 UVD_CGC_CTRL__RBC_MODE_MASK |
 617		 UVD_CGC_CTRL__LMI_MC_MODE_MASK |
 618		 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
 619		 UVD_CGC_CTRL__IDCT_MODE_MASK |
 620		 UVD_CGC_CTRL__MPRD_MODE_MASK |
 621		 UVD_CGC_CTRL__MPC_MODE_MASK |
 622		 UVD_CGC_CTRL__LBSI_MODE_MASK |
 623		 UVD_CGC_CTRL__LRBBM_MODE_MASK |
 624		 UVD_CGC_CTRL__WCB_MODE_MASK |
 625		 UVD_CGC_CTRL__VCPU_MODE_MASK |
 626		 UVD_CGC_CTRL__SCPU_MODE_MASK);
 627	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 628		UVD, 0, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
 629
 630	/* turn off clock gating */
 631	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 632		UVD, 0, mmUVD_CGC_GATE), 0, sram_sel, indirect);
 633
 634	/* turn on SUVD clock gating */
 635	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 636		UVD, 0, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
 637
 638	/* turn on sw mode in UVD_SUVD_CGC_CTRL */
 639	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 640		UVD, 0, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
 641}
 642
 643/**
 644 * vcn_v2_0_enable_clock_gating - enable VCN clock gating
 645 *
 646 * @adev: amdgpu_device pointer
 647 *
 648 * Enable clock gating for VCN block
 649 */
 650static void vcn_v2_0_enable_clock_gating(struct amdgpu_device *adev)
 651{
 652	uint32_t data = 0;
 653
 654	if (amdgpu_sriov_vf(adev))
 655		return;
 656
 657	/* enable UVD CGC */
 658	data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 659	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 660		data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 661	else
 662		data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 663	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 664	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 665	WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 666
 667	data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 668	data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 669		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 670		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 671		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 672		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 673		| UVD_CGC_CTRL__SYS_MODE_MASK
 674		| UVD_CGC_CTRL__UDEC_MODE_MASK
 675		| UVD_CGC_CTRL__MPEG2_MODE_MASK
 676		| UVD_CGC_CTRL__REGS_MODE_MASK
 677		| UVD_CGC_CTRL__RBC_MODE_MASK
 678		| UVD_CGC_CTRL__LMI_MC_MODE_MASK
 679		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 680		| UVD_CGC_CTRL__IDCT_MODE_MASK
 681		| UVD_CGC_CTRL__MPRD_MODE_MASK
 682		| UVD_CGC_CTRL__MPC_MODE_MASK
 683		| UVD_CGC_CTRL__LBSI_MODE_MASK
 684		| UVD_CGC_CTRL__LRBBM_MODE_MASK
 685		| UVD_CGC_CTRL__WCB_MODE_MASK
 686		| UVD_CGC_CTRL__VCPU_MODE_MASK
 687		| UVD_CGC_CTRL__SCPU_MODE_MASK);
 688	WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 689
 690	data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
 691	data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 692		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 693		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 694		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 695		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 696		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 697		| UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 698		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 699		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 700		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 701	WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
 702}
 703
 704static void vcn_v2_0_disable_static_power_gating(struct amdgpu_device *adev)
 705{
 706	uint32_t data = 0;
 707
 708	if (amdgpu_sriov_vf(adev))
 709		return;
 710
 711	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 712		data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 713			| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
 714			| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 715			| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
 716			| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 717			| 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
 718			| 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
 719			| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 720			| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 721			| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT);
 722
 723		WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
 724		SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS,
 725			UVD_PGFSM_STATUS__UVDM_UVDU_PWR_ON_2_0, 0xFFFFF);
 726	} else {
 727		data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 728			| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
 729			| 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 730			| 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
 731			| 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 732			| 1 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
 733			| 1 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
 734			| 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 735			| 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 736			| 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT);
 737		WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
 738		SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, 0,  0xFFFFF);
 739	}
 740
 741	/* polling UVD_PGFSM_STATUS to confirm UVDM_PWR_STATUS,
 742	 * UVDU_PWR_STATUS are 0 (power on) */
 743
 744	data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
 745	data &= ~0x103;
 746	if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
 747		data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
 748			UVD_POWER_STATUS__UVD_PG_EN_MASK;
 749
 750	WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
 751}
 752
 753static void vcn_v2_0_enable_static_power_gating(struct amdgpu_device *adev)
 754{
 755	uint32_t data = 0;
 756
 757	if (amdgpu_sriov_vf(adev))
 758		return;
 759
 760	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 761		/* Before power off, this indicator has to be turned on */
 762		data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
 763		data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
 764		data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
 765		WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
 766
 767
 768		data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 769			| 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
 770			| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 771			| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
 772			| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 773			| 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
 774			| 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
 775			| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 776			| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 777			| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT);
 778
 779		WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
 780
 781		data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
 782			| 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
 783			| 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
 784			| 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
 785			| 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
 786			| 2 << UVD_PGFSM_STATUS__UVDIL_PWR_STATUS__SHIFT
 787			| 2 << UVD_PGFSM_STATUS__UVDIR_PWR_STATUS__SHIFT
 788			| 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
 789			| 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
 790			| 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT);
 791		SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, data, 0xFFFFF);
 792	}
 793}
 794
 795static int vcn_v2_0_start_dpg_mode(struct amdgpu_device *adev, bool indirect)
 796{
 797	volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared.cpu_addr;
 798	struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec;
 799	uint32_t rb_bufsz, tmp;
 800
 801	vcn_v2_0_enable_static_power_gating(adev);
 802
 803	/* enable dynamic power gating mode */
 804	tmp = RREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS);
 805	tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
 806	tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
 807	WREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS, tmp);
 808
 809	if (indirect)
 810		adev->vcn.inst->dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst->dpg_sram_cpu_addr;
 811
 812	/* enable clock gating */
 813	vcn_v2_0_clock_gating_dpg_mode(adev, 0, indirect);
 814
 815	/* enable VCPU clock */
 816	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
 817	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
 818	tmp |= UVD_VCPU_CNTL__MIF_WR_LOW_THRESHOLD_BP_MASK;
 819	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 820		UVD, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect);
 821
 822	/* disable master interupt */
 823	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 824		UVD, 0, mmUVD_MASTINT_EN), 0, 0, indirect);
 825
 826	/* setup mmUVD_LMI_CTRL */
 827	tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 828		UVD_LMI_CTRL__REQ_MODE_MASK |
 829		UVD_LMI_CTRL__CRC_RESET_MASK |
 830		UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
 831		UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 832		UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
 833		(8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
 834		0x00100000L);
 835	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 836		UVD, 0, mmUVD_LMI_CTRL), tmp, 0, indirect);
 837
 838	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 839		UVD, 0, mmUVD_MPC_CNTL),
 840		0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
 841
 842	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 843		UVD, 0, mmUVD_MPC_SET_MUXA0),
 844		((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
 845		 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
 846		 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
 847		 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
 848
 849	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 850		UVD, 0, mmUVD_MPC_SET_MUXB0),
 851		((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
 852		 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
 853		 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
 854		 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
 855
 856	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 857		UVD, 0, mmUVD_MPC_SET_MUX),
 858		((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
 859		 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
 860		 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
 861
 862	vcn_v2_0_mc_resume_dpg_mode(adev, indirect);
 863
 864	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 865		UVD, 0, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
 866	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 867		UVD, 0, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
 868
 869	/* release VCPU reset to boot */
 870	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 871		UVD, 0, mmUVD_SOFT_RESET), 0, 0, indirect);
 872
 873	/* enable LMI MC and UMC channels */
 874	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 875		UVD, 0, mmUVD_LMI_CTRL2),
 876		0x1F << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT, 0, indirect);
 877
 878	/* enable master interrupt */
 879	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 880		UVD, 0, mmUVD_MASTINT_EN),
 881		UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
 882
 883	if (indirect)
 884		amdgpu_vcn_psp_update_sram(adev, 0, 0);
 
 
 885
 886	/* force RBC into idle state */
 887	rb_bufsz = order_base_2(ring->ring_size);
 888	tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
 889	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
 890	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
 891	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
 892	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
 893	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
 894
 895	/* Stall DPG before WPTR/RPTR reset */
 896	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS),
 897		UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
 898		~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
 899	fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;
 900
 901	/* set the write pointer delay */
 902	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL, 0);
 903
 904	/* set the wb address */
 905	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR,
 906		(upper_32_bits(ring->gpu_addr) >> 2));
 907
 908	/* program the RB_BASE for ring buffer */
 909	WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
 910		lower_32_bits(ring->gpu_addr));
 911	WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
 912		upper_32_bits(ring->gpu_addr));
 913
 914	/* Initialize the ring buffer's read and write pointers */
 915	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
 916
 917	WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2, 0);
 918
 919	ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
 920	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
 921		lower_32_bits(ring->wptr));
 922
 923	fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;
 924	/* Unstall DPG */
 925	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS),
 926		0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
 927	return 0;
 928}
 929
 930static int vcn_v2_0_start(struct amdgpu_device *adev)
 931{
 932	volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared.cpu_addr;
 933	struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec;
 934	uint32_t rb_bufsz, tmp;
 935	uint32_t lmi_swap_cntl;
 936	int i, j, r;
 937
 938	if (adev->pm.dpm_enabled)
 939		amdgpu_dpm_enable_uvd(adev, true);
 940
 941	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
 942		return vcn_v2_0_start_dpg_mode(adev, adev->vcn.indirect_sram);
 943
 944	vcn_v2_0_disable_static_power_gating(adev);
 945
 946	/* set uvd status busy */
 947	tmp = RREG32_SOC15(UVD, 0, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
 948	WREG32_SOC15(UVD, 0, mmUVD_STATUS, tmp);
 949
 950	/*SW clock gating */
 951	vcn_v2_0_disable_clock_gating(adev);
 952
 953	/* enable VCPU clock */
 954	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL),
 955		UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
 956
 957	/* disable master interrupt */
 958	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN), 0,
 959		~UVD_MASTINT_EN__VCPU_EN_MASK);
 960
 961	/* setup mmUVD_LMI_CTRL */
 962	tmp = RREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL);
 963	WREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL, tmp |
 964		UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK	|
 965		UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
 966		UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 967		UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
 968
 969	/* setup mmUVD_MPC_CNTL */
 970	tmp = RREG32_SOC15(UVD, 0, mmUVD_MPC_CNTL);
 971	tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
 972	tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
 973	WREG32_SOC15(VCN, 0, mmUVD_MPC_CNTL, tmp);
 974
 975	/* setup UVD_MPC_SET_MUXA0 */
 976	WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA0,
 977		((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
 978		(0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
 979		(0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
 980		(0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
 981
 982	/* setup UVD_MPC_SET_MUXB0 */
 983	WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB0,
 984		((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
 985		(0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
 986		(0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
 987		(0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
 988
 989	/* setup mmUVD_MPC_SET_MUX */
 990	WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUX,
 991		((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
 992		(0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
 993		(0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
 994
 995	vcn_v2_0_mc_resume(adev);
 996
 997	/* release VCPU reset to boot */
 998	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
 999		~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1000
1001	/* enable LMI MC and UMC channels */
1002	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
1003		~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1004
1005	tmp = RREG32_SOC15(VCN, 0, mmUVD_SOFT_RESET);
1006	tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1007	tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1008	WREG32_SOC15(VCN, 0, mmUVD_SOFT_RESET, tmp);
1009
1010	/* disable byte swapping */
1011	lmi_swap_cntl = 0;
1012#ifdef __BIG_ENDIAN
1013	/* swap (8 in 32) RB and IB */
1014	lmi_swap_cntl = 0xa;
1015#endif
1016	WREG32_SOC15(UVD, 0, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl);
1017
1018	for (i = 0; i < 10; ++i) {
1019		uint32_t status;
1020
1021		for (j = 0; j < 100; ++j) {
1022			status = RREG32_SOC15(UVD, 0, mmUVD_STATUS);
1023			if (status & 2)
1024				break;
1025			mdelay(10);
1026		}
1027		r = 0;
1028		if (status & 2)
1029			break;
1030
1031		DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n");
1032		WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1033			UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
1034			~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1035		mdelay(10);
1036		WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
1037			~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1038		mdelay(10);
1039		r = -1;
1040	}
1041
1042	if (r) {
1043		DRM_ERROR("VCN decode not responding, giving up!!!\n");
1044		return r;
1045	}
1046
1047	/* enable master interrupt */
1048	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
1049		UVD_MASTINT_EN__VCPU_EN_MASK,
1050		~UVD_MASTINT_EN__VCPU_EN_MASK);
1051
1052	/* clear the busy bit of VCN_STATUS */
1053	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0,
1054		~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1055
1056	WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_VMID, 0);
1057
1058	/* force RBC into idle state */
1059	rb_bufsz = order_base_2(ring->ring_size);
1060	tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1061	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1062	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1063	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1064	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1065	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
1066
1067	fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;
1068	/* program the RB_BASE for ring buffer */
1069	WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1070		lower_32_bits(ring->gpu_addr));
1071	WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1072		upper_32_bits(ring->gpu_addr));
1073
1074	/* Initialize the ring buffer's read and write pointers */
1075	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
1076
1077	ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
1078	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
1079			lower_32_bits(ring->wptr));
1080	fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;
1081
1082	fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET;
1083	ring = &adev->vcn.inst->ring_enc[0];
1084	WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1085	WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1086	WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
1087	WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1088	WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
1089	fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET;
1090
1091	fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET;
1092	ring = &adev->vcn.inst->ring_enc[1];
1093	WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1094	WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1095	WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1096	WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1097	WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
1098	fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET;
1099
1100	return 0;
1101}
1102
1103static int vcn_v2_0_stop_dpg_mode(struct amdgpu_device *adev)
1104{
1105	struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
1106	uint32_t tmp;
1107
1108	vcn_v2_0_pause_dpg_mode(adev, 0, &state);
1109	/* Wait for power status to be 1 */
1110	SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 1,
1111		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1112
1113	/* wait for read ptr to be equal to write ptr */
1114	tmp = RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
1115	SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1116
1117	tmp = RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
1118	SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);
1119
1120	tmp = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1121	SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);
1122
1123	SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 1,
1124		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1125
1126	/* disable dynamic power gating mode */
1127	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS), 0,
1128			~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1129
1130	return 0;
1131}
1132
1133static int vcn_v2_0_stop(struct amdgpu_device *adev)
1134{
1135	uint32_t tmp;
1136	int r;
1137
1138	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1139		r = vcn_v2_0_stop_dpg_mode(adev);
1140		if (r)
1141			return r;
1142		goto power_off;
1143	}
1144
1145	/* wait for uvd idle */
1146	r = SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1147	if (r)
1148		return r;
1149
1150	tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1151		UVD_LMI_STATUS__READ_CLEAN_MASK |
1152		UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1153		UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1154	r = SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_LMI_STATUS, tmp, tmp);
1155	if (r)
1156		return r;
1157
1158	/* stall UMC channel */
1159	tmp = RREG32_SOC15(VCN, 0, mmUVD_LMI_CTRL2);
1160	tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1161	WREG32_SOC15(VCN, 0, mmUVD_LMI_CTRL2, tmp);
1162
1163	tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
1164		UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1165	r = SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_LMI_STATUS, tmp, tmp);
1166	if (r)
1167		return r;
1168
1169	/* disable VCPU clock */
1170	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL), 0,
1171		~(UVD_VCPU_CNTL__CLK_EN_MASK));
1172
1173	/* reset LMI UMC */
1174	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1175		UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK,
1176		~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);
1177
1178	/* reset LMI */
1179	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1180		UVD_SOFT_RESET__LMI_SOFT_RESET_MASK,
1181		~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK);
1182
1183	/* reset VCPU */
1184	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1185		UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
1186		~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1187
1188	/* clear status */
1189	WREG32_SOC15(VCN, 0, mmUVD_STATUS, 0);
1190
1191	vcn_v2_0_enable_clock_gating(adev);
1192	vcn_v2_0_enable_static_power_gating(adev);
1193
1194power_off:
1195	if (adev->pm.dpm_enabled)
1196		amdgpu_dpm_enable_uvd(adev, false);
1197
1198	return 0;
1199}
1200
1201static int vcn_v2_0_pause_dpg_mode(struct amdgpu_device *adev,
1202				int inst_idx, struct dpg_pause_state *new_state)
1203{
1204	struct amdgpu_ring *ring;
1205	uint32_t reg_data = 0;
1206	int ret_code;
1207
1208	/* pause/unpause if state is changed */
1209	if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1210		DRM_DEBUG("dpg pause state changed %d -> %d",
1211			adev->vcn.inst[inst_idx].pause_state.fw_based,	new_state->fw_based);
1212		reg_data = RREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE) &
1213			(~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1214
1215		if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1216			ret_code = SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 0x1,
1217				UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1218
1219			if (!ret_code) {
1220				volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared.cpu_addr;
1221				/* pause DPG */
1222				reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1223				WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
1224
1225				/* wait for ACK */
1226				SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_DPG_PAUSE,
1227					   UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1228					   UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1229
1230				/* Stall DPG before WPTR/RPTR reset */
1231				WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS),
1232					   UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1233					   ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1234				/* Restore */
1235				fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET;
1236				ring = &adev->vcn.inst->ring_enc[0];
1237				ring->wptr = 0;
1238				WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
1239				WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1240				WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
1241				WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1242				WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1243				fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET;
1244
1245				fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET;
1246				ring = &adev->vcn.inst->ring_enc[1];
1247				ring->wptr = 0;
1248				WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1249				WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1250				WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
1251				WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1252				WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1253				fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET;
1254
1255				fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;
1256				WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
1257					   RREG32_SOC15(UVD, 0, mmUVD_SCRATCH2) & 0x7FFFFFFF);
1258				fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;
1259				/* Unstall DPG */
1260				WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS),
1261					   0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1262
1263				SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
1264					   UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON,
1265					   UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1266			}
1267		} else {
1268			/* unpause dpg, no need to wait */
1269			reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1270			WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
1271		}
1272		adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1273	}
1274
1275	return 0;
1276}
1277
1278static bool vcn_v2_0_is_idle(void *handle)
1279{
1280	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1281
1282	return (RREG32_SOC15(VCN, 0, mmUVD_STATUS) == UVD_STATUS__IDLE);
1283}
1284
1285static int vcn_v2_0_wait_for_idle(void *handle)
1286{
1287	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1288	int ret;
1289
1290	ret = SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_STATUS, UVD_STATUS__IDLE,
1291		UVD_STATUS__IDLE);
1292
1293	return ret;
1294}
1295
1296static int vcn_v2_0_set_clockgating_state(void *handle,
1297					  enum amd_clockgating_state state)
1298{
1299	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1300	bool enable = (state == AMD_CG_STATE_GATE);
1301
1302	if (amdgpu_sriov_vf(adev))
1303		return 0;
1304
1305	if (enable) {
1306		/* wait for STATUS to clear */
1307		if (!vcn_v2_0_is_idle(handle))
1308			return -EBUSY;
1309		vcn_v2_0_enable_clock_gating(adev);
1310	} else {
1311		/* disable HW gating and enable Sw gating */
1312		vcn_v2_0_disable_clock_gating(adev);
1313	}
1314	return 0;
1315}
1316
1317/**
1318 * vcn_v2_0_dec_ring_get_rptr - get read pointer
1319 *
1320 * @ring: amdgpu_ring pointer
1321 *
1322 * Returns the current hardware read pointer
1323 */
1324static uint64_t vcn_v2_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
1325{
1326	struct amdgpu_device *adev = ring->adev;
1327
1328	return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
1329}
1330
1331/**
1332 * vcn_v2_0_dec_ring_get_wptr - get write pointer
1333 *
1334 * @ring: amdgpu_ring pointer
1335 *
1336 * Returns the current hardware write pointer
1337 */
1338static uint64_t vcn_v2_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
1339{
1340	struct amdgpu_device *adev = ring->adev;
1341
1342	if (ring->use_doorbell)
1343		return *ring->wptr_cpu_addr;
1344	else
1345		return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR);
1346}
1347
1348/**
1349 * vcn_v2_0_dec_ring_set_wptr - set write pointer
1350 *
1351 * @ring: amdgpu_ring pointer
1352 *
1353 * Commits the write pointer to the hardware
1354 */
1355static void vcn_v2_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
1356{
1357	struct amdgpu_device *adev = ring->adev;
1358
1359	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1360		WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2,
1361			lower_32_bits(ring->wptr) | 0x80000000);
1362
1363	if (ring->use_doorbell) {
1364		*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1365		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1366	} else {
1367		WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1368	}
1369}
1370
1371/**
1372 * vcn_v2_0_dec_ring_insert_start - insert a start command
1373 *
1374 * @ring: amdgpu_ring pointer
1375 *
1376 * Write a start command to the ring.
1377 */
1378void vcn_v2_0_dec_ring_insert_start(struct amdgpu_ring *ring)
1379{
1380	struct amdgpu_device *adev = ring->adev;
1381
1382	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0));
1383	amdgpu_ring_write(ring, 0);
1384	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1385	amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_PACKET_START << 1));
1386}
1387
1388/**
1389 * vcn_v2_0_dec_ring_insert_end - insert a end command
1390 *
1391 * @ring: amdgpu_ring pointer
1392 *
1393 * Write a end command to the ring.
1394 */
1395void vcn_v2_0_dec_ring_insert_end(struct amdgpu_ring *ring)
1396{
1397	struct amdgpu_device *adev = ring->adev;
1398
1399	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1400	amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_PACKET_END << 1));
1401}
1402
1403/**
1404 * vcn_v2_0_dec_ring_insert_nop - insert a nop command
1405 *
1406 * @ring: amdgpu_ring pointer
1407 * @count: the number of NOP packets to insert
1408 *
1409 * Write a nop command to the ring.
1410 */
1411void vcn_v2_0_dec_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
1412{
1413	struct amdgpu_device *adev = ring->adev;
1414	int i;
1415
1416	WARN_ON(ring->wptr % 2 || count % 2);
1417
1418	for (i = 0; i < count / 2; i++) {
1419		amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.nop, 0));
1420		amdgpu_ring_write(ring, 0);
1421	}
1422}
1423
1424/**
1425 * vcn_v2_0_dec_ring_emit_fence - emit an fence & trap command
1426 *
1427 * @ring: amdgpu_ring pointer
1428 * @addr: address
1429 * @seq: sequence number
1430 * @flags: fence related flags
1431 *
1432 * Write a fence and a trap command to the ring.
1433 */
1434void vcn_v2_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
1435				unsigned flags)
1436{
1437	struct amdgpu_device *adev = ring->adev;
1438
1439	WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1440	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.context_id, 0));
1441	amdgpu_ring_write(ring, seq);
1442
1443	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0));
1444	amdgpu_ring_write(ring, addr & 0xffffffff);
1445
1446	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0));
1447	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xff);
1448
1449	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1450	amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_FENCE << 1));
1451
1452	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0));
1453	amdgpu_ring_write(ring, 0);
1454
1455	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0));
1456	amdgpu_ring_write(ring, 0);
1457
1458	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1459
1460	amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_TRAP << 1));
1461}
1462
1463/**
1464 * vcn_v2_0_dec_ring_emit_ib - execute indirect buffer
1465 *
1466 * @ring: amdgpu_ring pointer
1467 * @job: job to retrieve vmid from
1468 * @ib: indirect buffer to execute
1469 * @flags: unused
1470 *
1471 * Write ring commands to execute the indirect buffer
1472 */
1473void vcn_v2_0_dec_ring_emit_ib(struct amdgpu_ring *ring,
1474			       struct amdgpu_job *job,
1475			       struct amdgpu_ib *ib,
1476			       uint32_t flags)
1477{
1478	struct amdgpu_device *adev = ring->adev;
1479	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
1480
1481	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.ib_vmid, 0));
1482	amdgpu_ring_write(ring, vmid);
1483
1484	amdgpu_ring_write(ring,	PACKET0(adev->vcn.internal.ib_bar_low, 0));
1485	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1486	amdgpu_ring_write(ring,	PACKET0(adev->vcn.internal.ib_bar_high, 0));
1487	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1488	amdgpu_ring_write(ring,	PACKET0(adev->vcn.internal.ib_size, 0));
1489	amdgpu_ring_write(ring, ib->length_dw);
1490}
1491
1492void vcn_v2_0_dec_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1493				uint32_t val, uint32_t mask)
1494{
1495	struct amdgpu_device *adev = ring->adev;
1496
1497	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0));
1498	amdgpu_ring_write(ring, reg << 2);
1499
1500	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0));
1501	amdgpu_ring_write(ring, val);
1502
1503	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.gp_scratch8, 0));
1504	amdgpu_ring_write(ring, mask);
1505
1506	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1507
1508	amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_REG_READ_COND_WAIT << 1));
1509}
1510
1511void vcn_v2_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
1512				unsigned vmid, uint64_t pd_addr)
1513{
1514	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
1515	uint32_t data0, data1, mask;
1516
1517	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1518
1519	/* wait for register write */
1520	data0 = hub->ctx0_ptb_addr_lo32 + vmid * hub->ctx_addr_distance;
1521	data1 = lower_32_bits(pd_addr);
1522	mask = 0xffffffff;
1523	vcn_v2_0_dec_ring_emit_reg_wait(ring, data0, data1, mask);
1524}
1525
1526void vcn_v2_0_dec_ring_emit_wreg(struct amdgpu_ring *ring,
1527				uint32_t reg, uint32_t val)
1528{
1529	struct amdgpu_device *adev = ring->adev;
1530
1531	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0));
1532	amdgpu_ring_write(ring, reg << 2);
1533
1534	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0));
1535	amdgpu_ring_write(ring, val);
1536
1537	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1538
1539	amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_WRITE_REG << 1));
1540}
1541
1542/**
1543 * vcn_v2_0_enc_ring_get_rptr - get enc read pointer
1544 *
1545 * @ring: amdgpu_ring pointer
1546 *
1547 * Returns the current hardware enc read pointer
1548 */
1549static uint64_t vcn_v2_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
1550{
1551	struct amdgpu_device *adev = ring->adev;
1552
1553	if (ring == &adev->vcn.inst->ring_enc[0])
1554		return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR);
1555	else
1556		return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2);
1557}
1558
1559 /**
1560 * vcn_v2_0_enc_ring_get_wptr - get enc write pointer
1561 *
1562 * @ring: amdgpu_ring pointer
1563 *
1564 * Returns the current hardware enc write pointer
1565 */
1566static uint64_t vcn_v2_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
1567{
1568	struct amdgpu_device *adev = ring->adev;
1569
1570	if (ring == &adev->vcn.inst->ring_enc[0]) {
1571		if (ring->use_doorbell)
1572			return *ring->wptr_cpu_addr;
1573		else
1574			return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
1575	} else {
1576		if (ring->use_doorbell)
1577			return *ring->wptr_cpu_addr;
1578		else
1579			return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
1580	}
1581}
1582
1583 /**
1584 * vcn_v2_0_enc_ring_set_wptr - set enc write pointer
1585 *
1586 * @ring: amdgpu_ring pointer
1587 *
1588 * Commits the enc write pointer to the hardware
1589 */
1590static void vcn_v2_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
1591{
1592	struct amdgpu_device *adev = ring->adev;
1593
1594	if (ring == &adev->vcn.inst->ring_enc[0]) {
1595		if (ring->use_doorbell) {
1596			*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1597			WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1598		} else {
1599			WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1600		}
1601	} else {
1602		if (ring->use_doorbell) {
1603			*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1604			WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1605		} else {
1606			WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1607		}
1608	}
1609}
1610
1611/**
1612 * vcn_v2_0_enc_ring_emit_fence - emit an enc fence & trap command
1613 *
1614 * @ring: amdgpu_ring pointer
1615 * @addr: address
1616 * @seq: sequence number
1617 * @flags: fence related flags
1618 *
1619 * Write enc a fence and a trap command to the ring.
1620 */
1621void vcn_v2_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
1622				u64 seq, unsigned flags)
1623{
1624	WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1625
1626	amdgpu_ring_write(ring, VCN_ENC_CMD_FENCE);
1627	amdgpu_ring_write(ring, addr);
1628	amdgpu_ring_write(ring, upper_32_bits(addr));
1629	amdgpu_ring_write(ring, seq);
1630	amdgpu_ring_write(ring, VCN_ENC_CMD_TRAP);
1631}
1632
1633void vcn_v2_0_enc_ring_insert_end(struct amdgpu_ring *ring)
1634{
1635	amdgpu_ring_write(ring, VCN_ENC_CMD_END);
1636}
1637
1638/**
1639 * vcn_v2_0_enc_ring_emit_ib - enc execute indirect buffer
1640 *
1641 * @ring: amdgpu_ring pointer
1642 * @job: job to retrive vmid from
1643 * @ib: indirect buffer to execute
1644 * @flags: unused
1645 *
1646 * Write enc ring commands to execute the indirect buffer
1647 */
1648void vcn_v2_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
1649			       struct amdgpu_job *job,
1650			       struct amdgpu_ib *ib,
1651			       uint32_t flags)
1652{
1653	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
1654
1655	amdgpu_ring_write(ring, VCN_ENC_CMD_IB);
1656	amdgpu_ring_write(ring, vmid);
1657	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1658	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1659	amdgpu_ring_write(ring, ib->length_dw);
1660}
1661
1662void vcn_v2_0_enc_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1663				uint32_t val, uint32_t mask)
1664{
1665	amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT);
1666	amdgpu_ring_write(ring, reg << 2);
1667	amdgpu_ring_write(ring, mask);
1668	amdgpu_ring_write(ring, val);
1669}
1670
1671void vcn_v2_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
1672				unsigned int vmid, uint64_t pd_addr)
1673{
1674	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
1675
1676	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1677
1678	/* wait for reg writes */
1679	vcn_v2_0_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 +
1680					vmid * hub->ctx_addr_distance,
1681					lower_32_bits(pd_addr), 0xffffffff);
1682}
1683
1684void vcn_v2_0_enc_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val)
1685{
1686	amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE);
1687	amdgpu_ring_write(ring,	reg << 2);
1688	amdgpu_ring_write(ring, val);
1689}
1690
1691static int vcn_v2_0_set_interrupt_state(struct amdgpu_device *adev,
1692					struct amdgpu_irq_src *source,
1693					unsigned type,
1694					enum amdgpu_interrupt_state state)
1695{
1696	return 0;
1697}
1698
1699static int vcn_v2_0_process_interrupt(struct amdgpu_device *adev,
1700				      struct amdgpu_irq_src *source,
1701				      struct amdgpu_iv_entry *entry)
1702{
1703	DRM_DEBUG("IH: VCN TRAP\n");
1704
1705	switch (entry->src_id) {
1706	case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
1707		amdgpu_fence_process(&adev->vcn.inst->ring_dec);
1708		break;
1709	case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
1710		amdgpu_fence_process(&adev->vcn.inst->ring_enc[0]);
1711		break;
1712	case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
1713		amdgpu_fence_process(&adev->vcn.inst->ring_enc[1]);
1714		break;
1715	default:
1716		DRM_ERROR("Unhandled interrupt: %d %d\n",
1717			  entry->src_id, entry->src_data[0]);
1718		break;
1719	}
1720
1721	return 0;
1722}
1723
1724int vcn_v2_0_dec_ring_test_ring(struct amdgpu_ring *ring)
1725{
1726	struct amdgpu_device *adev = ring->adev;
1727	uint32_t tmp = 0;
1728	unsigned i;
1729	int r;
1730
1731	if (amdgpu_sriov_vf(adev))
1732		return 0;
1733
1734	WREG32(adev->vcn.inst[ring->me].external.scratch9, 0xCAFEDEAD);
1735	r = amdgpu_ring_alloc(ring, 4);
1736	if (r)
1737		return r;
1738	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1739	amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_PACKET_START << 1));
1740	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.scratch9, 0));
1741	amdgpu_ring_write(ring, 0xDEADBEEF);
1742	amdgpu_ring_commit(ring);
1743	for (i = 0; i < adev->usec_timeout; i++) {
1744		tmp = RREG32(adev->vcn.inst[ring->me].external.scratch9);
1745		if (tmp == 0xDEADBEEF)
1746			break;
1747		udelay(1);
1748	}
1749
1750	if (i >= adev->usec_timeout)
1751		r = -ETIMEDOUT;
1752
1753	return r;
1754}
1755
1756
1757static int vcn_v2_0_set_powergating_state(void *handle,
1758					  enum amd_powergating_state state)
1759{
1760	/* This doesn't actually powergate the VCN block.
1761	 * That's done in the dpm code via the SMC.  This
1762	 * just re-inits the block as necessary.  The actual
1763	 * gating still happens in the dpm code.  We should
1764	 * revisit this when there is a cleaner line between
1765	 * the smc and the hw blocks
1766	 */
1767	int ret;
1768	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1769
1770	if (amdgpu_sriov_vf(adev)) {
1771		adev->vcn.cur_state = AMD_PG_STATE_UNGATE;
1772		return 0;
1773	}
1774
1775	if (state == adev->vcn.cur_state)
1776		return 0;
1777
1778	if (state == AMD_PG_STATE_GATE)
1779		ret = vcn_v2_0_stop(adev);
1780	else
1781		ret = vcn_v2_0_start(adev);
1782
1783	if (!ret)
1784		adev->vcn.cur_state = state;
1785	return ret;
1786}
1787
1788static int vcn_v2_0_start_mmsch(struct amdgpu_device *adev,
1789				struct amdgpu_mm_table *table)
1790{
1791	uint32_t data = 0, loop;
1792	uint64_t addr = table->gpu_addr;
1793	struct mmsch_v2_0_init_header *header;
1794	uint32_t size;
1795	int i;
1796
1797	header = (struct mmsch_v2_0_init_header *)table->cpu_addr;
1798	size = header->header_size + header->vcn_table_size;
1799
1800	/* 1, write to vce_mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
1801	 * of memory descriptor location
1802	 */
1803	WREG32_SOC15(UVD, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(addr));
1804	WREG32_SOC15(UVD, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(addr));
1805
1806	/* 2, update vmid of descriptor */
1807	data = RREG32_SOC15(UVD, 0, mmMMSCH_VF_VMID);
1808	data &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1809	/* use domain0 for MM scheduler */
1810	data |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1811	WREG32_SOC15(UVD, 0, mmMMSCH_VF_VMID, data);
1812
1813	/* 3, notify mmsch about the size of this descriptor */
1814	WREG32_SOC15(UVD, 0, mmMMSCH_VF_CTX_SIZE, size);
1815
1816	/* 4, set resp to zero */
1817	WREG32_SOC15(UVD, 0, mmMMSCH_VF_MAILBOX_RESP, 0);
1818
1819	adev->vcn.inst->ring_dec.wptr = 0;
1820	adev->vcn.inst->ring_dec.wptr_old = 0;
1821	vcn_v2_0_dec_ring_set_wptr(&adev->vcn.inst->ring_dec);
1822
1823	for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
1824		adev->vcn.inst->ring_enc[i].wptr = 0;
1825		adev->vcn.inst->ring_enc[i].wptr_old = 0;
1826		vcn_v2_0_enc_ring_set_wptr(&adev->vcn.inst->ring_enc[i]);
1827	}
1828
1829	/* 5, kick off the initialization and wait until
1830	 * VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero
1831	 */
1832	WREG32_SOC15(UVD, 0, mmMMSCH_VF_MAILBOX_HOST, 0x10000001);
1833
1834	data = RREG32_SOC15(UVD, 0, mmMMSCH_VF_MAILBOX_RESP);
1835	loop = 1000;
1836	while ((data & 0x10000002) != 0x10000002) {
1837		udelay(10);
1838		data = RREG32_SOC15(UVD, 0, mmMMSCH_VF_MAILBOX_RESP);
1839		loop--;
1840		if (!loop)
1841			break;
1842	}
1843
1844	if (!loop) {
1845		DRM_ERROR("failed to init MMSCH, " \
1846			"mmMMSCH_VF_MAILBOX_RESP = 0x%08x\n", data);
1847		return -EBUSY;
1848	}
1849
1850	return 0;
1851}
1852
1853static int vcn_v2_0_start_sriov(struct amdgpu_device *adev)
1854{
1855	int r;
1856	uint32_t tmp;
1857	struct amdgpu_ring *ring;
1858	uint32_t offset, size;
1859	uint32_t table_size = 0;
1860	struct mmsch_v2_0_cmd_direct_write direct_wt = { {0} };
1861	struct mmsch_v2_0_cmd_direct_read_modify_write direct_rd_mod_wt = { {0} };
1862	struct mmsch_v2_0_cmd_end end = { {0} };
1863	struct mmsch_v2_0_init_header *header;
1864	uint32_t *init_table = adev->virt.mm_table.cpu_addr;
1865	uint8_t i = 0;
1866
1867	header = (struct mmsch_v2_0_init_header *)init_table;
1868	direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE;
1869	direct_rd_mod_wt.cmd_header.command_type =
1870		MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1871	end.cmd_header.command_type = MMSCH_COMMAND__END;
1872
1873	if (header->vcn_table_offset == 0 && header->vcn_table_size == 0) {
1874		header->version = MMSCH_VERSION;
1875		header->header_size = sizeof(struct mmsch_v2_0_init_header) >> 2;
1876
1877		header->vcn_table_offset = header->header_size;
1878
1879		init_table += header->vcn_table_offset;
1880
1881		size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[0]->size + 4);
1882
1883		MMSCH_V2_0_INSERT_DIRECT_RD_MOD_WT(
1884			SOC15_REG_OFFSET(UVD, i, mmUVD_STATUS),
1885			0xFFFFFFFF, 0x00000004);
1886
1887		/* mc resume*/
1888		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1889			MMSCH_V2_0_INSERT_DIRECT_WT(
1890				SOC15_REG_OFFSET(UVD, i,
1891					mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1892				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo);
1893			MMSCH_V2_0_INSERT_DIRECT_WT(
1894				SOC15_REG_OFFSET(UVD, i,
1895					mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1896				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi);
1897			offset = 0;
1898		} else {
1899			MMSCH_V2_0_INSERT_DIRECT_WT(
1900				SOC15_REG_OFFSET(UVD, i,
1901					mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1902				lower_32_bits(adev->vcn.inst->gpu_addr));
1903			MMSCH_V2_0_INSERT_DIRECT_WT(
1904				SOC15_REG_OFFSET(UVD, i,
1905					mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1906				upper_32_bits(adev->vcn.inst->gpu_addr));
1907			offset = size;
1908		}
1909
1910		MMSCH_V2_0_INSERT_DIRECT_WT(
1911			SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET0),
1912			0);
1913		MMSCH_V2_0_INSERT_DIRECT_WT(
1914			SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE0),
1915			size);
1916
1917		MMSCH_V2_0_INSERT_DIRECT_WT(
1918			SOC15_REG_OFFSET(UVD, i,
1919				mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1920			lower_32_bits(adev->vcn.inst->gpu_addr + offset));
1921		MMSCH_V2_0_INSERT_DIRECT_WT(
1922			SOC15_REG_OFFSET(UVD, i,
1923				mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1924			upper_32_bits(adev->vcn.inst->gpu_addr + offset));
1925		MMSCH_V2_0_INSERT_DIRECT_WT(
1926			SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET1),
1927			0);
1928		MMSCH_V2_0_INSERT_DIRECT_WT(
1929			SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE1),
1930			AMDGPU_VCN_STACK_SIZE);
1931
1932		MMSCH_V2_0_INSERT_DIRECT_WT(
1933			SOC15_REG_OFFSET(UVD, i,
1934				mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1935			lower_32_bits(adev->vcn.inst->gpu_addr + offset +
1936				AMDGPU_VCN_STACK_SIZE));
1937		MMSCH_V2_0_INSERT_DIRECT_WT(
1938			SOC15_REG_OFFSET(UVD, i,
1939				mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1940			upper_32_bits(adev->vcn.inst->gpu_addr + offset +
1941				AMDGPU_VCN_STACK_SIZE));
1942		MMSCH_V2_0_INSERT_DIRECT_WT(
1943			SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET2),
1944			0);
1945		MMSCH_V2_0_INSERT_DIRECT_WT(
1946			SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE2),
1947			AMDGPU_VCN_CONTEXT_SIZE);
1948
1949		for (r = 0; r < adev->vcn.num_enc_rings; ++r) {
1950			ring = &adev->vcn.inst->ring_enc[r];
1951			ring->wptr = 0;
1952			MMSCH_V2_0_INSERT_DIRECT_WT(
1953				SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_LO),
1954				lower_32_bits(ring->gpu_addr));
1955			MMSCH_V2_0_INSERT_DIRECT_WT(
1956				SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_HI),
1957				upper_32_bits(ring->gpu_addr));
1958			MMSCH_V2_0_INSERT_DIRECT_WT(
1959				SOC15_REG_OFFSET(UVD, i, mmUVD_RB_SIZE),
1960				ring->ring_size / 4);
1961		}
1962
1963		ring = &adev->vcn.inst->ring_dec;
1964		ring->wptr = 0;
1965		MMSCH_V2_0_INSERT_DIRECT_WT(
1966			SOC15_REG_OFFSET(UVD, i,
1967				mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
1968			lower_32_bits(ring->gpu_addr));
1969		MMSCH_V2_0_INSERT_DIRECT_WT(
1970			SOC15_REG_OFFSET(UVD, i,
1971				mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
1972			upper_32_bits(ring->gpu_addr));
1973		/* force RBC into idle state */
1974		tmp = order_base_2(ring->ring_size);
1975		tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, tmp);
1976		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1977		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1978		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1979		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1980		MMSCH_V2_0_INSERT_DIRECT_WT(
1981			SOC15_REG_OFFSET(UVD, i, mmUVD_RBC_RB_CNTL), tmp);
1982
1983		/* add end packet */
1984		tmp = sizeof(struct mmsch_v2_0_cmd_end);
1985		memcpy((void *)init_table, &end, tmp);
1986		table_size += (tmp / 4);
1987		header->vcn_table_size = table_size;
1988
1989	}
1990	return vcn_v2_0_start_mmsch(adev, &adev->virt.mm_table);
1991}
1992
1993static const struct amd_ip_funcs vcn_v2_0_ip_funcs = {
1994	.name = "vcn_v2_0",
1995	.early_init = vcn_v2_0_early_init,
1996	.late_init = NULL,
1997	.sw_init = vcn_v2_0_sw_init,
1998	.sw_fini = vcn_v2_0_sw_fini,
1999	.hw_init = vcn_v2_0_hw_init,
2000	.hw_fini = vcn_v2_0_hw_fini,
2001	.suspend = vcn_v2_0_suspend,
2002	.resume = vcn_v2_0_resume,
2003	.is_idle = vcn_v2_0_is_idle,
2004	.wait_for_idle = vcn_v2_0_wait_for_idle,
2005	.check_soft_reset = NULL,
2006	.pre_soft_reset = NULL,
2007	.soft_reset = NULL,
2008	.post_soft_reset = NULL,
2009	.set_clockgating_state = vcn_v2_0_set_clockgating_state,
2010	.set_powergating_state = vcn_v2_0_set_powergating_state,
2011};
2012
2013static const struct amdgpu_ring_funcs vcn_v2_0_dec_ring_vm_funcs = {
2014	.type = AMDGPU_RING_TYPE_VCN_DEC,
2015	.align_mask = 0xf,
2016	.secure_submission_supported = true,
 
2017	.get_rptr = vcn_v2_0_dec_ring_get_rptr,
2018	.get_wptr = vcn_v2_0_dec_ring_get_wptr,
2019	.set_wptr = vcn_v2_0_dec_ring_set_wptr,
2020	.emit_frame_size =
2021		SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
2022		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
2023		8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
2024		14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
2025		6,
2026	.emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
2027	.emit_ib = vcn_v2_0_dec_ring_emit_ib,
2028	.emit_fence = vcn_v2_0_dec_ring_emit_fence,
2029	.emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
2030	.test_ring = vcn_v2_0_dec_ring_test_ring,
2031	.test_ib = amdgpu_vcn_dec_ring_test_ib,
2032	.insert_nop = vcn_v2_0_dec_ring_insert_nop,
2033	.insert_start = vcn_v2_0_dec_ring_insert_start,
2034	.insert_end = vcn_v2_0_dec_ring_insert_end,
2035	.pad_ib = amdgpu_ring_generic_pad_ib,
2036	.begin_use = amdgpu_vcn_ring_begin_use,
2037	.end_use = amdgpu_vcn_ring_end_use,
2038	.emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
2039	.emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
2040	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2041};
2042
2043static const struct amdgpu_ring_funcs vcn_v2_0_enc_ring_vm_funcs = {
2044	.type = AMDGPU_RING_TYPE_VCN_ENC,
2045	.align_mask = 0x3f,
2046	.nop = VCN_ENC_CMD_NO_OP,
 
2047	.get_rptr = vcn_v2_0_enc_ring_get_rptr,
2048	.get_wptr = vcn_v2_0_enc_ring_get_wptr,
2049	.set_wptr = vcn_v2_0_enc_ring_set_wptr,
2050	.emit_frame_size =
2051		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2052		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
2053		4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
2054		5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
2055		1, /* vcn_v2_0_enc_ring_insert_end */
2056	.emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
2057	.emit_ib = vcn_v2_0_enc_ring_emit_ib,
2058	.emit_fence = vcn_v2_0_enc_ring_emit_fence,
2059	.emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
2060	.test_ring = amdgpu_vcn_enc_ring_test_ring,
2061	.test_ib = amdgpu_vcn_enc_ring_test_ib,
2062	.insert_nop = amdgpu_ring_insert_nop,
2063	.insert_end = vcn_v2_0_enc_ring_insert_end,
2064	.pad_ib = amdgpu_ring_generic_pad_ib,
2065	.begin_use = amdgpu_vcn_ring_begin_use,
2066	.end_use = amdgpu_vcn_ring_end_use,
2067	.emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
2068	.emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
2069	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2070};
2071
2072static void vcn_v2_0_set_dec_ring_funcs(struct amdgpu_device *adev)
2073{
2074	adev->vcn.inst->ring_dec.funcs = &vcn_v2_0_dec_ring_vm_funcs;
2075	DRM_INFO("VCN decode is enabled in VM mode\n");
2076}
2077
2078static void vcn_v2_0_set_enc_ring_funcs(struct amdgpu_device *adev)
2079{
2080	int i;
2081
2082	for (i = 0; i < adev->vcn.num_enc_rings; ++i)
2083		adev->vcn.inst->ring_enc[i].funcs = &vcn_v2_0_enc_ring_vm_funcs;
2084
2085	DRM_INFO("VCN encode is enabled in VM mode\n");
2086}
2087
2088static const struct amdgpu_irq_src_funcs vcn_v2_0_irq_funcs = {
2089	.set = vcn_v2_0_set_interrupt_state,
2090	.process = vcn_v2_0_process_interrupt,
2091};
2092
2093static void vcn_v2_0_set_irq_funcs(struct amdgpu_device *adev)
2094{
2095	adev->vcn.inst->irq.num_types = adev->vcn.num_enc_rings + 1;
2096	adev->vcn.inst->irq.funcs = &vcn_v2_0_irq_funcs;
2097}
2098
2099const struct amdgpu_ip_block_version vcn_v2_0_ip_block =
2100{
2101		.type = AMD_IP_BLOCK_TYPE_VCN,
2102		.major = 2,
2103		.minor = 0,
2104		.rev = 0,
2105		.funcs = &vcn_v2_0_ip_funcs,
2106};