1/*
   2 * Copyright 2014 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Christian König <christian.koenig@amd.com>
  23 */
  24
  25#include <linux/firmware.h>
  26
  27#include "amdgpu.h"
  28#include "amdgpu_uvd.h"
  29#include "vid.h"
  30#include "uvd/uvd_6_0_d.h"
  31#include "uvd/uvd_6_0_sh_mask.h"
  32#include "oss/oss_2_0_d.h"
  33#include "oss/oss_2_0_sh_mask.h"
  34#include "smu/smu_7_1_3_d.h"
  35#include "smu/smu_7_1_3_sh_mask.h"
  36#include "bif/bif_5_1_d.h"
  37#include "gmc/gmc_8_1_d.h"
  38#include "vi.h"
  39#include "ivsrcid/ivsrcid_vislands30.h"
  40
  41/* Polaris10/11/12 firmware version */
  42#define FW_1_130_16 ((1 << 24) | (130 << 16) | (16 << 8))
  43
  44static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev);
  45static void uvd_v6_0_set_enc_ring_funcs(struct amdgpu_device *adev);
  46
  47static void uvd_v6_0_set_irq_funcs(struct amdgpu_device *adev);
  48static int uvd_v6_0_start(struct amdgpu_device *adev);
  49static void uvd_v6_0_stop(struct amdgpu_device *adev);
  50static void uvd_v6_0_set_sw_clock_gating(struct amdgpu_device *adev);
  51static int uvd_v6_0_set_clockgating_state(void *handle,
  52					  enum amd_clockgating_state state);
  53static void uvd_v6_0_enable_mgcg(struct amdgpu_device *adev,
  54				 bool enable);
  55
  56/**
  57* uvd_v6_0_enc_support - get encode support status
  58*
  59* @adev: amdgpu_device pointer
  60*
  61* Returns the current hardware encode support status
  62*/
  63static inline bool uvd_v6_0_enc_support(struct amdgpu_device *adev)
  64{
  65	return ((adev->asic_type >= CHIP_POLARIS10) &&
  66			(adev->asic_type <= CHIP_VEGAM) &&
  67			(!adev->uvd.fw_version || adev->uvd.fw_version >= FW_1_130_16));
  68}
  69
  70/**
  71 * uvd_v6_0_ring_get_rptr - get read pointer
  72 *
  73 * @ring: amdgpu_ring pointer
  74 *
  75 * Returns the current hardware read pointer
  76 */
  77static uint64_t uvd_v6_0_ring_get_rptr(struct amdgpu_ring *ring)
  78{
  79	struct amdgpu_device *adev = ring->adev;
  80
  81	return RREG32(mmUVD_RBC_RB_RPTR);
  82}
  83
  84/**
  85 * uvd_v6_0_enc_ring_get_rptr - get enc read pointer
  86 *
  87 * @ring: amdgpu_ring pointer
  88 *
  89 * Returns the current hardware enc read pointer
  90 */
  91static uint64_t uvd_v6_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
  92{
  93	struct amdgpu_device *adev = ring->adev;
  94
  95	if (ring == &adev->uvd.inst->ring_enc[0])
  96		return RREG32(mmUVD_RB_RPTR);
  97	else
  98		return RREG32(mmUVD_RB_RPTR2);
  99}
 100/**
 101 * uvd_v6_0_ring_get_wptr - get write pointer
 102 *
 103 * @ring: amdgpu_ring pointer
 104 *
 105 * Returns the current hardware write pointer
 106 */
 107static uint64_t uvd_v6_0_ring_get_wptr(struct amdgpu_ring *ring)
 108{
 109	struct amdgpu_device *adev = ring->adev;
 110
 111	return RREG32(mmUVD_RBC_RB_WPTR);
 112}
 113
 114/**
 115 * uvd_v6_0_enc_ring_get_wptr - get enc write pointer
 116 *
 117 * @ring: amdgpu_ring pointer
 118 *
 119 * Returns the current hardware enc write pointer
 120 */
 121static uint64_t uvd_v6_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
 122{
 123	struct amdgpu_device *adev = ring->adev;
 124
 125	if (ring == &adev->uvd.inst->ring_enc[0])
 126		return RREG32(mmUVD_RB_WPTR);
 127	else
 128		return RREG32(mmUVD_RB_WPTR2);
 129}
 130
 131/**
 132 * uvd_v6_0_ring_set_wptr - set write pointer
 133 *
 134 * @ring: amdgpu_ring pointer
 135 *
 136 * Commits the write pointer to the hardware
 137 */
 138static void uvd_v6_0_ring_set_wptr(struct amdgpu_ring *ring)
 139{
 140	struct amdgpu_device *adev = ring->adev;
 141
 142	WREG32(mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
 143}
 144
 145/**
 146 * uvd_v6_0_enc_ring_set_wptr - set enc write pointer
 147 *
 148 * @ring: amdgpu_ring pointer
 149 *
 150 * Commits the enc write pointer to the hardware
 151 */
 152static void uvd_v6_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
 153{
 154	struct amdgpu_device *adev = ring->adev;
 155
 156	if (ring == &adev->uvd.inst->ring_enc[0])
 157		WREG32(mmUVD_RB_WPTR,
 158			lower_32_bits(ring->wptr));
 159	else
 160		WREG32(mmUVD_RB_WPTR2,
 161			lower_32_bits(ring->wptr));
 162}
 163
 164/**
 165 * uvd_v6_0_enc_ring_test_ring - test if UVD ENC ring is working
 166 *
 167 * @ring: the engine to test on
 168 *
 169 */
 170static int uvd_v6_0_enc_ring_test_ring(struct amdgpu_ring *ring)
 171{
 172	struct amdgpu_device *adev = ring->adev;
 173	uint32_t rptr;
 174	unsigned i;
 175	int r;
 176
 177	r = amdgpu_ring_alloc(ring, 16);
 178	if (r)
 179		return r;
 180
 181	rptr = amdgpu_ring_get_rptr(ring);
 182
 183	amdgpu_ring_write(ring, HEVC_ENC_CMD_END);
 184	amdgpu_ring_commit(ring);
 185
 186	for (i = 0; i < adev->usec_timeout; i++) {
 187		if (amdgpu_ring_get_rptr(ring) != rptr)
 188			break;
 189		udelay(1);
 190	}
 191
 192	if (i >= adev->usec_timeout)
 193		r = -ETIMEDOUT;
 194
 195	return r;
 196}
 197
 198/**
 199 * uvd_v6_0_enc_get_create_msg - generate a UVD ENC create msg
 200 *
 201 * @adev: amdgpu_device pointer
 202 * @ring: ring we should submit the msg to
 203 * @handle: session handle to use
 
 204 * @fence: optional fence to return
 205 *
 206 * Open up a stream for HW test
 207 */
 208static int uvd_v6_0_enc_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
 209				       struct amdgpu_bo *bo,
 210				       struct dma_fence **fence)
 211{
 212	const unsigned ib_size_dw = 16;
 213	struct amdgpu_job *job;
 214	struct amdgpu_ib *ib;
 215	struct dma_fence *f = NULL;
 216	uint64_t addr;
 217	int i, r;
 218
 219	r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4,
 220					AMDGPU_IB_POOL_DIRECT, &job);
 221	if (r)
 222		return r;
 223
 224	ib = &job->ibs[0];
 225	addr = amdgpu_bo_gpu_offset(bo);
 226
 227	ib->length_dw = 0;
 228	ib->ptr[ib->length_dw++] = 0x00000018;
 229	ib->ptr[ib->length_dw++] = 0x00000001; /* session info */
 230	ib->ptr[ib->length_dw++] = handle;
 231	ib->ptr[ib->length_dw++] = 0x00010000;
 232	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
 233	ib->ptr[ib->length_dw++] = addr;
 234
 235	ib->ptr[ib->length_dw++] = 0x00000014;
 236	ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
 237	ib->ptr[ib->length_dw++] = 0x0000001c;
 238	ib->ptr[ib->length_dw++] = 0x00000001;
 239	ib->ptr[ib->length_dw++] = 0x00000000;
 240
 241	ib->ptr[ib->length_dw++] = 0x00000008;
 242	ib->ptr[ib->length_dw++] = 0x08000001; /* op initialize */
 243
 244	for (i = ib->length_dw; i < ib_size_dw; ++i)
 245		ib->ptr[i] = 0x0;
 246
 247	r = amdgpu_job_submit_direct(job, ring, &f);
 248	if (r)
 249		goto err;
 250
 251	if (fence)
 252		*fence = dma_fence_get(f);
 253	dma_fence_put(f);
 254	return 0;
 255
 256err:
 257	amdgpu_job_free(job);
 258	return r;
 259}
 260
 261/**
 262 * uvd_v6_0_enc_get_destroy_msg - generate a UVD ENC destroy msg
 263 *
 264 * @adev: amdgpu_device pointer
 265 * @ring: ring we should submit the msg to
 266 * @handle: session handle to use
 
 267 * @fence: optional fence to return
 268 *
 269 * Close up a stream for HW test or if userspace failed to do so
 270 */
 271static int uvd_v6_0_enc_get_destroy_msg(struct amdgpu_ring *ring,
 272					uint32_t handle,
 273					struct amdgpu_bo *bo,
 274					struct dma_fence **fence)
 275{
 276	const unsigned ib_size_dw = 16;
 277	struct amdgpu_job *job;
 278	struct amdgpu_ib *ib;
 279	struct dma_fence *f = NULL;
 280	uint64_t addr;
 281	int i, r;
 282
 283	r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4,
 284					AMDGPU_IB_POOL_DIRECT, &job);
 285	if (r)
 286		return r;
 287
 288	ib = &job->ibs[0];
 289	addr = amdgpu_bo_gpu_offset(bo);
 290
 291	ib->length_dw = 0;
 292	ib->ptr[ib->length_dw++] = 0x00000018;
 293	ib->ptr[ib->length_dw++] = 0x00000001; /* session info */
 294	ib->ptr[ib->length_dw++] = handle;
 295	ib->ptr[ib->length_dw++] = 0x00010000;
 296	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
 297	ib->ptr[ib->length_dw++] = addr;
 298
 299	ib->ptr[ib->length_dw++] = 0x00000014;
 300	ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
 301	ib->ptr[ib->length_dw++] = 0x0000001c;
 302	ib->ptr[ib->length_dw++] = 0x00000001;
 303	ib->ptr[ib->length_dw++] = 0x00000000;
 304
 305	ib->ptr[ib->length_dw++] = 0x00000008;
 306	ib->ptr[ib->length_dw++] = 0x08000002; /* op close session */
 307
 308	for (i = ib->length_dw; i < ib_size_dw; ++i)
 309		ib->ptr[i] = 0x0;
 310
 311	r = amdgpu_job_submit_direct(job, ring, &f);
 312	if (r)
 313		goto err;
 314
 315	if (fence)
 316		*fence = dma_fence_get(f);
 317	dma_fence_put(f);
 318	return 0;
 319
 320err:
 321	amdgpu_job_free(job);
 322	return r;
 323}
 324
 325/**
 326 * uvd_v6_0_enc_ring_test_ib - test if UVD ENC IBs are working
 327 *
 328 * @ring: the engine to test on
 
 329 *
 330 */
 331static int uvd_v6_0_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 332{
 333	struct dma_fence *fence = NULL;
 334	struct amdgpu_bo *bo = NULL;
 335	long r;
 336
 337	r = amdgpu_bo_create_reserved(ring->adev, 128 * 1024, PAGE_SIZE,
 338				      AMDGPU_GEM_DOMAIN_VRAM,
 339				      &bo, NULL, NULL);
 340	if (r)
 341		return r;
 342
 343	r = uvd_v6_0_enc_get_create_msg(ring, 1, bo, NULL);
 344	if (r)
 345		goto error;
 346
 347	r = uvd_v6_0_enc_get_destroy_msg(ring, 1, bo, &fence);
 348	if (r)
 349		goto error;
 350
 351	r = dma_fence_wait_timeout(fence, false, timeout);
 352	if (r == 0)
 353		r = -ETIMEDOUT;
 354	else if (r > 0)
 355		r = 0;
 356
 357error:
 358	dma_fence_put(fence);
 
 359	amdgpu_bo_unreserve(bo);
 360	amdgpu_bo_unref(&bo);
 361	return r;
 362}
 363
 364static int uvd_v6_0_early_init(void *handle)
 365{
 366	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 367	adev->uvd.num_uvd_inst = 1;
 368
 369	if (!(adev->flags & AMD_IS_APU) &&
 370	    (RREG32_SMC(ixCC_HARVEST_FUSES) & CC_HARVEST_FUSES__UVD_DISABLE_MASK))
 371		return -ENOENT;
 372
 373	uvd_v6_0_set_ring_funcs(adev);
 374
 375	if (uvd_v6_0_enc_support(adev)) {
 376		adev->uvd.num_enc_rings = 2;
 377		uvd_v6_0_set_enc_ring_funcs(adev);
 378	}
 379
 380	uvd_v6_0_set_irq_funcs(adev);
 381
 382	return 0;
 383}
 384
 385static int uvd_v6_0_sw_init(void *handle)
 386{
 387	struct amdgpu_ring *ring;
 388	int i, r;
 389	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 390
 391	/* UVD TRAP */
 392	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_UVD_SYSTEM_MESSAGE, &adev->uvd.inst->irq);
 393	if (r)
 394		return r;
 395
 396	/* UVD ENC TRAP */
 397	if (uvd_v6_0_enc_support(adev)) {
 398		for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
 399			r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, i + VISLANDS30_IV_SRCID_UVD_ENC_GEN_PURP, &adev->uvd.inst->irq);
 400			if (r)
 401				return r;
 402		}
 403	}
 404
 405	r = amdgpu_uvd_sw_init(adev);
 406	if (r)
 407		return r;
 408
 409	if (!uvd_v6_0_enc_support(adev)) {
 410		for (i = 0; i < adev->uvd.num_enc_rings; ++i)
 411			adev->uvd.inst->ring_enc[i].funcs = NULL;
 412
 413		adev->uvd.inst->irq.num_types = 1;
 414		adev->uvd.num_enc_rings = 0;
 415
 416		DRM_INFO("UVD ENC is disabled\n");
 417	}
 418
 419	ring = &adev->uvd.inst->ring;
 420	sprintf(ring->name, "uvd");
 421	r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.inst->irq, 0,
 422			     AMDGPU_RING_PRIO_DEFAULT);
 423	if (r)
 424		return r;
 425
 426	r = amdgpu_uvd_resume(adev);
 427	if (r)
 428		return r;
 429
 430	if (uvd_v6_0_enc_support(adev)) {
 431		for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
 432			ring = &adev->uvd.inst->ring_enc[i];
 433			sprintf(ring->name, "uvd_enc%d", i);
 434			r = amdgpu_ring_init(adev, ring, 512,
 435					     &adev->uvd.inst->irq, 0,
 436					     AMDGPU_RING_PRIO_DEFAULT);
 437			if (r)
 438				return r;
 439		}
 440	}
 441
 442	r = amdgpu_uvd_entity_init(adev);
 443
 444	return r;
 445}
 446
 447static int uvd_v6_0_sw_fini(void *handle)
 448{
 449	int i, r;
 450	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 451
 452	r = amdgpu_uvd_suspend(adev);
 453	if (r)
 454		return r;
 455
 456	if (uvd_v6_0_enc_support(adev)) {
 457		for (i = 0; i < adev->uvd.num_enc_rings; ++i)
 458			amdgpu_ring_fini(&adev->uvd.inst->ring_enc[i]);
 459	}
 460
 461	return amdgpu_uvd_sw_fini(adev);
 462}
 463
 464/**
 465 * uvd_v6_0_hw_init - start and test UVD block
 466 *
 467 * @adev: amdgpu_device pointer
 468 *
 469 * Initialize the hardware, boot up the VCPU and do some testing
 470 */
 471static int uvd_v6_0_hw_init(void *handle)
 472{
 473	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 474	struct amdgpu_ring *ring = &adev->uvd.inst->ring;
 475	uint32_t tmp;
 476	int i, r;
 477
 478	amdgpu_asic_set_uvd_clocks(adev, 10000, 10000);
 479	uvd_v6_0_set_clockgating_state(adev, AMD_CG_STATE_UNGATE);
 480	uvd_v6_0_enable_mgcg(adev, true);
 481
 482	r = amdgpu_ring_test_helper(ring);
 483	if (r)
 484		goto done;
 485
 486	r = amdgpu_ring_alloc(ring, 10);
 487	if (r) {
 488		DRM_ERROR("amdgpu: ring failed to lock UVD ring (%d).\n", r);
 489		goto done;
 490	}
 491
 492	tmp = PACKET0(mmUVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL, 0);
 493	amdgpu_ring_write(ring, tmp);
 494	amdgpu_ring_write(ring, 0xFFFFF);
 495
 496	tmp = PACKET0(mmUVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL, 0);
 497	amdgpu_ring_write(ring, tmp);
 498	amdgpu_ring_write(ring, 0xFFFFF);
 499
 500	tmp = PACKET0(mmUVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL, 0);
 501	amdgpu_ring_write(ring, tmp);
 502	amdgpu_ring_write(ring, 0xFFFFF);
 503
 504	/* Clear timeout status bits */
 505	amdgpu_ring_write(ring, PACKET0(mmUVD_SEMA_TIMEOUT_STATUS, 0));
 506	amdgpu_ring_write(ring, 0x8);
 507
 508	amdgpu_ring_write(ring, PACKET0(mmUVD_SEMA_CNTL, 0));
 509	amdgpu_ring_write(ring, 3);
 510
 511	amdgpu_ring_commit(ring);
 512
 513	if (uvd_v6_0_enc_support(adev)) {
 514		for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
 515			ring = &adev->uvd.inst->ring_enc[i];
 516			r = amdgpu_ring_test_helper(ring);
 517			if (r)
 518				goto done;
 519		}
 520	}
 521
 522done:
 523	if (!r) {
 524		if (uvd_v6_0_enc_support(adev))
 525			DRM_INFO("UVD and UVD ENC initialized successfully.\n");
 526		else
 527			DRM_INFO("UVD initialized successfully.\n");
 528	}
 529
 530	return r;
 531}
 532
 533/**
 534 * uvd_v6_0_hw_fini - stop the hardware block
 535 *
 536 * @adev: amdgpu_device pointer
 537 *
 538 * Stop the UVD block, mark ring as not ready any more
 539 */
 540static int uvd_v6_0_hw_fini(void *handle)
 541{
 542	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 543
 544	if (RREG32(mmUVD_STATUS) != 0)
 545		uvd_v6_0_stop(adev);
 546
 547	return 0;
 548}
 549
 550static int uvd_v6_0_suspend(void *handle)
 551{
 552	int r;
 553	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 554
 555	r = uvd_v6_0_hw_fini(adev);
 556	if (r)
 557		return r;
 558
 559	return amdgpu_uvd_suspend(adev);
 560}
 561
 562static int uvd_v6_0_resume(void *handle)
 563{
 564	int r;
 565	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 566
 567	r = amdgpu_uvd_resume(adev);
 568	if (r)
 569		return r;
 570
 571	return uvd_v6_0_hw_init(adev);
 572}
 573
 574/**
 575 * uvd_v6_0_mc_resume - memory controller programming
 576 *
 577 * @adev: amdgpu_device pointer
 578 *
 579 * Let the UVD memory controller know it's offsets
 580 */
 581static void uvd_v6_0_mc_resume(struct amdgpu_device *adev)
 582{
 583	uint64_t offset;
 584	uint32_t size;
 585
 586	/* programm memory controller bits 0-27 */
 587	WREG32(mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 588			lower_32_bits(adev->uvd.inst->gpu_addr));
 589	WREG32(mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 590			upper_32_bits(adev->uvd.inst->gpu_addr));
 591
 592	offset = AMDGPU_UVD_FIRMWARE_OFFSET;
 593	size = AMDGPU_UVD_FIRMWARE_SIZE(adev);
 594	WREG32(mmUVD_VCPU_CACHE_OFFSET0, offset >> 3);
 595	WREG32(mmUVD_VCPU_CACHE_SIZE0, size);
 596
 597	offset += size;
 598	size = AMDGPU_UVD_HEAP_SIZE;
 599	WREG32(mmUVD_VCPU_CACHE_OFFSET1, offset >> 3);
 600	WREG32(mmUVD_VCPU_CACHE_SIZE1, size);
 601
 602	offset += size;
 603	size = AMDGPU_UVD_STACK_SIZE +
 604	       (AMDGPU_UVD_SESSION_SIZE * adev->uvd.max_handles);
 605	WREG32(mmUVD_VCPU_CACHE_OFFSET2, offset >> 3);
 606	WREG32(mmUVD_VCPU_CACHE_SIZE2, size);
 607
 608	WREG32(mmUVD_UDEC_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
 609	WREG32(mmUVD_UDEC_DB_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
 610	WREG32(mmUVD_UDEC_DBW_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
 611
 612	WREG32(mmUVD_GP_SCRATCH4, adev->uvd.max_handles);
 613}
 614
 615#if 0
 616static void cz_set_uvd_clock_gating_branches(struct amdgpu_device *adev,
 617		bool enable)
 618{
 619	u32 data, data1;
 620
 621	data = RREG32(mmUVD_CGC_GATE);
 622	data1 = RREG32(mmUVD_SUVD_CGC_GATE);
 623	if (enable) {
 624		data |= UVD_CGC_GATE__SYS_MASK |
 625				UVD_CGC_GATE__UDEC_MASK |
 626				UVD_CGC_GATE__MPEG2_MASK |
 627				UVD_CGC_GATE__RBC_MASK |
 628				UVD_CGC_GATE__LMI_MC_MASK |
 629				UVD_CGC_GATE__IDCT_MASK |
 630				UVD_CGC_GATE__MPRD_MASK |
 631				UVD_CGC_GATE__MPC_MASK |
 632				UVD_CGC_GATE__LBSI_MASK |
 633				UVD_CGC_GATE__LRBBM_MASK |
 634				UVD_CGC_GATE__UDEC_RE_MASK |
 635				UVD_CGC_GATE__UDEC_CM_MASK |
 636				UVD_CGC_GATE__UDEC_IT_MASK |
 637				UVD_CGC_GATE__UDEC_DB_MASK |
 638				UVD_CGC_GATE__UDEC_MP_MASK |
 639				UVD_CGC_GATE__WCB_MASK |
 640				UVD_CGC_GATE__VCPU_MASK |
 641				UVD_CGC_GATE__SCPU_MASK;
 642		data1 |= UVD_SUVD_CGC_GATE__SRE_MASK |
 643				UVD_SUVD_CGC_GATE__SIT_MASK |
 644				UVD_SUVD_CGC_GATE__SMP_MASK |
 645				UVD_SUVD_CGC_GATE__SCM_MASK |
 646				UVD_SUVD_CGC_GATE__SDB_MASK |
 647				UVD_SUVD_CGC_GATE__SRE_H264_MASK |
 648				UVD_SUVD_CGC_GATE__SRE_HEVC_MASK |
 649				UVD_SUVD_CGC_GATE__SIT_H264_MASK |
 650				UVD_SUVD_CGC_GATE__SIT_HEVC_MASK |
 651				UVD_SUVD_CGC_GATE__SCM_H264_MASK |
 652				UVD_SUVD_CGC_GATE__SCM_HEVC_MASK |
 653				UVD_SUVD_CGC_GATE__SDB_H264_MASK |
 654				UVD_SUVD_CGC_GATE__SDB_HEVC_MASK;
 655	} else {
 656		data &= ~(UVD_CGC_GATE__SYS_MASK |
 657				UVD_CGC_GATE__UDEC_MASK |
 658				UVD_CGC_GATE__MPEG2_MASK |
 659				UVD_CGC_GATE__RBC_MASK |
 660				UVD_CGC_GATE__LMI_MC_MASK |
 661				UVD_CGC_GATE__LMI_UMC_MASK |
 662				UVD_CGC_GATE__IDCT_MASK |
 663				UVD_CGC_GATE__MPRD_MASK |
 664				UVD_CGC_GATE__MPC_MASK |
 665				UVD_CGC_GATE__LBSI_MASK |
 666				UVD_CGC_GATE__LRBBM_MASK |
 667				UVD_CGC_GATE__UDEC_RE_MASK |
 668				UVD_CGC_GATE__UDEC_CM_MASK |
 669				UVD_CGC_GATE__UDEC_IT_MASK |
 670				UVD_CGC_GATE__UDEC_DB_MASK |
 671				UVD_CGC_GATE__UDEC_MP_MASK |
 672				UVD_CGC_GATE__WCB_MASK |
 673				UVD_CGC_GATE__VCPU_MASK |
 674				UVD_CGC_GATE__SCPU_MASK);
 675		data1 &= ~(UVD_SUVD_CGC_GATE__SRE_MASK |
 676				UVD_SUVD_CGC_GATE__SIT_MASK |
 677				UVD_SUVD_CGC_GATE__SMP_MASK |
 678				UVD_SUVD_CGC_GATE__SCM_MASK |
 679				UVD_SUVD_CGC_GATE__SDB_MASK |
 680				UVD_SUVD_CGC_GATE__SRE_H264_MASK |
 681				UVD_SUVD_CGC_GATE__SRE_HEVC_MASK |
 682				UVD_SUVD_CGC_GATE__SIT_H264_MASK |
 683				UVD_SUVD_CGC_GATE__SIT_HEVC_MASK |
 684				UVD_SUVD_CGC_GATE__SCM_H264_MASK |
 685				UVD_SUVD_CGC_GATE__SCM_HEVC_MASK |
 686				UVD_SUVD_CGC_GATE__SDB_H264_MASK |
 687				UVD_SUVD_CGC_GATE__SDB_HEVC_MASK);
 688	}
 689	WREG32(mmUVD_CGC_GATE, data);
 690	WREG32(mmUVD_SUVD_CGC_GATE, data1);
 691}
 692#endif
 693
 694/**
 695 * uvd_v6_0_start - start UVD block
 696 *
 697 * @adev: amdgpu_device pointer
 698 *
 699 * Setup and start the UVD block
 700 */
 701static int uvd_v6_0_start(struct amdgpu_device *adev)
 702{
 703	struct amdgpu_ring *ring = &adev->uvd.inst->ring;
 704	uint32_t rb_bufsz, tmp;
 705	uint32_t lmi_swap_cntl;
 706	uint32_t mp_swap_cntl;
 707	int i, j, r;
 708
 709	/* disable DPG */
 710	WREG32_P(mmUVD_POWER_STATUS, 0, ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
 711
 712	/* disable byte swapping */
 713	lmi_swap_cntl = 0;
 714	mp_swap_cntl = 0;
 715
 716	uvd_v6_0_mc_resume(adev);
 717
 718	/* disable interupt */
 719	WREG32_FIELD(UVD_MASTINT_EN, VCPU_EN, 0);
 720
 721	/* stall UMC and register bus before resetting VCPU */
 722	WREG32_FIELD(UVD_LMI_CTRL2, STALL_ARB_UMC, 1);
 723	mdelay(1);
 724
 725	/* put LMI, VCPU, RBC etc... into reset */
 726	WREG32(mmUVD_SOFT_RESET,
 727		UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |
 728		UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |
 729		UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |
 730		UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |
 731		UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |
 732		UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |
 733		UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |
 734		UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);
 735	mdelay(5);
 736
 737	/* take UVD block out of reset */
 738	WREG32_FIELD(SRBM_SOFT_RESET, SOFT_RESET_UVD, 0);
 739	mdelay(5);
 740
 741	/* initialize UVD memory controller */
 742	WREG32(mmUVD_LMI_CTRL,
 743		(0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
 744		UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 745		UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 746		UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
 747		UVD_LMI_CTRL__REQ_MODE_MASK |
 748		UVD_LMI_CTRL__DISABLE_ON_FWV_FAIL_MASK);
 749
 750#ifdef __BIG_ENDIAN
 751	/* swap (8 in 32) RB and IB */
 752	lmi_swap_cntl = 0xa;
 753	mp_swap_cntl = 0;
 754#endif
 755	WREG32(mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl);
 756	WREG32(mmUVD_MP_SWAP_CNTL, mp_swap_cntl);
 757
 758	WREG32(mmUVD_MPC_SET_MUXA0, 0x40c2040);
 759	WREG32(mmUVD_MPC_SET_MUXA1, 0x0);
 760	WREG32(mmUVD_MPC_SET_MUXB0, 0x40c2040);
 761	WREG32(mmUVD_MPC_SET_MUXB1, 0x0);
 762	WREG32(mmUVD_MPC_SET_ALU, 0);
 763	WREG32(mmUVD_MPC_SET_MUX, 0x88);
 764
 765	/* take all subblocks out of reset, except VCPU */
 766	WREG32(mmUVD_SOFT_RESET, UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 767	mdelay(5);
 768
 769	/* enable VCPU clock */
 770	WREG32(mmUVD_VCPU_CNTL, UVD_VCPU_CNTL__CLK_EN_MASK);
 771
 772	/* enable UMC */
 773	WREG32_FIELD(UVD_LMI_CTRL2, STALL_ARB_UMC, 0);
 774
 775	/* boot up the VCPU */
 776	WREG32(mmUVD_SOFT_RESET, 0);
 777	mdelay(10);
 778
 779	for (i = 0; i < 10; ++i) {
 780		uint32_t status;
 781
 782		for (j = 0; j < 100; ++j) {
 783			status = RREG32(mmUVD_STATUS);
 784			if (status & 2)
 785				break;
 786			mdelay(10);
 787		}
 788		r = 0;
 789		if (status & 2)
 790			break;
 791
 792		DRM_ERROR("UVD not responding, trying to reset the VCPU!!!\n");
 793		WREG32_FIELD(UVD_SOFT_RESET, VCPU_SOFT_RESET, 1);
 794		mdelay(10);
 795		WREG32_FIELD(UVD_SOFT_RESET, VCPU_SOFT_RESET, 0);
 796		mdelay(10);
 797		r = -1;
 798	}
 799
 800	if (r) {
 801		DRM_ERROR("UVD not responding, giving up!!!\n");
 802		return r;
 803	}
 804	/* enable master interrupt */
 805	WREG32_P(mmUVD_MASTINT_EN,
 806		(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),
 807		~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK));
 808
 809	/* clear the bit 4 of UVD_STATUS */
 810	WREG32_P(mmUVD_STATUS, 0, ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
 811
 812	/* force RBC into idle state */
 813	rb_bufsz = order_base_2(ring->ring_size);
 814	tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
 815	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
 816	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
 817	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_WPTR_POLL_EN, 0);
 818	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
 819	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
 820	WREG32(mmUVD_RBC_RB_CNTL, tmp);
 821
 822	/* set the write pointer delay */
 823	WREG32(mmUVD_RBC_RB_WPTR_CNTL, 0);
 824
 825	/* set the wb address */
 826	WREG32(mmUVD_RBC_RB_RPTR_ADDR, (upper_32_bits(ring->gpu_addr) >> 2));
 827
 828	/* programm the RB_BASE for ring buffer */
 829	WREG32(mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
 830			lower_32_bits(ring->gpu_addr));
 831	WREG32(mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
 832			upper_32_bits(ring->gpu_addr));
 833
 834	/* Initialize the ring buffer's read and write pointers */
 835	WREG32(mmUVD_RBC_RB_RPTR, 0);
 836
 837	ring->wptr = RREG32(mmUVD_RBC_RB_RPTR);
 838	WREG32(mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
 839
 840	WREG32_FIELD(UVD_RBC_RB_CNTL, RB_NO_FETCH, 0);
 841
 842	if (uvd_v6_0_enc_support(adev)) {
 843		ring = &adev->uvd.inst->ring_enc[0];
 844		WREG32(mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
 845		WREG32(mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
 846		WREG32(mmUVD_RB_BASE_LO, ring->gpu_addr);
 847		WREG32(mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
 848		WREG32(mmUVD_RB_SIZE, ring->ring_size / 4);
 849
 850		ring = &adev->uvd.inst->ring_enc[1];
 851		WREG32(mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
 852		WREG32(mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
 853		WREG32(mmUVD_RB_BASE_LO2, ring->gpu_addr);
 854		WREG32(mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
 855		WREG32(mmUVD_RB_SIZE2, ring->ring_size / 4);
 856	}
 857
 858	return 0;
 859}
 860
 861/**
 862 * uvd_v6_0_stop - stop UVD block
 863 *
 864 * @adev: amdgpu_device pointer
 865 *
 866 * stop the UVD block
 867 */
 868static void uvd_v6_0_stop(struct amdgpu_device *adev)
 869{
 870	/* force RBC into idle state */
 871	WREG32(mmUVD_RBC_RB_CNTL, 0x11010101);
 872
 873	/* Stall UMC and register bus before resetting VCPU */
 874	WREG32_P(mmUVD_LMI_CTRL2, 1 << 8, ~(1 << 8));
 875	mdelay(1);
 876
 877	/* put VCPU into reset */
 878	WREG32(mmUVD_SOFT_RESET, UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 879	mdelay(5);
 880
 881	/* disable VCPU clock */
 882	WREG32(mmUVD_VCPU_CNTL, 0x0);
 883
 884	/* Unstall UMC and register bus */
 885	WREG32_P(mmUVD_LMI_CTRL2, 0, ~(1 << 8));
 886
 887	WREG32(mmUVD_STATUS, 0);
 888}
 889
 890/**
 891 * uvd_v6_0_ring_emit_fence - emit an fence & trap command
 892 *
 893 * @ring: amdgpu_ring pointer
 894 * @fence: fence to emit
 
 
 895 *
 896 * Write a fence and a trap command to the ring.
 897 */
 898static void uvd_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 899				     unsigned flags)
 900{
 901	WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
 902
 903	amdgpu_ring_write(ring, PACKET0(mmUVD_CONTEXT_ID, 0));
 904	amdgpu_ring_write(ring, seq);
 905	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0));
 906	amdgpu_ring_write(ring, addr & 0xffffffff);
 907	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0));
 908	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xff);
 909	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD, 0));
 910	amdgpu_ring_write(ring, 0);
 911
 912	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0));
 913	amdgpu_ring_write(ring, 0);
 914	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0));
 915	amdgpu_ring_write(ring, 0);
 916	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD, 0));
 917	amdgpu_ring_write(ring, 2);
 918}
 919
 920/**
 921 * uvd_v6_0_enc_ring_emit_fence - emit an enc fence & trap command
 922 *
 923 * @ring: amdgpu_ring pointer
 924 * @fence: fence to emit
 
 
 925 *
 926 * Write enc a fence and a trap command to the ring.
 927 */
 928static void uvd_v6_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
 929			u64 seq, unsigned flags)
 930{
 931	WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
 932
 933	amdgpu_ring_write(ring, HEVC_ENC_CMD_FENCE);
 934	amdgpu_ring_write(ring, addr);
 935	amdgpu_ring_write(ring, upper_32_bits(addr));
 936	amdgpu_ring_write(ring, seq);
 937	amdgpu_ring_write(ring, HEVC_ENC_CMD_TRAP);
 938}
 939
 940/**
 941 * uvd_v6_0_ring_emit_hdp_flush - skip HDP flushing
 942 *
 943 * @ring: amdgpu_ring pointer
 944 */
 945static void uvd_v6_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 946{
 947	/* The firmware doesn't seem to like touching registers at this point. */
 948}
 949
 950/**
 951 * uvd_v6_0_ring_test_ring - register write test
 952 *
 953 * @ring: amdgpu_ring pointer
 954 *
 955 * Test if we can successfully write to the context register
 956 */
 957static int uvd_v6_0_ring_test_ring(struct amdgpu_ring *ring)
 958{
 959	struct amdgpu_device *adev = ring->adev;
 960	uint32_t tmp = 0;
 961	unsigned i;
 962	int r;
 963
 964	WREG32(mmUVD_CONTEXT_ID, 0xCAFEDEAD);
 965	r = amdgpu_ring_alloc(ring, 3);
 966	if (r)
 967		return r;
 968
 969	amdgpu_ring_write(ring, PACKET0(mmUVD_CONTEXT_ID, 0));
 970	amdgpu_ring_write(ring, 0xDEADBEEF);
 971	amdgpu_ring_commit(ring);
 972	for (i = 0; i < adev->usec_timeout; i++) {
 973		tmp = RREG32(mmUVD_CONTEXT_ID);
 974		if (tmp == 0xDEADBEEF)
 975			break;
 976		udelay(1);
 977	}
 978
 979	if (i >= adev->usec_timeout)
 980		r = -ETIMEDOUT;
 981
 982	return r;
 983}
 984
 985/**
 986 * uvd_v6_0_ring_emit_ib - execute indirect buffer
 987 *
 988 * @ring: amdgpu_ring pointer
 
 989 * @ib: indirect buffer to execute
 
 990 *
 991 * Write ring commands to execute the indirect buffer
 992 */
 993static void uvd_v6_0_ring_emit_ib(struct amdgpu_ring *ring,
 994				  struct amdgpu_job *job,
 995				  struct amdgpu_ib *ib,
 996				  uint32_t flags)
 997{
 998	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
 999
1000	amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_VMID, 0));
1001	amdgpu_ring_write(ring, vmid);
1002
1003	amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_64BIT_BAR_LOW, 0));
1004	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1005	amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH, 0));
1006	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1007	amdgpu_ring_write(ring, PACKET0(mmUVD_RBC_IB_SIZE, 0));
1008	amdgpu_ring_write(ring, ib->length_dw);
1009}
1010
1011/**
1012 * uvd_v6_0_enc_ring_emit_ib - enc execute indirect buffer
1013 *
1014 * @ring: amdgpu_ring pointer
 
1015 * @ib: indirect buffer to execute
 
1016 *
1017 * Write enc ring commands to execute the indirect buffer
1018 */
1019static void uvd_v6_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
1020					struct amdgpu_job *job,
1021					struct amdgpu_ib *ib,
1022					uint32_t flags)
1023{
1024	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
1025
1026	amdgpu_ring_write(ring, HEVC_ENC_CMD_IB_VM);
1027	amdgpu_ring_write(ring, vmid);
1028	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1029	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1030	amdgpu_ring_write(ring, ib->length_dw);
1031}
1032
1033static void uvd_v6_0_ring_emit_wreg(struct amdgpu_ring *ring,
1034				    uint32_t reg, uint32_t val)
1035{
1036	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0));
1037	amdgpu_ring_write(ring, reg << 2);
1038	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0));
1039	amdgpu_ring_write(ring, val);
1040	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD, 0));
1041	amdgpu_ring_write(ring, 0x8);
1042}
1043
1044static void uvd_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
1045					unsigned vmid, uint64_t pd_addr)
1046{
1047	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1048
1049	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0));
1050	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2);
1051	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0));
1052	amdgpu_ring_write(ring, 0);
1053	amdgpu_ring_write(ring, PACKET0(mmUVD_GP_SCRATCH8, 0));
1054	amdgpu_ring_write(ring, 1 << vmid); /* mask */
1055	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD, 0));
1056	amdgpu_ring_write(ring, 0xC);
1057}
1058
1059static void uvd_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1060{
1061	uint32_t seq = ring->fence_drv.sync_seq;
1062	uint64_t addr = ring->fence_drv.gpu_addr;
1063
1064	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0));
1065	amdgpu_ring_write(ring, lower_32_bits(addr));
1066	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0));
1067	amdgpu_ring_write(ring, upper_32_bits(addr));
1068	amdgpu_ring_write(ring, PACKET0(mmUVD_GP_SCRATCH8, 0));
1069	amdgpu_ring_write(ring, 0xffffffff); /* mask */
1070	amdgpu_ring_write(ring, PACKET0(mmUVD_GP_SCRATCH9, 0));
1071	amdgpu_ring_write(ring, seq);
1072	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD, 0));
1073	amdgpu_ring_write(ring, 0xE);
1074}
1075
1076static void uvd_v6_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
1077{
1078	int i;
1079
1080	WARN_ON(ring->wptr % 2 || count % 2);
1081
1082	for (i = 0; i < count / 2; i++) {
1083		amdgpu_ring_write(ring, PACKET0(mmUVD_NO_OP, 0));
1084		amdgpu_ring_write(ring, 0);
1085	}
1086}
1087
1088static void uvd_v6_0_enc_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1089{
1090	uint32_t seq = ring->fence_drv.sync_seq;
1091	uint64_t addr = ring->fence_drv.gpu_addr;
1092
1093	amdgpu_ring_write(ring, HEVC_ENC_CMD_WAIT_GE);
1094	amdgpu_ring_write(ring, lower_32_bits(addr));
1095	amdgpu_ring_write(ring, upper_32_bits(addr));
1096	amdgpu_ring_write(ring, seq);
1097}
1098
1099static void uvd_v6_0_enc_ring_insert_end(struct amdgpu_ring *ring)
1100{
1101	amdgpu_ring_write(ring, HEVC_ENC_CMD_END);
1102}
1103
1104static void uvd_v6_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
1105					    unsigned int vmid, uint64_t pd_addr)
1106{
1107	amdgpu_ring_write(ring, HEVC_ENC_CMD_UPDATE_PTB);
1108	amdgpu_ring_write(ring, vmid);
1109	amdgpu_ring_write(ring, pd_addr >> 12);
1110
1111	amdgpu_ring_write(ring, HEVC_ENC_CMD_FLUSH_TLB);
1112	amdgpu_ring_write(ring, vmid);
1113}
1114
1115static bool uvd_v6_0_is_idle(void *handle)
1116{
1117	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1118
1119	return !(RREG32(mmSRBM_STATUS) & SRBM_STATUS__UVD_BUSY_MASK);
1120}
1121
1122static int uvd_v6_0_wait_for_idle(void *handle)
1123{
1124	unsigned i;
1125	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1126
1127	for (i = 0; i < adev->usec_timeout; i++) {
1128		if (uvd_v6_0_is_idle(handle))
1129			return 0;
1130	}
1131	return -ETIMEDOUT;
1132}
1133
1134#define AMDGPU_UVD_STATUS_BUSY_MASK    0xfd
1135static bool uvd_v6_0_check_soft_reset(void *handle)
1136{
1137	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1138	u32 srbm_soft_reset = 0;
1139	u32 tmp = RREG32(mmSRBM_STATUS);
1140
1141	if (REG_GET_FIELD(tmp, SRBM_STATUS, UVD_RQ_PENDING) ||
1142	    REG_GET_FIELD(tmp, SRBM_STATUS, UVD_BUSY) ||
1143	    (RREG32(mmUVD_STATUS) & AMDGPU_UVD_STATUS_BUSY_MASK))
1144		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_UVD, 1);
1145
1146	if (srbm_soft_reset) {
1147		adev->uvd.inst->srbm_soft_reset = srbm_soft_reset;
1148		return true;
1149	} else {
1150		adev->uvd.inst->srbm_soft_reset = 0;
1151		return false;
1152	}
1153}
1154
1155static int uvd_v6_0_pre_soft_reset(void *handle)
1156{
1157	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1158
1159	if (!adev->uvd.inst->srbm_soft_reset)
1160		return 0;
1161
1162	uvd_v6_0_stop(adev);
1163	return 0;
1164}
1165
1166static int uvd_v6_0_soft_reset(void *handle)
1167{
1168	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1169	u32 srbm_soft_reset;
1170
1171	if (!adev->uvd.inst->srbm_soft_reset)
1172		return 0;
1173	srbm_soft_reset = adev->uvd.inst->srbm_soft_reset;
1174
1175	if (srbm_soft_reset) {
1176		u32 tmp;
1177
1178		tmp = RREG32(mmSRBM_SOFT_RESET);
1179		tmp |= srbm_soft_reset;
1180		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1181		WREG32(mmSRBM_SOFT_RESET, tmp);
1182		tmp = RREG32(mmSRBM_SOFT_RESET);
1183
1184		udelay(50);
1185
1186		tmp &= ~srbm_soft_reset;
1187		WREG32(mmSRBM_SOFT_RESET, tmp);
1188		tmp = RREG32(mmSRBM_SOFT_RESET);
1189
1190		/* Wait a little for things to settle down */
1191		udelay(50);
1192	}
1193
1194	return 0;
1195}
1196
1197static int uvd_v6_0_post_soft_reset(void *handle)
1198{
1199	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1200
1201	if (!adev->uvd.inst->srbm_soft_reset)
1202		return 0;
1203
1204	mdelay(5);
1205
1206	return uvd_v6_0_start(adev);
1207}
1208
1209static int uvd_v6_0_set_interrupt_state(struct amdgpu_device *adev,
1210					struct amdgpu_irq_src *source,
1211					unsigned type,
1212					enum amdgpu_interrupt_state state)
1213{
1214	// TODO
1215	return 0;
1216}
1217
1218static int uvd_v6_0_process_interrupt(struct amdgpu_device *adev,
1219				      struct amdgpu_irq_src *source,
1220				      struct amdgpu_iv_entry *entry)
1221{
1222	bool int_handled = true;
1223	DRM_DEBUG("IH: UVD TRAP\n");
1224
1225	switch (entry->src_id) {
1226	case 124:
1227		amdgpu_fence_process(&adev->uvd.inst->ring);
1228		break;
1229	case 119:
1230		if (likely(uvd_v6_0_enc_support(adev)))
1231			amdgpu_fence_process(&adev->uvd.inst->ring_enc[0]);
1232		else
1233			int_handled = false;
1234		break;
1235	case 120:
1236		if (likely(uvd_v6_0_enc_support(adev)))
1237			amdgpu_fence_process(&adev->uvd.inst->ring_enc[1]);
1238		else
1239			int_handled = false;
1240		break;
1241	}
1242
1243	if (false == int_handled)
1244			DRM_ERROR("Unhandled interrupt: %d %d\n",
1245			  entry->src_id, entry->src_data[0]);
1246
1247	return 0;
1248}
1249
1250static void uvd_v6_0_enable_clock_gating(struct amdgpu_device *adev, bool enable)
1251{
1252	uint32_t data1, data3;
1253
1254	data1 = RREG32(mmUVD_SUVD_CGC_GATE);
1255	data3 = RREG32(mmUVD_CGC_GATE);
1256
1257	data1 |= UVD_SUVD_CGC_GATE__SRE_MASK |
1258		     UVD_SUVD_CGC_GATE__SIT_MASK |
1259		     UVD_SUVD_CGC_GATE__SMP_MASK |
1260		     UVD_SUVD_CGC_GATE__SCM_MASK |
1261		     UVD_SUVD_CGC_GATE__SDB_MASK |
1262		     UVD_SUVD_CGC_GATE__SRE_H264_MASK |
1263		     UVD_SUVD_CGC_GATE__SRE_HEVC_MASK |
1264		     UVD_SUVD_CGC_GATE__SIT_H264_MASK |
1265		     UVD_SUVD_CGC_GATE__SIT_HEVC_MASK |
1266		     UVD_SUVD_CGC_GATE__SCM_H264_MASK |
1267		     UVD_SUVD_CGC_GATE__SCM_HEVC_MASK |
1268		     UVD_SUVD_CGC_GATE__SDB_H264_MASK |
1269		     UVD_SUVD_CGC_GATE__SDB_HEVC_MASK;
1270
1271	if (enable) {
1272		data3 |= (UVD_CGC_GATE__SYS_MASK       |
1273			UVD_CGC_GATE__UDEC_MASK      |
1274			UVD_CGC_GATE__MPEG2_MASK     |
1275			UVD_CGC_GATE__RBC_MASK       |
1276			UVD_CGC_GATE__LMI_MC_MASK    |
1277			UVD_CGC_GATE__LMI_UMC_MASK   |
1278			UVD_CGC_GATE__IDCT_MASK      |
1279			UVD_CGC_GATE__MPRD_MASK      |
1280			UVD_CGC_GATE__MPC_MASK       |
1281			UVD_CGC_GATE__LBSI_MASK      |
1282			UVD_CGC_GATE__LRBBM_MASK     |
1283			UVD_CGC_GATE__UDEC_RE_MASK   |
1284			UVD_CGC_GATE__UDEC_CM_MASK   |
1285			UVD_CGC_GATE__UDEC_IT_MASK   |
1286			UVD_CGC_GATE__UDEC_DB_MASK   |
1287			UVD_CGC_GATE__UDEC_MP_MASK   |
1288			UVD_CGC_GATE__WCB_MASK       |
1289			UVD_CGC_GATE__JPEG_MASK      |
1290			UVD_CGC_GATE__SCPU_MASK      |
1291			UVD_CGC_GATE__JPEG2_MASK);
1292		/* only in pg enabled, we can gate clock to vcpu*/
1293		if (adev->pg_flags & AMD_PG_SUPPORT_UVD)
1294			data3 |= UVD_CGC_GATE__VCPU_MASK;
1295
1296		data3 &= ~UVD_CGC_GATE__REGS_MASK;
1297	} else {
1298		data3 = 0;
1299	}
1300
1301	WREG32(mmUVD_SUVD_CGC_GATE, data1);
1302	WREG32(mmUVD_CGC_GATE, data3);
1303}
1304
1305static void uvd_v6_0_set_sw_clock_gating(struct amdgpu_device *adev)
1306{
1307	uint32_t data, data2;
1308
1309	data = RREG32(mmUVD_CGC_CTRL);
1310	data2 = RREG32(mmUVD_SUVD_CGC_CTRL);
1311
1312
1313	data &= ~(UVD_CGC_CTRL__CLK_OFF_DELAY_MASK |
1314		  UVD_CGC_CTRL__CLK_GATE_DLY_TIMER_MASK);
1315
1316
1317	data |= UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK |
1318		(1 << REG_FIELD_SHIFT(UVD_CGC_CTRL, CLK_GATE_DLY_TIMER)) |
1319		(4 << REG_FIELD_SHIFT(UVD_CGC_CTRL, CLK_OFF_DELAY));
1320
1321	data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
1322			UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
1323			UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
1324			UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
1325			UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
1326			UVD_CGC_CTRL__SYS_MODE_MASK |
1327			UVD_CGC_CTRL__UDEC_MODE_MASK |
1328			UVD_CGC_CTRL__MPEG2_MODE_MASK |
1329			UVD_CGC_CTRL__REGS_MODE_MASK |
1330			UVD_CGC_CTRL__RBC_MODE_MASK |
1331			UVD_CGC_CTRL__LMI_MC_MODE_MASK |
1332			UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
1333			UVD_CGC_CTRL__IDCT_MODE_MASK |
1334			UVD_CGC_CTRL__MPRD_MODE_MASK |
1335			UVD_CGC_CTRL__MPC_MODE_MASK |
1336			UVD_CGC_CTRL__LBSI_MODE_MASK |
1337			UVD_CGC_CTRL__LRBBM_MODE_MASK |
1338			UVD_CGC_CTRL__WCB_MODE_MASK |
1339			UVD_CGC_CTRL__VCPU_MODE_MASK |
1340			UVD_CGC_CTRL__JPEG_MODE_MASK |
1341			UVD_CGC_CTRL__SCPU_MODE_MASK |
1342			UVD_CGC_CTRL__JPEG2_MODE_MASK);
1343	data2 &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK |
1344			UVD_SUVD_CGC_CTRL__SIT_MODE_MASK |
1345			UVD_SUVD_CGC_CTRL__SMP_MODE_MASK |
1346			UVD_SUVD_CGC_CTRL__SCM_MODE_MASK |
1347			UVD_SUVD_CGC_CTRL__SDB_MODE_MASK);
1348
1349	WREG32(mmUVD_CGC_CTRL, data);
1350	WREG32(mmUVD_SUVD_CGC_CTRL, data2);
1351}
1352
1353#if 0
1354static void uvd_v6_0_set_hw_clock_gating(struct amdgpu_device *adev)
1355{
1356	uint32_t data, data1, cgc_flags, suvd_flags;
1357
1358	data = RREG32(mmUVD_CGC_GATE);
1359	data1 = RREG32(mmUVD_SUVD_CGC_GATE);
1360
1361	cgc_flags = UVD_CGC_GATE__SYS_MASK |
1362		UVD_CGC_GATE__UDEC_MASK |
1363		UVD_CGC_GATE__MPEG2_MASK |
1364		UVD_CGC_GATE__RBC_MASK |
1365		UVD_CGC_GATE__LMI_MC_MASK |
1366		UVD_CGC_GATE__IDCT_MASK |
1367		UVD_CGC_GATE__MPRD_MASK |
1368		UVD_CGC_GATE__MPC_MASK |
1369		UVD_CGC_GATE__LBSI_MASK |
1370		UVD_CGC_GATE__LRBBM_MASK |
1371		UVD_CGC_GATE__UDEC_RE_MASK |
1372		UVD_CGC_GATE__UDEC_CM_MASK |
1373		UVD_CGC_GATE__UDEC_IT_MASK |
1374		UVD_CGC_GATE__UDEC_DB_MASK |
1375		UVD_CGC_GATE__UDEC_MP_MASK |
1376		UVD_CGC_GATE__WCB_MASK |
1377		UVD_CGC_GATE__VCPU_MASK |
1378		UVD_CGC_GATE__SCPU_MASK |
1379		UVD_CGC_GATE__JPEG_MASK |
1380		UVD_CGC_GATE__JPEG2_MASK;
1381
1382	suvd_flags = UVD_SUVD_CGC_GATE__SRE_MASK |
1383				UVD_SUVD_CGC_GATE__SIT_MASK |
1384				UVD_SUVD_CGC_GATE__SMP_MASK |
1385				UVD_SUVD_CGC_GATE__SCM_MASK |
1386				UVD_SUVD_CGC_GATE__SDB_MASK;
1387
1388	data |= cgc_flags;
1389	data1 |= suvd_flags;
1390
1391	WREG32(mmUVD_CGC_GATE, data);
1392	WREG32(mmUVD_SUVD_CGC_GATE, data1);
1393}
1394#endif
1395
1396static void uvd_v6_0_enable_mgcg(struct amdgpu_device *adev,
1397				 bool enable)
1398{
1399	u32 orig, data;
1400
1401	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_UVD_MGCG)) {
1402		data = RREG32_UVD_CTX(ixUVD_CGC_MEM_CTRL);
1403		data |= 0xfff;
1404		WREG32_UVD_CTX(ixUVD_CGC_MEM_CTRL, data);
1405
1406		orig = data = RREG32(mmUVD_CGC_CTRL);
1407		data |= UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
1408		if (orig != data)
1409			WREG32(mmUVD_CGC_CTRL, data);
1410	} else {
1411		data = RREG32_UVD_CTX(ixUVD_CGC_MEM_CTRL);
1412		data &= ~0xfff;
1413		WREG32_UVD_CTX(ixUVD_CGC_MEM_CTRL, data);
1414
1415		orig = data = RREG32(mmUVD_CGC_CTRL);
1416		data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
1417		if (orig != data)
1418			WREG32(mmUVD_CGC_CTRL, data);
1419	}
1420}
1421
1422static int uvd_v6_0_set_clockgating_state(void *handle,
1423					  enum amd_clockgating_state state)
1424{
1425	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1426	bool enable = (state == AMD_CG_STATE_GATE);
1427
1428	if (enable) {
1429		/* wait for STATUS to clear */
1430		if (uvd_v6_0_wait_for_idle(handle))
1431			return -EBUSY;
1432		uvd_v6_0_enable_clock_gating(adev, true);
1433		/* enable HW gates because UVD is idle */
1434/*		uvd_v6_0_set_hw_clock_gating(adev); */
1435	} else {
1436		/* disable HW gating and enable Sw gating */
1437		uvd_v6_0_enable_clock_gating(adev, false);
1438	}
1439	uvd_v6_0_set_sw_clock_gating(adev);
1440	return 0;
1441}
1442
1443static int uvd_v6_0_set_powergating_state(void *handle,
1444					  enum amd_powergating_state state)
1445{
1446	/* This doesn't actually powergate the UVD block.
1447	 * That's done in the dpm code via the SMC.  This
1448	 * just re-inits the block as necessary.  The actual
1449	 * gating still happens in the dpm code.  We should
1450	 * revisit this when there is a cleaner line between
1451	 * the smc and the hw blocks
1452	 */
1453	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1454	int ret = 0;
1455
1456	WREG32(mmUVD_POWER_STATUS, UVD_POWER_STATUS__UVD_PG_EN_MASK);
1457
1458	if (state == AMD_PG_STATE_GATE) {
1459		uvd_v6_0_stop(adev);
1460	} else {
1461		ret = uvd_v6_0_start(adev);
1462		if (ret)
1463			goto out;
1464	}
1465
1466out:
1467	return ret;
1468}
1469
1470static void uvd_v6_0_get_clockgating_state(void *handle, u32 *flags)
1471{
1472	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1473	int data;
1474
1475	mutex_lock(&adev->pm.mutex);
1476
1477	if (adev->flags & AMD_IS_APU)
1478		data = RREG32_SMC(ixCURRENT_PG_STATUS_APU);
1479	else
1480		data = RREG32_SMC(ixCURRENT_PG_STATUS);
1481
1482	if (data & CURRENT_PG_STATUS__UVD_PG_STATUS_MASK) {
1483		DRM_INFO("Cannot get clockgating state when UVD is powergated.\n");
1484		goto out;
1485	}
1486
1487	/* AMD_CG_SUPPORT_UVD_MGCG */
1488	data = RREG32(mmUVD_CGC_CTRL);
1489	if (data & UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK)
1490		*flags |= AMD_CG_SUPPORT_UVD_MGCG;
1491
1492out:
1493	mutex_unlock(&adev->pm.mutex);
1494}
1495
1496static const struct amd_ip_funcs uvd_v6_0_ip_funcs = {
1497	.name = "uvd_v6_0",
1498	.early_init = uvd_v6_0_early_init,
1499	.late_init = NULL,
1500	.sw_init = uvd_v6_0_sw_init,
1501	.sw_fini = uvd_v6_0_sw_fini,
1502	.hw_init = uvd_v6_0_hw_init,
1503	.hw_fini = uvd_v6_0_hw_fini,
1504	.suspend = uvd_v6_0_suspend,
1505	.resume = uvd_v6_0_resume,
1506	.is_idle = uvd_v6_0_is_idle,
1507	.wait_for_idle = uvd_v6_0_wait_for_idle,
1508	.check_soft_reset = uvd_v6_0_check_soft_reset,
1509	.pre_soft_reset = uvd_v6_0_pre_soft_reset,
1510	.soft_reset = uvd_v6_0_soft_reset,
1511	.post_soft_reset = uvd_v6_0_post_soft_reset,
1512	.set_clockgating_state = uvd_v6_0_set_clockgating_state,
1513	.set_powergating_state = uvd_v6_0_set_powergating_state,
1514	.get_clockgating_state = uvd_v6_0_get_clockgating_state,
1515};
1516
1517static const struct amdgpu_ring_funcs uvd_v6_0_ring_phys_funcs = {
1518	.type = AMDGPU_RING_TYPE_UVD,
1519	.align_mask = 0xf,
1520	.support_64bit_ptrs = false,
1521	.no_user_fence = true,
1522	.get_rptr = uvd_v6_0_ring_get_rptr,
1523	.get_wptr = uvd_v6_0_ring_get_wptr,
1524	.set_wptr = uvd_v6_0_ring_set_wptr,
1525	.parse_cs = amdgpu_uvd_ring_parse_cs,
1526	.emit_frame_size =
1527		6 + /* hdp invalidate */
1528		10 + /* uvd_v6_0_ring_emit_pipeline_sync */
1529		14, /* uvd_v6_0_ring_emit_fence x1 no user fence */
1530	.emit_ib_size = 8, /* uvd_v6_0_ring_emit_ib */
1531	.emit_ib = uvd_v6_0_ring_emit_ib,
1532	.emit_fence = uvd_v6_0_ring_emit_fence,
1533	.emit_hdp_flush = uvd_v6_0_ring_emit_hdp_flush,
1534	.test_ring = uvd_v6_0_ring_test_ring,
1535	.test_ib = amdgpu_uvd_ring_test_ib,
1536	.insert_nop = uvd_v6_0_ring_insert_nop,
1537	.pad_ib = amdgpu_ring_generic_pad_ib,
1538	.begin_use = amdgpu_uvd_ring_begin_use,
1539	.end_use = amdgpu_uvd_ring_end_use,
1540	.emit_wreg = uvd_v6_0_ring_emit_wreg,
1541};
1542
1543static const struct amdgpu_ring_funcs uvd_v6_0_ring_vm_funcs = {
1544	.type = AMDGPU_RING_TYPE_UVD,
1545	.align_mask = 0xf,
1546	.support_64bit_ptrs = false,
1547	.no_user_fence = true,
1548	.get_rptr = uvd_v6_0_ring_get_rptr,
1549	.get_wptr = uvd_v6_0_ring_get_wptr,
1550	.set_wptr = uvd_v6_0_ring_set_wptr,
1551	.emit_frame_size =
1552		6 + /* hdp invalidate */
1553		10 + /* uvd_v6_0_ring_emit_pipeline_sync */
1554		VI_FLUSH_GPU_TLB_NUM_WREG * 6 + 8 + /* uvd_v6_0_ring_emit_vm_flush */
1555		14 + 14, /* uvd_v6_0_ring_emit_fence x2 vm fence */
1556	.emit_ib_size = 8, /* uvd_v6_0_ring_emit_ib */
1557	.emit_ib = uvd_v6_0_ring_emit_ib,
1558	.emit_fence = uvd_v6_0_ring_emit_fence,
1559	.emit_vm_flush = uvd_v6_0_ring_emit_vm_flush,
1560	.emit_pipeline_sync = uvd_v6_0_ring_emit_pipeline_sync,
1561	.emit_hdp_flush = uvd_v6_0_ring_emit_hdp_flush,
1562	.test_ring = uvd_v6_0_ring_test_ring,
1563	.test_ib = amdgpu_uvd_ring_test_ib,
1564	.insert_nop = uvd_v6_0_ring_insert_nop,
1565	.pad_ib = amdgpu_ring_generic_pad_ib,
1566	.begin_use = amdgpu_uvd_ring_begin_use,
1567	.end_use = amdgpu_uvd_ring_end_use,
1568	.emit_wreg = uvd_v6_0_ring_emit_wreg,
1569};
1570
1571static const struct amdgpu_ring_funcs uvd_v6_0_enc_ring_vm_funcs = {
1572	.type = AMDGPU_RING_TYPE_UVD_ENC,
1573	.align_mask = 0x3f,
1574	.nop = HEVC_ENC_CMD_NO_OP,
1575	.support_64bit_ptrs = false,
1576	.no_user_fence = true,
1577	.get_rptr = uvd_v6_0_enc_ring_get_rptr,
1578	.get_wptr = uvd_v6_0_enc_ring_get_wptr,
1579	.set_wptr = uvd_v6_0_enc_ring_set_wptr,
1580	.emit_frame_size =
1581		4 + /* uvd_v6_0_enc_ring_emit_pipeline_sync */
1582		5 + /* uvd_v6_0_enc_ring_emit_vm_flush */
1583		5 + 5 + /* uvd_v6_0_enc_ring_emit_fence x2 vm fence */
1584		1, /* uvd_v6_0_enc_ring_insert_end */
1585	.emit_ib_size = 5, /* uvd_v6_0_enc_ring_emit_ib */
1586	.emit_ib = uvd_v6_0_enc_ring_emit_ib,
1587	.emit_fence = uvd_v6_0_enc_ring_emit_fence,
1588	.emit_vm_flush = uvd_v6_0_enc_ring_emit_vm_flush,
1589	.emit_pipeline_sync = uvd_v6_0_enc_ring_emit_pipeline_sync,
1590	.test_ring = uvd_v6_0_enc_ring_test_ring,
1591	.test_ib = uvd_v6_0_enc_ring_test_ib,
1592	.insert_nop = amdgpu_ring_insert_nop,
1593	.insert_end = uvd_v6_0_enc_ring_insert_end,
1594	.pad_ib = amdgpu_ring_generic_pad_ib,
1595	.begin_use = amdgpu_uvd_ring_begin_use,
1596	.end_use = amdgpu_uvd_ring_end_use,
1597};
1598
1599static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev)
1600{
1601	if (adev->asic_type >= CHIP_POLARIS10) {
1602		adev->uvd.inst->ring.funcs = &uvd_v6_0_ring_vm_funcs;
1603		DRM_INFO("UVD is enabled in VM mode\n");
1604	} else {
1605		adev->uvd.inst->ring.funcs = &uvd_v6_0_ring_phys_funcs;
1606		DRM_INFO("UVD is enabled in physical mode\n");
1607	}
1608}
1609
1610static void uvd_v6_0_set_enc_ring_funcs(struct amdgpu_device *adev)
1611{
1612	int i;
1613
1614	for (i = 0; i < adev->uvd.num_enc_rings; ++i)
1615		adev->uvd.inst->ring_enc[i].funcs = &uvd_v6_0_enc_ring_vm_funcs;
1616
1617	DRM_INFO("UVD ENC is enabled in VM mode\n");
1618}
1619
1620static const struct amdgpu_irq_src_funcs uvd_v6_0_irq_funcs = {
1621	.set = uvd_v6_0_set_interrupt_state,
1622	.process = uvd_v6_0_process_interrupt,
1623};
1624
1625static void uvd_v6_0_set_irq_funcs(struct amdgpu_device *adev)
1626{
1627	if (uvd_v6_0_enc_support(adev))
1628		adev->uvd.inst->irq.num_types = adev->uvd.num_enc_rings + 1;
1629	else
1630		adev->uvd.inst->irq.num_types = 1;
1631
1632	adev->uvd.inst->irq.funcs = &uvd_v6_0_irq_funcs;
1633}
1634
1635const struct amdgpu_ip_block_version uvd_v6_0_ip_block =
1636{
1637		.type = AMD_IP_BLOCK_TYPE_UVD,
1638		.major = 6,
1639		.minor = 0,
1640		.rev = 0,
1641		.funcs = &uvd_v6_0_ip_funcs,
1642};
1643
1644const struct amdgpu_ip_block_version uvd_v6_2_ip_block =
1645{
1646		.type = AMD_IP_BLOCK_TYPE_UVD,
1647		.major = 6,
1648		.minor = 2,
1649		.rev = 0,
1650		.funcs = &uvd_v6_0_ip_funcs,
1651};
1652
1653const struct amdgpu_ip_block_version uvd_v6_3_ip_block =
1654{
1655		.type = AMD_IP_BLOCK_TYPE_UVD,
1656		.major = 6,
1657		.minor = 3,
1658		.rev = 0,
1659		.funcs = &uvd_v6_0_ip_funcs,
1660};