1/*
   2 * Copyright 2014 Advanced Micro Devices, Inc.
   3 * All Rights Reserved.
   4 *
   5 * Permission is hereby granted, free of charge, to any person obtaining a
   6 * copy of this software and associated documentation files (the
   7 * "Software"), to deal in the Software without restriction, including
   8 * without limitation the rights to use, copy, modify, merge, publish,
   9 * distribute, sub license, and/or sell copies of the Software, and to
  10 * permit persons to whom the Software is furnished to do so, subject to
  11 * the following conditions:
  12 *
  13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20 *
  21 * The above copyright notice and this permission notice (including the
  22 * next paragraph) shall be included in all copies or substantial portions
  23 * of the Software.
  24 *
  25 * Authors: Christian König <christian.koenig@amd.com>
  26 */
  27
  28#include <linux/firmware.h>
  29
  30#include "amdgpu.h"
  31#include "amdgpu_vce.h"
  32#include "vid.h"
  33#include "vce/vce_3_0_d.h"
  34#include "vce/vce_3_0_sh_mask.h"
  35#include "oss/oss_3_0_d.h"
  36#include "oss/oss_3_0_sh_mask.h"
  37#include "gca/gfx_8_0_d.h"
  38#include "smu/smu_7_1_2_d.h"
  39#include "smu/smu_7_1_2_sh_mask.h"
  40#include "gca/gfx_8_0_sh_mask.h"
  41#include "ivsrcid/ivsrcid_vislands30.h"
  42
  43
  44#define GRBM_GFX_INDEX__VCE_INSTANCE__SHIFT	0x04
  45#define GRBM_GFX_INDEX__VCE_INSTANCE_MASK	0x10
  46#define GRBM_GFX_INDEX__VCE_ALL_PIPE		0x07
  47
  48#define mmVCE_LMI_VCPU_CACHE_40BIT_BAR0	0x8616
  49#define mmVCE_LMI_VCPU_CACHE_40BIT_BAR1	0x8617
  50#define mmVCE_LMI_VCPU_CACHE_40BIT_BAR2	0x8618
  51#define mmGRBM_GFX_INDEX_DEFAULT 0xE0000000
  52
  53#define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK	0x02
  54
  55#define VCE_V3_0_FW_SIZE	(384 * 1024)
  56#define VCE_V3_0_STACK_SIZE	(64 * 1024)
  57#define VCE_V3_0_DATA_SIZE	((16 * 1024 * AMDGPU_MAX_VCE_HANDLES) + (52 * 1024))
  58
  59#define FW_52_8_3	((52 << 24) | (8 << 16) | (3 << 8))
  60
  61#define GET_VCE_INSTANCE(i)  ((i) << GRBM_GFX_INDEX__VCE_INSTANCE__SHIFT \
  62					| GRBM_GFX_INDEX__VCE_ALL_PIPE)
  63
  64static void vce_v3_0_mc_resume(struct amdgpu_device *adev, int idx);
  65static void vce_v3_0_set_ring_funcs(struct amdgpu_device *adev);
  66static void vce_v3_0_set_irq_funcs(struct amdgpu_device *adev);
  67static int vce_v3_0_wait_for_idle(void *handle);
  68static int vce_v3_0_set_clockgating_state(void *handle,
  69					  enum amd_clockgating_state state);
  70/**
  71 * vce_v3_0_ring_get_rptr - get read pointer
  72 *
  73 * @ring: amdgpu_ring pointer
  74 *
  75 * Returns the current hardware read pointer
  76 */
  77static uint64_t vce_v3_0_ring_get_rptr(struct amdgpu_ring *ring)
  78{
  79	struct amdgpu_device *adev = ring->adev;
  80	u32 v;
  81
  82	mutex_lock(&adev->grbm_idx_mutex);
  83	if (adev->vce.harvest_config == 0 ||
  84		adev->vce.harvest_config == AMDGPU_VCE_HARVEST_VCE1)
  85		WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(0));
  86	else if (adev->vce.harvest_config == AMDGPU_VCE_HARVEST_VCE0)
  87		WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(1));
  88
  89	if (ring->me == 0)
  90		v = RREG32(mmVCE_RB_RPTR);
  91	else if (ring->me == 1)
  92		v = RREG32(mmVCE_RB_RPTR2);
  93	else
  94		v = RREG32(mmVCE_RB_RPTR3);
  95
  96	WREG32(mmGRBM_GFX_INDEX, mmGRBM_GFX_INDEX_DEFAULT);
  97	mutex_unlock(&adev->grbm_idx_mutex);
  98
  99	return v;
 100}
 101
 102/**
 103 * vce_v3_0_ring_get_wptr - get write pointer
 104 *
 105 * @ring: amdgpu_ring pointer
 106 *
 107 * Returns the current hardware write pointer
 108 */
 109static uint64_t vce_v3_0_ring_get_wptr(struct amdgpu_ring *ring)
 110{
 111	struct amdgpu_device *adev = ring->adev;
 112	u32 v;
 113
 114	mutex_lock(&adev->grbm_idx_mutex);
 115	if (adev->vce.harvest_config == 0 ||
 116		adev->vce.harvest_config == AMDGPU_VCE_HARVEST_VCE1)
 117		WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(0));
 118	else if (adev->vce.harvest_config == AMDGPU_VCE_HARVEST_VCE0)
 119		WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(1));
 120
 121	if (ring->me == 0)
 122		v = RREG32(mmVCE_RB_WPTR);
 123	else if (ring->me == 1)
 124		v = RREG32(mmVCE_RB_WPTR2);
 125	else
 126		v = RREG32(mmVCE_RB_WPTR3);
 127
 128	WREG32(mmGRBM_GFX_INDEX, mmGRBM_GFX_INDEX_DEFAULT);
 129	mutex_unlock(&adev->grbm_idx_mutex);
 130
 131	return v;
 132}
 133
 134/**
 135 * vce_v3_0_ring_set_wptr - set write pointer
 136 *
 137 * @ring: amdgpu_ring pointer
 138 *
 139 * Commits the write pointer to the hardware
 140 */
 141static void vce_v3_0_ring_set_wptr(struct amdgpu_ring *ring)
 142{
 143	struct amdgpu_device *adev = ring->adev;
 144
 145	mutex_lock(&adev->grbm_idx_mutex);
 146	if (adev->vce.harvest_config == 0 ||
 147		adev->vce.harvest_config == AMDGPU_VCE_HARVEST_VCE1)
 148		WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(0));
 149	else if (adev->vce.harvest_config == AMDGPU_VCE_HARVEST_VCE0)
 150		WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(1));
 151
 152	if (ring->me == 0)
 153		WREG32(mmVCE_RB_WPTR, lower_32_bits(ring->wptr));
 154	else if (ring->me == 1)
 155		WREG32(mmVCE_RB_WPTR2, lower_32_bits(ring->wptr));
 156	else
 157		WREG32(mmVCE_RB_WPTR3, lower_32_bits(ring->wptr));
 158
 159	WREG32(mmGRBM_GFX_INDEX, mmGRBM_GFX_INDEX_DEFAULT);
 160	mutex_unlock(&adev->grbm_idx_mutex);
 161}
 162
 163static void vce_v3_0_override_vce_clock_gating(struct amdgpu_device *adev, bool override)
 164{
 165	WREG32_FIELD(VCE_RB_ARB_CTRL, VCE_CGTT_OVERRIDE, override ? 1 : 0);
 166}
 167
 168static void vce_v3_0_set_vce_sw_clock_gating(struct amdgpu_device *adev,
 169					     bool gated)
 170{
 171	u32 data;
 172
 173	/* Set Override to disable Clock Gating */
 174	vce_v3_0_override_vce_clock_gating(adev, true);
 175
 176	/* This function enables MGCG which is controlled by firmware.
 177	   With the clocks in the gated state the core is still
 178	   accessible but the firmware will throttle the clocks on the
 179	   fly as necessary.
 180	*/
 181	if (!gated) {
 182		data = RREG32(mmVCE_CLOCK_GATING_B);
 183		data |= 0x1ff;
 184		data &= ~0xef0000;
 185		WREG32(mmVCE_CLOCK_GATING_B, data);
 186
 187		data = RREG32(mmVCE_UENC_CLOCK_GATING);
 188		data |= 0x3ff000;
 189		data &= ~0xffc00000;
 190		WREG32(mmVCE_UENC_CLOCK_GATING, data);
 191
 192		data = RREG32(mmVCE_UENC_CLOCK_GATING_2);
 193		data |= 0x2;
 194		data &= ~0x00010000;
 195		WREG32(mmVCE_UENC_CLOCK_GATING_2, data);
 196
 197		data = RREG32(mmVCE_UENC_REG_CLOCK_GATING);
 198		data |= 0x37f;
 199		WREG32(mmVCE_UENC_REG_CLOCK_GATING, data);
 200
 201		data = RREG32(mmVCE_UENC_DMA_DCLK_CTRL);
 202		data |= VCE_UENC_DMA_DCLK_CTRL__WRDMCLK_FORCEON_MASK |
 203			VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON_MASK |
 204			VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON_MASK  |
 205			0x8;
 206		WREG32(mmVCE_UENC_DMA_DCLK_CTRL, data);
 207	} else {
 208		data = RREG32(mmVCE_CLOCK_GATING_B);
 209		data &= ~0x80010;
 210		data |= 0xe70008;
 211		WREG32(mmVCE_CLOCK_GATING_B, data);
 212
 213		data = RREG32(mmVCE_UENC_CLOCK_GATING);
 214		data |= 0xffc00000;
 215		WREG32(mmVCE_UENC_CLOCK_GATING, data);
 216
 217		data = RREG32(mmVCE_UENC_CLOCK_GATING_2);
 218		data |= 0x10000;
 219		WREG32(mmVCE_UENC_CLOCK_GATING_2, data);
 220
 221		data = RREG32(mmVCE_UENC_REG_CLOCK_GATING);
 222		data &= ~0x3ff;
 223		WREG32(mmVCE_UENC_REG_CLOCK_GATING, data);
 224
 225		data = RREG32(mmVCE_UENC_DMA_DCLK_CTRL);
 226		data &= ~(VCE_UENC_DMA_DCLK_CTRL__WRDMCLK_FORCEON_MASK |
 227			  VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON_MASK |
 228			  VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON_MASK  |
 229			  0x8);
 230		WREG32(mmVCE_UENC_DMA_DCLK_CTRL, data);
 231	}
 232	vce_v3_0_override_vce_clock_gating(adev, false);
 233}
 234
 235static int vce_v3_0_firmware_loaded(struct amdgpu_device *adev)
 236{
 237	int i, j;
 238
 239	for (i = 0; i < 10; ++i) {
 240		for (j = 0; j < 100; ++j) {
 241			uint32_t status = RREG32(mmVCE_STATUS);
 242
 243			if (status & VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK)
 244				return 0;
 245			mdelay(10);
 246		}
 247
 248		DRM_ERROR("VCE not responding, trying to reset the ECPU!!!\n");
 249		WREG32_FIELD(VCE_SOFT_RESET, ECPU_SOFT_RESET, 1);
 250		mdelay(10);
 251		WREG32_FIELD(VCE_SOFT_RESET, ECPU_SOFT_RESET, 0);
 252		mdelay(10);
 253	}
 254
 255	return -ETIMEDOUT;
 256}
 257
 258/**
 259 * vce_v3_0_start - start VCE block
 260 *
 261 * @adev: amdgpu_device pointer
 262 *
 263 * Setup and start the VCE block
 264 */
 265static int vce_v3_0_start(struct amdgpu_device *adev)
 266{
 267	struct amdgpu_ring *ring;
 268	int idx, r;
 269
 270	mutex_lock(&adev->grbm_idx_mutex);
 271	for (idx = 0; idx < 2; ++idx) {
 272		if (adev->vce.harvest_config & (1 << idx))
 273			continue;
 274
 275		WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(idx));
 276
 277		/* Program instance 0 reg space for two instances or instance 0 case
 278		program instance 1 reg space for only instance 1 available case */
 279		if (idx != 1 || adev->vce.harvest_config == AMDGPU_VCE_HARVEST_VCE0) {
 280			ring = &adev->vce.ring[0];
 281			WREG32(mmVCE_RB_RPTR, lower_32_bits(ring->wptr));
 282			WREG32(mmVCE_RB_WPTR, lower_32_bits(ring->wptr));
 283			WREG32(mmVCE_RB_BASE_LO, ring->gpu_addr);
 284			WREG32(mmVCE_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
 285			WREG32(mmVCE_RB_SIZE, ring->ring_size / 4);
 286
 287			ring = &adev->vce.ring[1];
 288			WREG32(mmVCE_RB_RPTR2, lower_32_bits(ring->wptr));
 289			WREG32(mmVCE_RB_WPTR2, lower_32_bits(ring->wptr));
 290			WREG32(mmVCE_RB_BASE_LO2, ring->gpu_addr);
 291			WREG32(mmVCE_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
 292			WREG32(mmVCE_RB_SIZE2, ring->ring_size / 4);
 293
 294			ring = &adev->vce.ring[2];
 295			WREG32(mmVCE_RB_RPTR3, lower_32_bits(ring->wptr));
 296			WREG32(mmVCE_RB_WPTR3, lower_32_bits(ring->wptr));
 297			WREG32(mmVCE_RB_BASE_LO3, ring->gpu_addr);
 298			WREG32(mmVCE_RB_BASE_HI3, upper_32_bits(ring->gpu_addr));
 299			WREG32(mmVCE_RB_SIZE3, ring->ring_size / 4);
 300		}
 301
 302		vce_v3_0_mc_resume(adev, idx);
 303		WREG32_FIELD(VCE_STATUS, JOB_BUSY, 1);
 304
 305		if (adev->asic_type >= CHIP_STONEY)
 306			WREG32_P(mmVCE_VCPU_CNTL, 1, ~0x200001);
 307		else
 308			WREG32_FIELD(VCE_VCPU_CNTL, CLK_EN, 1);
 309
 310		WREG32_FIELD(VCE_SOFT_RESET, ECPU_SOFT_RESET, 0);
 311		mdelay(100);
 312
 313		r = vce_v3_0_firmware_loaded(adev);
 314
 315		/* clear BUSY flag */
 316		WREG32_FIELD(VCE_STATUS, JOB_BUSY, 0);
 317
 318		if (r) {
 319			DRM_ERROR("VCE not responding, giving up!!!\n");
 320			mutex_unlock(&adev->grbm_idx_mutex);
 321			return r;
 322		}
 323	}
 324
 325	WREG32(mmGRBM_GFX_INDEX, mmGRBM_GFX_INDEX_DEFAULT);
 326	mutex_unlock(&adev->grbm_idx_mutex);
 327
 328	return 0;
 329}
 330
 331static int vce_v3_0_stop(struct amdgpu_device *adev)
 332{
 333	int idx;
 334
 335	mutex_lock(&adev->grbm_idx_mutex);
 336	for (idx = 0; idx < 2; ++idx) {
 337		if (adev->vce.harvest_config & (1 << idx))
 338			continue;
 339
 340		WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(idx));
 341
 342		if (adev->asic_type >= CHIP_STONEY)
 343			WREG32_P(mmVCE_VCPU_CNTL, 0, ~0x200001);
 344		else
 345			WREG32_FIELD(VCE_VCPU_CNTL, CLK_EN, 0);
 346
 347		/* hold on ECPU */
 348		WREG32_FIELD(VCE_SOFT_RESET, ECPU_SOFT_RESET, 1);
 349
 350		/* clear VCE STATUS */
 351		WREG32(mmVCE_STATUS, 0);
 352	}
 353
 354	WREG32(mmGRBM_GFX_INDEX, mmGRBM_GFX_INDEX_DEFAULT);
 355	mutex_unlock(&adev->grbm_idx_mutex);
 356
 357	return 0;
 358}
 359
 360#define ixVCE_HARVEST_FUSE_MACRO__ADDRESS     0xC0014074
 361#define VCE_HARVEST_FUSE_MACRO__SHIFT       27
 362#define VCE_HARVEST_FUSE_MACRO__MASK        0x18000000
 363
 364static unsigned vce_v3_0_get_harvest_config(struct amdgpu_device *adev)
 365{
 366	u32 tmp;
 367
 368	if ((adev->asic_type == CHIP_FIJI) ||
 369	    (adev->asic_type == CHIP_STONEY))
 370		return AMDGPU_VCE_HARVEST_VCE1;
 371
 372	if (adev->flags & AMD_IS_APU)
 373		tmp = (RREG32_SMC(ixVCE_HARVEST_FUSE_MACRO__ADDRESS) &
 374		       VCE_HARVEST_FUSE_MACRO__MASK) >>
 375			VCE_HARVEST_FUSE_MACRO__SHIFT;
 376	else
 377		tmp = (RREG32_SMC(ixCC_HARVEST_FUSES) &
 378		       CC_HARVEST_FUSES__VCE_DISABLE_MASK) >>
 379			CC_HARVEST_FUSES__VCE_DISABLE__SHIFT;
 380
 381	switch (tmp) {
 382	case 1:
 383		return AMDGPU_VCE_HARVEST_VCE0;
 384	case 2:
 385		return AMDGPU_VCE_HARVEST_VCE1;
 386	case 3:
 387		return AMDGPU_VCE_HARVEST_VCE0 | AMDGPU_VCE_HARVEST_VCE1;
 388	default:
 389		if ((adev->asic_type == CHIP_POLARIS10) ||
 390		    (adev->asic_type == CHIP_POLARIS11) ||
 391		    (adev->asic_type == CHIP_POLARIS12) ||
 392		    (adev->asic_type == CHIP_VEGAM))
 393			return AMDGPU_VCE_HARVEST_VCE1;
 394
 395		return 0;
 396	}
 397}
 398
 399static int vce_v3_0_early_init(void *handle)
 400{
 401	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 402
 403	adev->vce.harvest_config = vce_v3_0_get_harvest_config(adev);
 404
 405	if ((adev->vce.harvest_config &
 406	     (AMDGPU_VCE_HARVEST_VCE0 | AMDGPU_VCE_HARVEST_VCE1)) ==
 407	    (AMDGPU_VCE_HARVEST_VCE0 | AMDGPU_VCE_HARVEST_VCE1))
 408		return -ENOENT;
 409
 410	adev->vce.num_rings = 3;
 411
 412	vce_v3_0_set_ring_funcs(adev);
 413	vce_v3_0_set_irq_funcs(adev);
 414
 415	return 0;
 416}
 417
 418static int vce_v3_0_sw_init(void *handle)
 419{
 420	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 421	struct amdgpu_ring *ring;
 422	int r, i;
 423
 424	/* VCE */
 425	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_VCE_TRAP, &adev->vce.irq);
 426	if (r)
 427		return r;
 428
 429	r = amdgpu_vce_sw_init(adev, VCE_V3_0_FW_SIZE +
 430		(VCE_V3_0_STACK_SIZE + VCE_V3_0_DATA_SIZE) * 2);
 431	if (r)
 432		return r;
 433
 434	/* 52.8.3 required for 3 ring support */
 435	if (adev->vce.fw_version < FW_52_8_3)
 436		adev->vce.num_rings = 2;
 437
 438	r = amdgpu_vce_resume(adev);
 439	if (r)
 440		return r;
 441
 442	for (i = 0; i < adev->vce.num_rings; i++) {
 443		enum amdgpu_ring_priority_level hw_prio = amdgpu_vce_get_ring_prio(i);
 444
 445		ring = &adev->vce.ring[i];
 446		sprintf(ring->name, "vce%d", i);
 447		r = amdgpu_ring_init(adev, ring, 512, &adev->vce.irq, 0,
 448				     hw_prio, NULL);
 449		if (r)
 450			return r;
 451	}
 452
 453	return r;
 454}
 455
 456static int vce_v3_0_sw_fini(void *handle)
 457{
 458	int r;
 459	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 460
 461	r = amdgpu_vce_suspend(adev);
 462	if (r)
 463		return r;
 464
 465	return amdgpu_vce_sw_fini(adev);
 466}
 467
 468static int vce_v3_0_hw_init(void *handle)
 469{
 470	int r, i;
 471	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 472
 473	vce_v3_0_override_vce_clock_gating(adev, true);
 474
 475	amdgpu_asic_set_vce_clocks(adev, 10000, 10000);
 476
 477	for (i = 0; i < adev->vce.num_rings; i++) {
 478		r = amdgpu_ring_test_helper(&adev->vce.ring[i]);
 479		if (r)
 480			return r;
 481	}
 482
 483	DRM_INFO("VCE initialized successfully.\n");
 484
 485	return 0;
 486}
 487
 488static int vce_v3_0_hw_fini(void *handle)
 489{
 490	int r;
 491	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 492
 493	cancel_delayed_work_sync(&adev->vce.idle_work);
 494
 495	r = vce_v3_0_wait_for_idle(handle);
 496	if (r)
 497		return r;
 498
 499	vce_v3_0_stop(adev);
 500	return vce_v3_0_set_clockgating_state(adev, AMD_CG_STATE_GATE);
 501}
 502
 503static int vce_v3_0_suspend(void *handle)
 504{
 505	int r;
 506	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 507
 508	/*
 509	 * Proper cleanups before halting the HW engine:
 510	 *   - cancel the delayed idle work
 511	 *   - enable powergating
 512	 *   - enable clockgating
 513	 *   - disable dpm
 514	 *
 515	 * TODO: to align with the VCN implementation, move the
 516	 * jobs for clockgating/powergating/dpm setting to
 517	 * ->set_powergating_state().
 518	 */
 519	cancel_delayed_work_sync(&adev->vce.idle_work);
 520
 521	if (adev->pm.dpm_enabled) {
 522		amdgpu_dpm_enable_vce(adev, false);
 523	} else {
 524		amdgpu_asic_set_vce_clocks(adev, 0, 0);
 525		amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
 526						       AMD_PG_STATE_GATE);
 527		amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
 528						       AMD_CG_STATE_GATE);
 529	}
 530
 531	r = vce_v3_0_hw_fini(adev);
 532	if (r)
 533		return r;
 534
 535	return amdgpu_vce_suspend(adev);
 536}
 537
 538static int vce_v3_0_resume(void *handle)
 539{
 540	int r;
 541	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 542
 543	r = amdgpu_vce_resume(adev);
 544	if (r)
 545		return r;
 546
 547	return vce_v3_0_hw_init(adev);
 548}
 549
 550static void vce_v3_0_mc_resume(struct amdgpu_device *adev, int idx)
 551{
 552	uint32_t offset, size;
 553
 554	WREG32_P(mmVCE_CLOCK_GATING_A, 0, ~(1 << 16));
 555	WREG32_P(mmVCE_UENC_CLOCK_GATING, 0x1FF000, ~0xFF9FF000);
 556	WREG32_P(mmVCE_UENC_REG_CLOCK_GATING, 0x3F, ~0x3F);
 557	WREG32(mmVCE_CLOCK_GATING_B, 0x1FF);
 558
 559	WREG32(mmVCE_LMI_CTRL, 0x00398000);
 560	WREG32_P(mmVCE_LMI_CACHE_CTRL, 0x0, ~0x1);
 561	WREG32(mmVCE_LMI_SWAP_CNTL, 0);
 562	WREG32(mmVCE_LMI_SWAP_CNTL1, 0);
 563	WREG32(mmVCE_LMI_VM_CTRL, 0);
 564	WREG32_OR(mmVCE_VCPU_CNTL, 0x00100000);
 565
 566	if (adev->asic_type >= CHIP_STONEY) {
 567		WREG32(mmVCE_LMI_VCPU_CACHE_40BIT_BAR0, (adev->vce.gpu_addr >> 8));
 568		WREG32(mmVCE_LMI_VCPU_CACHE_40BIT_BAR1, (adev->vce.gpu_addr >> 8));
 569		WREG32(mmVCE_LMI_VCPU_CACHE_40BIT_BAR2, (adev->vce.gpu_addr >> 8));
 570	} else
 571		WREG32(mmVCE_LMI_VCPU_CACHE_40BIT_BAR, (adev->vce.gpu_addr >> 8));
 572	offset = AMDGPU_VCE_FIRMWARE_OFFSET;
 573	size = VCE_V3_0_FW_SIZE;
 574	WREG32(mmVCE_VCPU_CACHE_OFFSET0, offset & 0x7fffffff);
 575	WREG32(mmVCE_VCPU_CACHE_SIZE0, size);
 576
 577	if (idx == 0) {
 578		offset += size;
 579		size = VCE_V3_0_STACK_SIZE;
 580		WREG32(mmVCE_VCPU_CACHE_OFFSET1, offset & 0x7fffffff);
 581		WREG32(mmVCE_VCPU_CACHE_SIZE1, size);
 582		offset += size;
 583		size = VCE_V3_0_DATA_SIZE;
 584		WREG32(mmVCE_VCPU_CACHE_OFFSET2, offset & 0x7fffffff);
 585		WREG32(mmVCE_VCPU_CACHE_SIZE2, size);
 586	} else {
 587		offset += size + VCE_V3_0_STACK_SIZE + VCE_V3_0_DATA_SIZE;
 588		size = VCE_V3_0_STACK_SIZE;
 589		WREG32(mmVCE_VCPU_CACHE_OFFSET1, offset & 0xfffffff);
 590		WREG32(mmVCE_VCPU_CACHE_SIZE1, size);
 591		offset += size;
 592		size = VCE_V3_0_DATA_SIZE;
 593		WREG32(mmVCE_VCPU_CACHE_OFFSET2, offset & 0xfffffff);
 594		WREG32(mmVCE_VCPU_CACHE_SIZE2, size);
 595	}
 596
 597	WREG32_P(mmVCE_LMI_CTRL2, 0x0, ~0x100);
 598	WREG32_FIELD(VCE_SYS_INT_EN, VCE_SYS_INT_TRAP_INTERRUPT_EN, 1);
 599}
 600
 601static bool vce_v3_0_is_idle(void *handle)
 602{
 603	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 604	u32 mask = 0;
 605
 606	mask |= (adev->vce.harvest_config & AMDGPU_VCE_HARVEST_VCE0) ? 0 : SRBM_STATUS2__VCE0_BUSY_MASK;
 607	mask |= (adev->vce.harvest_config & AMDGPU_VCE_HARVEST_VCE1) ? 0 : SRBM_STATUS2__VCE1_BUSY_MASK;
 608
 609	return !(RREG32(mmSRBM_STATUS2) & mask);
 610}
 611
 612static int vce_v3_0_wait_for_idle(void *handle)
 613{
 614	unsigned i;
 615	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 616
 617	for (i = 0; i < adev->usec_timeout; i++)
 618		if (vce_v3_0_is_idle(handle))
 619			return 0;
 620
 621	return -ETIMEDOUT;
 622}
 623
 624#define  VCE_STATUS_VCPU_REPORT_AUTO_BUSY_MASK  0x00000008L   /* AUTO_BUSY */
 625#define  VCE_STATUS_VCPU_REPORT_RB0_BUSY_MASK   0x00000010L   /* RB0_BUSY */
 626#define  VCE_STATUS_VCPU_REPORT_RB1_BUSY_MASK   0x00000020L   /* RB1_BUSY */
 627#define  AMDGPU_VCE_STATUS_BUSY_MASK (VCE_STATUS_VCPU_REPORT_AUTO_BUSY_MASK | \
 628				      VCE_STATUS_VCPU_REPORT_RB0_BUSY_MASK)
 629
 630static bool vce_v3_0_check_soft_reset(void *handle)
 631{
 632	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 633	u32 srbm_soft_reset = 0;
 634
 635	/* According to VCE team , we should use VCE_STATUS instead
 636	 * SRBM_STATUS.VCE_BUSY bit for busy status checking.
 637	 * GRBM_GFX_INDEX.INSTANCE_INDEX is used to specify which VCE
 638	 * instance's registers are accessed
 639	 * (0 for 1st instance, 10 for 2nd instance).
 640	 *
 641	 *VCE_STATUS
 642	 *|UENC|ACPI|AUTO ACTIVE|RB1 |RB0 |RB2 |          |FW_LOADED|JOB |
 643	 *|----+----+-----------+----+----+----+----------+---------+----|
 644	 *|bit8|bit7|    bit6   |bit5|bit4|bit3|   bit2   |  bit1   |bit0|
 645	 *
 646	 * VCE team suggest use bit 3--bit 6 for busy status check
 647	 */
 648	mutex_lock(&adev->grbm_idx_mutex);
 649	WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(0));
 650	if (RREG32(mmVCE_STATUS) & AMDGPU_VCE_STATUS_BUSY_MASK) {
 651		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE0, 1);
 652		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1);
 653	}
 654	WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(1));
 655	if (RREG32(mmVCE_STATUS) & AMDGPU_VCE_STATUS_BUSY_MASK) {
 656		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE0, 1);
 657		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1);
 658	}
 659	WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(0));
 660	mutex_unlock(&adev->grbm_idx_mutex);
 661
 662	if (srbm_soft_reset) {
 663		adev->vce.srbm_soft_reset = srbm_soft_reset;
 664		return true;
 665	} else {
 666		adev->vce.srbm_soft_reset = 0;
 667		return false;
 668	}
 669}
 670
 671static int vce_v3_0_soft_reset(void *handle)
 672{
 673	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 674	u32 srbm_soft_reset;
 675
 676	if (!adev->vce.srbm_soft_reset)
 677		return 0;
 678	srbm_soft_reset = adev->vce.srbm_soft_reset;
 679
 680	if (srbm_soft_reset) {
 681		u32 tmp;
 682
 683		tmp = RREG32(mmSRBM_SOFT_RESET);
 684		tmp |= srbm_soft_reset;
 685		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
 686		WREG32(mmSRBM_SOFT_RESET, tmp);
 687		tmp = RREG32(mmSRBM_SOFT_RESET);
 688
 689		udelay(50);
 690
 691		tmp &= ~srbm_soft_reset;
 692		WREG32(mmSRBM_SOFT_RESET, tmp);
 693		tmp = RREG32(mmSRBM_SOFT_RESET);
 694
 695		/* Wait a little for things to settle down */
 696		udelay(50);
 697	}
 698
 699	return 0;
 700}
 701
 702static int vce_v3_0_pre_soft_reset(void *handle)
 703{
 704	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 705
 706	if (!adev->vce.srbm_soft_reset)
 707		return 0;
 708
 709	mdelay(5);
 710
 711	return vce_v3_0_suspend(adev);
 712}
 713
 714
 715static int vce_v3_0_post_soft_reset(void *handle)
 716{
 717	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 718
 719	if (!adev->vce.srbm_soft_reset)
 720		return 0;
 721
 722	mdelay(5);
 723
 724	return vce_v3_0_resume(adev);
 725}
 726
 727static int vce_v3_0_set_interrupt_state(struct amdgpu_device *adev,
 728					struct amdgpu_irq_src *source,
 729					unsigned type,
 730					enum amdgpu_interrupt_state state)
 731{
 732	uint32_t val = 0;
 733
 734	if (state == AMDGPU_IRQ_STATE_ENABLE)
 735		val |= VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK;
 736
 737	WREG32_P(mmVCE_SYS_INT_EN, val, ~VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);
 738	return 0;
 739}
 740
 741static int vce_v3_0_process_interrupt(struct amdgpu_device *adev,
 742				      struct amdgpu_irq_src *source,
 743				      struct amdgpu_iv_entry *entry)
 744{
 745	DRM_DEBUG("IH: VCE\n");
 746
 747	WREG32_FIELD(VCE_SYS_INT_STATUS, VCE_SYS_INT_TRAP_INTERRUPT_INT, 1);
 748
 749	switch (entry->src_data[0]) {
 750	case 0:
 751	case 1:
 752	case 2:
 753		amdgpu_fence_process(&adev->vce.ring[entry->src_data[0]]);
 754		break;
 755	default:
 756		DRM_ERROR("Unhandled interrupt: %d %d\n",
 757			  entry->src_id, entry->src_data[0]);
 758		break;
 759	}
 760
 761	return 0;
 762}
 763
 764static int vce_v3_0_set_clockgating_state(void *handle,
 765					  enum amd_clockgating_state state)
 766{
 767	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 768	bool enable = (state == AMD_CG_STATE_GATE);
 769	int i;
 770
 771	if (!(adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG))
 772		return 0;
 773
 774	mutex_lock(&adev->grbm_idx_mutex);
 775	for (i = 0; i < 2; i++) {
 776		/* Program VCE Instance 0 or 1 if not harvested */
 777		if (adev->vce.harvest_config & (1 << i))
 778			continue;
 779
 780		WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(i));
 781
 782		if (!enable) {
 783			/* initialize VCE_CLOCK_GATING_A: Clock ON/OFF delay */
 784			uint32_t data = RREG32(mmVCE_CLOCK_GATING_A);
 785			data &= ~(0xf | 0xff0);
 786			data |= ((0x0 << 0) | (0x04 << 4));
 787			WREG32(mmVCE_CLOCK_GATING_A, data);
 788
 789			/* initialize VCE_UENC_CLOCK_GATING: Clock ON/OFF delay */
 790			data = RREG32(mmVCE_UENC_CLOCK_GATING);
 791			data &= ~(0xf | 0xff0);
 792			data |= ((0x0 << 0) | (0x04 << 4));
 793			WREG32(mmVCE_UENC_CLOCK_GATING, data);
 794		}
 795
 796		vce_v3_0_set_vce_sw_clock_gating(adev, enable);
 797	}
 798
 799	WREG32(mmGRBM_GFX_INDEX, mmGRBM_GFX_INDEX_DEFAULT);
 800	mutex_unlock(&adev->grbm_idx_mutex);
 801
 802	return 0;
 803}
 804
 805static int vce_v3_0_set_powergating_state(void *handle,
 806					  enum amd_powergating_state state)
 807{
 808	/* This doesn't actually powergate the VCE block.
 809	 * That's done in the dpm code via the SMC.  This
 810	 * just re-inits the block as necessary.  The actual
 811	 * gating still happens in the dpm code.  We should
 812	 * revisit this when there is a cleaner line between
 813	 * the smc and the hw blocks
 814	 */
 815	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 816	int ret = 0;
 817
 818	if (state == AMD_PG_STATE_GATE) {
 819		ret = vce_v3_0_stop(adev);
 820		if (ret)
 821			goto out;
 822	} else {
 823		ret = vce_v3_0_start(adev);
 824		if (ret)
 825			goto out;
 826	}
 827
 828out:
 829	return ret;
 830}
 831
 832static void vce_v3_0_get_clockgating_state(void *handle, u64 *flags)
 833{
 834	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 835	int data;
 836
 837	mutex_lock(&adev->pm.mutex);
 838
 839	if (adev->flags & AMD_IS_APU)
 840		data = RREG32_SMC(ixCURRENT_PG_STATUS_APU);
 841	else
 842		data = RREG32_SMC(ixCURRENT_PG_STATUS);
 843
 844	if (data & CURRENT_PG_STATUS__VCE_PG_STATUS_MASK) {
 845		DRM_INFO("Cannot get clockgating state when VCE is powergated.\n");
 846		goto out;
 847	}
 848
 849	WREG32_FIELD(GRBM_GFX_INDEX, VCE_INSTANCE, 0);
 850
 851	/* AMD_CG_SUPPORT_VCE_MGCG */
 852	data = RREG32(mmVCE_CLOCK_GATING_A);
 853	if (data & (0x04 << 4))
 854		*flags |= AMD_CG_SUPPORT_VCE_MGCG;
 855
 856out:
 857	mutex_unlock(&adev->pm.mutex);
 858}
 859
 860static void vce_v3_0_ring_emit_ib(struct amdgpu_ring *ring,
 861				  struct amdgpu_job *job,
 862				  struct amdgpu_ib *ib,
 863				  uint32_t flags)
 864{
 865	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
 866
 867	amdgpu_ring_write(ring, VCE_CMD_IB_VM);
 868	amdgpu_ring_write(ring, vmid);
 869	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
 870	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 871	amdgpu_ring_write(ring, ib->length_dw);
 872}
 873
 874static void vce_v3_0_emit_vm_flush(struct amdgpu_ring *ring,
 875				   unsigned int vmid, uint64_t pd_addr)
 876{
 877	amdgpu_ring_write(ring, VCE_CMD_UPDATE_PTB);
 878	amdgpu_ring_write(ring, vmid);
 879	amdgpu_ring_write(ring, pd_addr >> 12);
 880
 881	amdgpu_ring_write(ring, VCE_CMD_FLUSH_TLB);
 882	amdgpu_ring_write(ring, vmid);
 883	amdgpu_ring_write(ring, VCE_CMD_END);
 884}
 885
 886static void vce_v3_0_emit_pipeline_sync(struct amdgpu_ring *ring)
 887{
 888	uint32_t seq = ring->fence_drv.sync_seq;
 889	uint64_t addr = ring->fence_drv.gpu_addr;
 890
 891	amdgpu_ring_write(ring, VCE_CMD_WAIT_GE);
 892	amdgpu_ring_write(ring, lower_32_bits(addr));
 893	amdgpu_ring_write(ring, upper_32_bits(addr));
 894	amdgpu_ring_write(ring, seq);
 895}
 896
 897static const struct amd_ip_funcs vce_v3_0_ip_funcs = {
 898	.name = "vce_v3_0",
 899	.early_init = vce_v3_0_early_init,
 900	.late_init = NULL,
 901	.sw_init = vce_v3_0_sw_init,
 902	.sw_fini = vce_v3_0_sw_fini,
 903	.hw_init = vce_v3_0_hw_init,
 904	.hw_fini = vce_v3_0_hw_fini,
 905	.suspend = vce_v3_0_suspend,
 906	.resume = vce_v3_0_resume,
 907	.is_idle = vce_v3_0_is_idle,
 908	.wait_for_idle = vce_v3_0_wait_for_idle,
 909	.check_soft_reset = vce_v3_0_check_soft_reset,
 910	.pre_soft_reset = vce_v3_0_pre_soft_reset,
 911	.soft_reset = vce_v3_0_soft_reset,
 912	.post_soft_reset = vce_v3_0_post_soft_reset,
 913	.set_clockgating_state = vce_v3_0_set_clockgating_state,
 914	.set_powergating_state = vce_v3_0_set_powergating_state,
 915	.get_clockgating_state = vce_v3_0_get_clockgating_state,
 916};
 917
 918static const struct amdgpu_ring_funcs vce_v3_0_ring_phys_funcs = {
 919	.type = AMDGPU_RING_TYPE_VCE,
 920	.align_mask = 0xf,
 921	.nop = VCE_CMD_NO_OP,
 922	.support_64bit_ptrs = false,
 923	.no_user_fence = true,
 924	.get_rptr = vce_v3_0_ring_get_rptr,
 925	.get_wptr = vce_v3_0_ring_get_wptr,
 926	.set_wptr = vce_v3_0_ring_set_wptr,
 927	.parse_cs = amdgpu_vce_ring_parse_cs,
 928	.emit_frame_size =
 929		4 + /* vce_v3_0_emit_pipeline_sync */
 930		6, /* amdgpu_vce_ring_emit_fence x1 no user fence */
 931	.emit_ib_size = 4, /* amdgpu_vce_ring_emit_ib */
 932	.emit_ib = amdgpu_vce_ring_emit_ib,
 933	.emit_fence = amdgpu_vce_ring_emit_fence,
 934	.test_ring = amdgpu_vce_ring_test_ring,
 935	.test_ib = amdgpu_vce_ring_test_ib,
 936	.insert_nop = amdgpu_ring_insert_nop,
 937	.pad_ib = amdgpu_ring_generic_pad_ib,
 938	.begin_use = amdgpu_vce_ring_begin_use,
 939	.end_use = amdgpu_vce_ring_end_use,
 940};
 941
 942static const struct amdgpu_ring_funcs vce_v3_0_ring_vm_funcs = {
 943	.type = AMDGPU_RING_TYPE_VCE,
 944	.align_mask = 0xf,
 945	.nop = VCE_CMD_NO_OP,
 946	.support_64bit_ptrs = false,
 947	.no_user_fence = true,
 948	.get_rptr = vce_v3_0_ring_get_rptr,
 949	.get_wptr = vce_v3_0_ring_get_wptr,
 950	.set_wptr = vce_v3_0_ring_set_wptr,
 951	.parse_cs = amdgpu_vce_ring_parse_cs_vm,
 952	.emit_frame_size =
 953		6 + /* vce_v3_0_emit_vm_flush */
 954		4 + /* vce_v3_0_emit_pipeline_sync */
 955		6 + 6, /* amdgpu_vce_ring_emit_fence x2 vm fence */
 956	.emit_ib_size = 5, /* vce_v3_0_ring_emit_ib */
 957	.emit_ib = vce_v3_0_ring_emit_ib,
 958	.emit_vm_flush = vce_v3_0_emit_vm_flush,
 959	.emit_pipeline_sync = vce_v3_0_emit_pipeline_sync,
 960	.emit_fence = amdgpu_vce_ring_emit_fence,
 961	.test_ring = amdgpu_vce_ring_test_ring,
 962	.test_ib = amdgpu_vce_ring_test_ib,
 963	.insert_nop = amdgpu_ring_insert_nop,
 964	.pad_ib = amdgpu_ring_generic_pad_ib,
 965	.begin_use = amdgpu_vce_ring_begin_use,
 966	.end_use = amdgpu_vce_ring_end_use,
 967};
 968
 969static void vce_v3_0_set_ring_funcs(struct amdgpu_device *adev)
 970{
 971	int i;
 972
 973	if (adev->asic_type >= CHIP_STONEY) {
 974		for (i = 0; i < adev->vce.num_rings; i++) {
 975			adev->vce.ring[i].funcs = &vce_v3_0_ring_vm_funcs;
 976			adev->vce.ring[i].me = i;
 977		}
 978		DRM_INFO("VCE enabled in VM mode\n");
 979	} else {
 980		for (i = 0; i < adev->vce.num_rings; i++) {
 981			adev->vce.ring[i].funcs = &vce_v3_0_ring_phys_funcs;
 982			adev->vce.ring[i].me = i;
 983		}
 984		DRM_INFO("VCE enabled in physical mode\n");
 985	}
 986}
 987
 988static const struct amdgpu_irq_src_funcs vce_v3_0_irq_funcs = {
 989	.set = vce_v3_0_set_interrupt_state,
 990	.process = vce_v3_0_process_interrupt,
 991};
 992
 993static void vce_v3_0_set_irq_funcs(struct amdgpu_device *adev)
 994{
 995	adev->vce.irq.num_types = 1;
 996	adev->vce.irq.funcs = &vce_v3_0_irq_funcs;
 997};
 998
 999const struct amdgpu_ip_block_version vce_v3_0_ip_block = {
1000	.type = AMD_IP_BLOCK_TYPE_VCE,
1001	.major = 3,
1002	.minor = 0,
1003	.rev = 0,
1004	.funcs = &vce_v3_0_ip_funcs,
1005};
1006
1007const struct amdgpu_ip_block_version vce_v3_1_ip_block = {
1008	.type = AMD_IP_BLOCK_TYPE_VCE,
1009	.major = 3,
1010	.minor = 1,
1011	.rev = 0,
1012	.funcs = &vce_v3_0_ip_funcs,
1013};
1014
1015const struct amdgpu_ip_block_version vce_v3_4_ip_block = {
1016	.type = AMD_IP_BLOCK_TYPE_VCE,
1017	.major = 3,
1018	.minor = 4,
1019	.rev = 0,
1020	.funcs = &vce_v3_0_ip_funcs,
1021};