1/*
   2 * Copyright 2016 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/firmware.h>
  25#include <linux/pci.h>
  26
  27#include <drm/drm_cache.h>
  28
  29#include "amdgpu.h"
  30#include "gmc_v9_0.h"
  31#include "amdgpu_atomfirmware.h"
  32#include "amdgpu_gem.h"
  33
  34#include "gc/gc_9_0_sh_mask.h"
  35#include "dce/dce_12_0_offset.h"
  36#include "dce/dce_12_0_sh_mask.h"
  37#include "vega10_enum.h"
  38#include "mmhub/mmhub_1_0_offset.h"
  39#include "athub/athub_1_0_sh_mask.h"
  40#include "athub/athub_1_0_offset.h"
  41#include "oss/osssys_4_0_offset.h"
  42
  43#include "soc15.h"
  44#include "soc15d.h"
  45#include "soc15_common.h"
  46#include "umc/umc_6_0_sh_mask.h"
  47
  48#include "gfxhub_v1_0.h"
  49#include "mmhub_v1_0.h"
  50#include "athub_v1_0.h"
  51#include "gfxhub_v1_1.h"
 
  52#include "mmhub_v9_4.h"
  53#include "mmhub_v1_7.h"
 
  54#include "umc_v6_1.h"
  55#include "umc_v6_0.h"
  56#include "umc_v6_7.h"
 
  57#include "hdp_v4_0.h"
  58#include "mca_v3_0.h"
  59
  60#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
  61
  62#include "amdgpu_ras.h"
  63#include "amdgpu_xgmi.h"
  64
  65#include "amdgpu_reset.h"
  66
  67/* add these here since we already include dce12 headers and these are for DCN */
  68#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x055d
  69#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
  70#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT                                        0x0
  71#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT                                       0x10
  72#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
  73#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
  74#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0                                                                  0x049d
  75#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX                                                         2
  76
  77#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2                                                          0x05ea
  78#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2_BASE_IDX                                                 2
  79
 
  80
  81static const char *gfxhub_client_ids[] = {
  82	"CB",
  83	"DB",
  84	"IA",
  85	"WD",
  86	"CPF",
  87	"CPC",
  88	"CPG",
  89	"RLC",
  90	"TCP",
  91	"SQC (inst)",
  92	"SQC (data)",
  93	"SQG",
  94	"PA",
  95};
  96
  97static const char *mmhub_client_ids_raven[][2] = {
  98	[0][0] = "MP1",
  99	[1][0] = "MP0",
 100	[2][0] = "VCN",
 101	[3][0] = "VCNU",
 102	[4][0] = "HDP",
 103	[5][0] = "DCE",
 104	[13][0] = "UTCL2",
 105	[19][0] = "TLS",
 106	[26][0] = "OSS",
 107	[27][0] = "SDMA0",
 108	[0][1] = "MP1",
 109	[1][1] = "MP0",
 110	[2][1] = "VCN",
 111	[3][1] = "VCNU",
 112	[4][1] = "HDP",
 113	[5][1] = "XDP",
 114	[6][1] = "DBGU0",
 115	[7][1] = "DCE",
 116	[8][1] = "DCEDWB0",
 117	[9][1] = "DCEDWB1",
 118	[26][1] = "OSS",
 119	[27][1] = "SDMA0",
 120};
 121
 122static const char *mmhub_client_ids_renoir[][2] = {
 123	[0][0] = "MP1",
 124	[1][0] = "MP0",
 125	[2][0] = "HDP",
 126	[4][0] = "DCEDMC",
 127	[5][0] = "DCEVGA",
 128	[13][0] = "UTCL2",
 129	[19][0] = "TLS",
 130	[26][0] = "OSS",
 131	[27][0] = "SDMA0",
 132	[28][0] = "VCN",
 133	[29][0] = "VCNU",
 134	[30][0] = "JPEG",
 135	[0][1] = "MP1",
 136	[1][1] = "MP0",
 137	[2][1] = "HDP",
 138	[3][1] = "XDP",
 139	[6][1] = "DBGU0",
 140	[7][1] = "DCEDMC",
 141	[8][1] = "DCEVGA",
 142	[9][1] = "DCEDWB",
 143	[26][1] = "OSS",
 144	[27][1] = "SDMA0",
 145	[28][1] = "VCN",
 146	[29][1] = "VCNU",
 147	[30][1] = "JPEG",
 148};
 149
 150static const char *mmhub_client_ids_vega10[][2] = {
 151	[0][0] = "MP0",
 152	[1][0] = "UVD",
 153	[2][0] = "UVDU",
 154	[3][0] = "HDP",
 155	[13][0] = "UTCL2",
 156	[14][0] = "OSS",
 157	[15][0] = "SDMA1",
 158	[32+0][0] = "VCE0",
 159	[32+1][0] = "VCE0U",
 160	[32+2][0] = "XDMA",
 161	[32+3][0] = "DCE",
 162	[32+4][0] = "MP1",
 163	[32+14][0] = "SDMA0",
 164	[0][1] = "MP0",
 165	[1][1] = "UVD",
 166	[2][1] = "UVDU",
 167	[3][1] = "DBGU0",
 168	[4][1] = "HDP",
 169	[5][1] = "XDP",
 170	[14][1] = "OSS",
 171	[15][1] = "SDMA0",
 172	[32+0][1] = "VCE0",
 173	[32+1][1] = "VCE0U",
 174	[32+2][1] = "XDMA",
 175	[32+3][1] = "DCE",
 176	[32+4][1] = "DCEDWB",
 177	[32+5][1] = "MP1",
 178	[32+6][1] = "DBGU1",
 179	[32+14][1] = "SDMA1",
 180};
 181
 182static const char *mmhub_client_ids_vega12[][2] = {
 183	[0][0] = "MP0",
 184	[1][0] = "VCE0",
 185	[2][0] = "VCE0U",
 186	[3][0] = "HDP",
 187	[13][0] = "UTCL2",
 188	[14][0] = "OSS",
 189	[15][0] = "SDMA1",
 190	[32+0][0] = "DCE",
 191	[32+1][0] = "XDMA",
 192	[32+2][0] = "UVD",
 193	[32+3][0] = "UVDU",
 194	[32+4][0] = "MP1",
 195	[32+15][0] = "SDMA0",
 196	[0][1] = "MP0",
 197	[1][1] = "VCE0",
 198	[2][1] = "VCE0U",
 199	[3][1] = "DBGU0",
 200	[4][1] = "HDP",
 201	[5][1] = "XDP",
 202	[14][1] = "OSS",
 203	[15][1] = "SDMA0",
 204	[32+0][1] = "DCE",
 205	[32+1][1] = "DCEDWB",
 206	[32+2][1] = "XDMA",
 207	[32+3][1] = "UVD",
 208	[32+4][1] = "UVDU",
 209	[32+5][1] = "MP1",
 210	[32+6][1] = "DBGU1",
 211	[32+15][1] = "SDMA1",
 212};
 213
 214static const char *mmhub_client_ids_vega20[][2] = {
 215	[0][0] = "XDMA",
 216	[1][0] = "DCE",
 217	[2][0] = "VCE0",
 218	[3][0] = "VCE0U",
 219	[4][0] = "UVD",
 220	[5][0] = "UVD1U",
 221	[13][0] = "OSS",
 222	[14][0] = "HDP",
 223	[15][0] = "SDMA0",
 224	[32+0][0] = "UVD",
 225	[32+1][0] = "UVDU",
 226	[32+2][0] = "MP1",
 227	[32+3][0] = "MP0",
 228	[32+12][0] = "UTCL2",
 229	[32+14][0] = "SDMA1",
 230	[0][1] = "XDMA",
 231	[1][1] = "DCE",
 232	[2][1] = "DCEDWB",
 233	[3][1] = "VCE0",
 234	[4][1] = "VCE0U",
 235	[5][1] = "UVD1",
 236	[6][1] = "UVD1U",
 237	[7][1] = "DBGU0",
 238	[8][1] = "XDP",
 239	[13][1] = "OSS",
 240	[14][1] = "HDP",
 241	[15][1] = "SDMA0",
 242	[32+0][1] = "UVD",
 243	[32+1][1] = "UVDU",
 244	[32+2][1] = "DBGU1",
 245	[32+3][1] = "MP1",
 246	[32+4][1] = "MP0",
 247	[32+14][1] = "SDMA1",
 248};
 249
 250static const char *mmhub_client_ids_arcturus[][2] = {
 251	[0][0] = "DBGU1",
 252	[1][0] = "XDP",
 253	[2][0] = "MP1",
 254	[14][0] = "HDP",
 255	[171][0] = "JPEG",
 256	[172][0] = "VCN",
 257	[173][0] = "VCNU",
 258	[203][0] = "JPEG1",
 259	[204][0] = "VCN1",
 260	[205][0] = "VCN1U",
 261	[256][0] = "SDMA0",
 262	[257][0] = "SDMA1",
 263	[258][0] = "SDMA2",
 264	[259][0] = "SDMA3",
 265	[260][0] = "SDMA4",
 266	[261][0] = "SDMA5",
 267	[262][0] = "SDMA6",
 268	[263][0] = "SDMA7",
 269	[384][0] = "OSS",
 270	[0][1] = "DBGU1",
 271	[1][1] = "XDP",
 272	[2][1] = "MP1",
 273	[14][1] = "HDP",
 274	[171][1] = "JPEG",
 275	[172][1] = "VCN",
 276	[173][1] = "VCNU",
 277	[203][1] = "JPEG1",
 278	[204][1] = "VCN1",
 279	[205][1] = "VCN1U",
 280	[256][1] = "SDMA0",
 281	[257][1] = "SDMA1",
 282	[258][1] = "SDMA2",
 283	[259][1] = "SDMA3",
 284	[260][1] = "SDMA4",
 285	[261][1] = "SDMA5",
 286	[262][1] = "SDMA6",
 287	[263][1] = "SDMA7",
 288	[384][1] = "OSS",
 289};
 290
 291static const char *mmhub_client_ids_aldebaran[][2] = {
 292	[2][0] = "MP1",
 293	[3][0] = "MP0",
 294	[32+1][0] = "DBGU_IO0",
 295	[32+2][0] = "DBGU_IO2",
 296	[32+4][0] = "MPIO",
 297	[96+11][0] = "JPEG0",
 298	[96+12][0] = "VCN0",
 299	[96+13][0] = "VCNU0",
 300	[128+11][0] = "JPEG1",
 301	[128+12][0] = "VCN1",
 302	[128+13][0] = "VCNU1",
 303	[160+1][0] = "XDP",
 304	[160+14][0] = "HDP",
 305	[256+0][0] = "SDMA0",
 306	[256+1][0] = "SDMA1",
 307	[256+2][0] = "SDMA2",
 308	[256+3][0] = "SDMA3",
 309	[256+4][0] = "SDMA4",
 310	[384+0][0] = "OSS",
 311	[2][1] = "MP1",
 312	[3][1] = "MP0",
 313	[32+1][1] = "DBGU_IO0",
 314	[32+2][1] = "DBGU_IO2",
 315	[32+4][1] = "MPIO",
 316	[96+11][1] = "JPEG0",
 317	[96+12][1] = "VCN0",
 318	[96+13][1] = "VCNU0",
 319	[128+11][1] = "JPEG1",
 320	[128+12][1] = "VCN1",
 321	[128+13][1] = "VCNU1",
 322	[160+1][1] = "XDP",
 323	[160+14][1] = "HDP",
 324	[256+0][1] = "SDMA0",
 325	[256+1][1] = "SDMA1",
 326	[256+2][1] = "SDMA2",
 327	[256+3][1] = "SDMA3",
 328	[256+4][1] = "SDMA4",
 329	[384+0][1] = "OSS",
 330};
 331
 332static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] =
 333{
 334	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
 335	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
 336};
 337
 338static const struct soc15_reg_golden golden_settings_athub_1_0_0[] =
 339{
 340	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
 341	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
 342};
 343
 344static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = {
 345	(0x000143c0 + 0x00000000),
 346	(0x000143c0 + 0x00000800),
 347	(0x000143c0 + 0x00001000),
 348	(0x000143c0 + 0x00001800),
 349	(0x000543c0 + 0x00000000),
 350	(0x000543c0 + 0x00000800),
 351	(0x000543c0 + 0x00001000),
 352	(0x000543c0 + 0x00001800),
 353	(0x000943c0 + 0x00000000),
 354	(0x000943c0 + 0x00000800),
 355	(0x000943c0 + 0x00001000),
 356	(0x000943c0 + 0x00001800),
 357	(0x000d43c0 + 0x00000000),
 358	(0x000d43c0 + 0x00000800),
 359	(0x000d43c0 + 0x00001000),
 360	(0x000d43c0 + 0x00001800),
 361	(0x001143c0 + 0x00000000),
 362	(0x001143c0 + 0x00000800),
 363	(0x001143c0 + 0x00001000),
 364	(0x001143c0 + 0x00001800),
 365	(0x001543c0 + 0x00000000),
 366	(0x001543c0 + 0x00000800),
 367	(0x001543c0 + 0x00001000),
 368	(0x001543c0 + 0x00001800),
 369	(0x001943c0 + 0x00000000),
 370	(0x001943c0 + 0x00000800),
 371	(0x001943c0 + 0x00001000),
 372	(0x001943c0 + 0x00001800),
 373	(0x001d43c0 + 0x00000000),
 374	(0x001d43c0 + 0x00000800),
 375	(0x001d43c0 + 0x00001000),
 376	(0x001d43c0 + 0x00001800),
 377};
 378
 379static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = {
 380	(0x000143e0 + 0x00000000),
 381	(0x000143e0 + 0x00000800),
 382	(0x000143e0 + 0x00001000),
 383	(0x000143e0 + 0x00001800),
 384	(0x000543e0 + 0x00000000),
 385	(0x000543e0 + 0x00000800),
 386	(0x000543e0 + 0x00001000),
 387	(0x000543e0 + 0x00001800),
 388	(0x000943e0 + 0x00000000),
 389	(0x000943e0 + 0x00000800),
 390	(0x000943e0 + 0x00001000),
 391	(0x000943e0 + 0x00001800),
 392	(0x000d43e0 + 0x00000000),
 393	(0x000d43e0 + 0x00000800),
 394	(0x000d43e0 + 0x00001000),
 395	(0x000d43e0 + 0x00001800),
 396	(0x001143e0 + 0x00000000),
 397	(0x001143e0 + 0x00000800),
 398	(0x001143e0 + 0x00001000),
 399	(0x001143e0 + 0x00001800),
 400	(0x001543e0 + 0x00000000),
 401	(0x001543e0 + 0x00000800),
 402	(0x001543e0 + 0x00001000),
 403	(0x001543e0 + 0x00001800),
 404	(0x001943e0 + 0x00000000),
 405	(0x001943e0 + 0x00000800),
 406	(0x001943e0 + 0x00001000),
 407	(0x001943e0 + 0x00001800),
 408	(0x001d43e0 + 0x00000000),
 409	(0x001d43e0 + 0x00000800),
 410	(0x001d43e0 + 0x00001000),
 411	(0x001d43e0 + 0x00001800),
 412};
 413
 414static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev,
 415		struct amdgpu_irq_src *src,
 416		unsigned type,
 417		enum amdgpu_interrupt_state state)
 418{
 419	u32 bits, i, tmp, reg;
 420
 421	/* Devices newer then VEGA10/12 shall have these programming
 422	     sequences performed by PSP BL */
 
 423	if (adev->asic_type >= CHIP_VEGA20)
 424		return 0;
 425
 426	bits = 0x7f;
 427
 428	switch (state) {
 429	case AMDGPU_IRQ_STATE_DISABLE:
 430		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
 431			reg = ecc_umc_mcumc_ctrl_addrs[i];
 432			tmp = RREG32(reg);
 433			tmp &= ~bits;
 434			WREG32(reg, tmp);
 435		}
 436		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
 437			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
 438			tmp = RREG32(reg);
 439			tmp &= ~bits;
 440			WREG32(reg, tmp);
 441		}
 442		break;
 443	case AMDGPU_IRQ_STATE_ENABLE:
 444		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
 445			reg = ecc_umc_mcumc_ctrl_addrs[i];
 446			tmp = RREG32(reg);
 447			tmp |= bits;
 448			WREG32(reg, tmp);
 449		}
 450		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
 451			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
 452			tmp = RREG32(reg);
 453			tmp |= bits;
 454			WREG32(reg, tmp);
 455		}
 456		break;
 457	default:
 458		break;
 459	}
 460
 461	return 0;
 462}
 463
 464static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
 465					struct amdgpu_irq_src *src,
 466					unsigned type,
 467					enum amdgpu_interrupt_state state)
 468{
 469	struct amdgpu_vmhub *hub;
 470	u32 tmp, reg, bits, i, j;
 471
 472	bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
 473		VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
 474		VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
 475		VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
 476		VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
 477		VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
 478		VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
 479
 480	switch (state) {
 481	case AMDGPU_IRQ_STATE_DISABLE:
 482		for (j = 0; j < adev->num_vmhubs; j++) {
 483			hub = &adev->vmhub[j];
 484			for (i = 0; i < 16; i++) {
 485				reg = hub->vm_context0_cntl + i;
 486
 487				if (j == AMDGPU_GFXHUB_0)
 488					tmp = RREG32_SOC15_IP(GC, reg);
 489				else
 
 
 
 
 
 
 490					tmp = RREG32_SOC15_IP(MMHUB, reg);
 
 
 491
 492				tmp &= ~bits;
 493
 494				if (j == AMDGPU_GFXHUB_0)
 495					WREG32_SOC15_IP(GC, reg, tmp);
 496				else
 497					WREG32_SOC15_IP(MMHUB, reg, tmp);
 
 
 498			}
 499		}
 500		break;
 501	case AMDGPU_IRQ_STATE_ENABLE:
 502		for (j = 0; j < adev->num_vmhubs; j++) {
 503			hub = &adev->vmhub[j];
 504			for (i = 0; i < 16; i++) {
 505				reg = hub->vm_context0_cntl + i;
 506
 507				if (j == AMDGPU_GFXHUB_0)
 508					tmp = RREG32_SOC15_IP(GC, reg);
 509				else
 
 
 
 
 
 
 510					tmp = RREG32_SOC15_IP(MMHUB, reg);
 
 
 511
 512				tmp |= bits;
 513
 514				if (j == AMDGPU_GFXHUB_0)
 515					WREG32_SOC15_IP(GC, reg, tmp);
 516				else
 517					WREG32_SOC15_IP(MMHUB, reg, tmp);
 
 
 518			}
 519		}
 520		break;
 521	default:
 522		break;
 523	}
 524
 525	return 0;
 526}
 527
 528static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
 529				      struct amdgpu_irq_src *source,
 530				      struct amdgpu_iv_entry *entry)
 531{
 532	bool retry_fault = !!(entry->src_data[1] & 0x80);
 533	bool write_fault = !!(entry->src_data[1] & 0x20);
 534	uint32_t status = 0, cid = 0, rw = 0;
 535	struct amdgpu_task_info task_info;
 536	struct amdgpu_vmhub *hub;
 537	const char *mmhub_cid;
 538	const char *hub_name;
 
 539	u64 addr;
 
 
 
 
 
 540
 541	addr = (u64)entry->src_data[0] << 12;
 542	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
 543
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 544	if (retry_fault) {
 545		/* Returning 1 here also prevents sending the IV to the KFD */
 
 
 
 
 
 
 
 
 
 546
 547		/* Process it onyl if it's the first fault for this address */
 548		if (entry->ih != &adev->irq.ih_soft &&
 549		    amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
 
 
 
 
 
 
 550					     entry->timestamp))
 551			return 1;
 552
 553		/* Delegate it to a different ring if the hardware hasn't
 554		 * already done it.
 555		 */
 556		if (entry->ih == &adev->irq.ih) {
 557			amdgpu_irq_delegate(adev, entry, 8);
 558			return 1;
 559		}
 560
 561		/* Try to handle the recoverable page faults by filling page
 562		 * tables
 563		 */
 564		if (amdgpu_vm_handle_fault(adev, entry->pasid, addr, write_fault))
 565			return 1;
 
 
 566	}
 567
 568	if (!printk_ratelimit())
 569		return 0;
 570
 571	if (entry->client_id == SOC15_IH_CLIENTID_VMC) {
 572		hub_name = "mmhub0";
 573		hub = &adev->vmhub[AMDGPU_MMHUB_0];
 574	} else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) {
 575		hub_name = "mmhub1";
 576		hub = &adev->vmhub[AMDGPU_MMHUB_1];
 577	} else {
 578		hub_name = "gfxhub0";
 579		hub = &adev->vmhub[AMDGPU_GFXHUB_0];
 
 
 
 580	}
 581
 582	memset(&task_info, 0, sizeof(struct amdgpu_task_info));
 583	amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
 584
 585	dev_err(adev->dev,
 586		"[%s] %s page fault (src_id:%u ring:%u vmid:%u "
 587		"pasid:%u, for process %s pid %d thread %s pid %d)\n",
 588		hub_name, retry_fault ? "retry" : "no-retry",
 589		entry->src_id, entry->ring_id, entry->vmid,
 590		entry->pasid, task_info.process_name, task_info.tgid,
 591		task_info.task_name, task_info.pid);
 592	dev_err(adev->dev, "  in page starting at address 0x%016llx from IH client 0x%x (%s)\n",
 593		addr, entry->client_id,
 594		soc15_ih_clientid_name[entry->client_id]);
 595
 
 
 
 
 
 
 596	if (amdgpu_sriov_vf(adev))
 597		return 0;
 598
 599	/*
 600	 * Issue a dummy read to wait for the status register to
 601	 * be updated to avoid reading an incorrect value due to
 602	 * the new fast GRBM interface.
 603	 */
 604	if ((entry->vmid_src == AMDGPU_GFXHUB_0) &&
 605	    (adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 4, 2)))
 606		RREG32(hub->vm_l2_pro_fault_status);
 607
 608	status = RREG32(hub->vm_l2_pro_fault_status);
 609	cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID);
 610	rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW);
 611	WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
 
 
 
 
 
 
 
 
 
 
 
 612
 
 
 
 
 613
 614	dev_err(adev->dev,
 615		"VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
 616		status);
 617	if (hub == &adev->vmhub[AMDGPU_GFXHUB_0]) {
 618		dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
 619			cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" :
 620			gfxhub_client_ids[cid],
 621			cid);
 622	} else {
 623		switch (adev->ip_versions[MMHUB_HWIP][0]) {
 624		case IP_VERSION(9, 0, 0):
 625			mmhub_cid = mmhub_client_ids_vega10[cid][rw];
 626			break;
 627		case IP_VERSION(9, 3, 0):
 628			mmhub_cid = mmhub_client_ids_vega12[cid][rw];
 629			break;
 630		case IP_VERSION(9, 4, 0):
 631			mmhub_cid = mmhub_client_ids_vega20[cid][rw];
 632			break;
 633		case IP_VERSION(9, 4, 1):
 634			mmhub_cid = mmhub_client_ids_arcturus[cid][rw];
 635			break;
 636		case IP_VERSION(9, 1, 0):
 637		case IP_VERSION(9, 2, 0):
 638			mmhub_cid = mmhub_client_ids_raven[cid][rw];
 639			break;
 640		case IP_VERSION(1, 5, 0):
 641		case IP_VERSION(2, 4, 0):
 642			mmhub_cid = mmhub_client_ids_renoir[cid][rw];
 643			break;
 
 644		case IP_VERSION(9, 4, 2):
 645			mmhub_cid = mmhub_client_ids_aldebaran[cid][rw];
 646			break;
 647		default:
 648			mmhub_cid = NULL;
 649			break;
 650		}
 651		dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
 652			mmhub_cid ? mmhub_cid : "unknown", cid);
 653	}
 654	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
 655		REG_GET_FIELD(status,
 656		VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
 657	dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
 658		REG_GET_FIELD(status,
 659		VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
 660	dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
 661		REG_GET_FIELD(status,
 662		VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
 663	dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
 664		REG_GET_FIELD(status,
 665		VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
 666	dev_err(adev->dev, "\t RW: 0x%x\n", rw);
 667	return 0;
 668}
 669
 670static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
 671	.set = gmc_v9_0_vm_fault_interrupt_state,
 672	.process = gmc_v9_0_process_interrupt,
 673};
 674
 675
 676static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = {
 677	.set = gmc_v9_0_ecc_interrupt_state,
 678	.process = amdgpu_umc_process_ecc_irq,
 679};
 680
 681static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
 682{
 683	adev->gmc.vm_fault.num_types = 1;
 684	adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
 685
 686	if (!amdgpu_sriov_vf(adev) &&
 687	    !adev->gmc.xgmi.connected_to_cpu) {
 
 688		adev->gmc.ecc_irq.num_types = 1;
 689		adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs;
 690	}
 691}
 692
 693static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid,
 694					uint32_t flush_type)
 695{
 696	u32 req = 0;
 697
 698	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
 699			    PER_VMID_INVALIDATE_REQ, 1 << vmid);
 700	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
 701	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
 702	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
 703	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
 704	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
 705	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
 706	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
 707			    CLEAR_PROTECTION_FAULT_STATUS_ADDR,	0);
 708
 709	return req;
 710}
 711
 712/**
 713 * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore
 714 *
 715 * @adev: amdgpu_device pointer
 716 * @vmhub: vmhub type
 717 *
 718 */
 719static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev,
 720				       uint32_t vmhub)
 721{
 722	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
 
 
 723		return false;
 724
 725	return ((vmhub == AMDGPU_MMHUB_0 ||
 726		 vmhub == AMDGPU_MMHUB_1) &&
 727		(!amdgpu_sriov_vf(adev)) &&
 728		(!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) &&
 729		   (adev->apu_flags & AMD_APU_IS_PICASSO))));
 730}
 731
 732static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
 733					uint8_t vmid, uint16_t *p_pasid)
 734{
 735	uint32_t value;
 736
 737	value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
 738		     + vmid);
 739	*p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
 740
 741	return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
 742}
 743
 744/*
 745 * GART
 746 * VMID 0 is the physical GPU addresses as used by the kernel.
 747 * VMIDs 1-15 are used for userspace clients and are handled
 748 * by the amdgpu vm/hsa code.
 749 */
 750
 751/**
 752 * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type
 753 *
 754 * @adev: amdgpu_device pointer
 755 * @vmid: vm instance to flush
 756 * @vmhub: which hub to flush
 757 * @flush_type: the flush type
 758 *
 759 * Flush the TLB for the requested page table using certain type.
 760 */
 761static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 762					uint32_t vmhub, uint32_t flush_type)
 763{
 764	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub);
 765	const unsigned eng = 17;
 766	u32 j, inv_req, inv_req2, tmp;
 767	struct amdgpu_vmhub *hub;
 768
 769	BUG_ON(vmhub >= adev->num_vmhubs);
 770
 771	hub = &adev->vmhub[vmhub];
 772	if (adev->gmc.xgmi.num_physical_nodes &&
 773	    adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0)) {
 774		/* Vega20+XGMI caches PTEs in TC and TLB. Add a
 775		 * heavy-weight TLB flush (type 2), which flushes
 776		 * both. Due to a race condition with concurrent
 777		 * memory accesses using the same TLB cache line, we
 778		 * still need a second TLB flush after this.
 779		 */
 780		inv_req = gmc_v9_0_get_invalidate_req(vmid, 2);
 781		inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type);
 782	} else {
 783		inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type);
 784		inv_req2 = 0;
 785	}
 786
 787	/* This is necessary for a HW workaround under SRIOV as well
 788	 * as GFXOFF under bare metal
 
 
 
 
 
 789	 */
 790	if (adev->gfx.kiq.ring.sched.ready &&
 791	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) &&
 792	    down_read_trylock(&adev->reset_domain->sem)) {
 793		uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
 794		uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
 795
 796		amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req,
 797						   1 << vmid);
 798		up_read(&adev->reset_domain->sem);
 799		return;
 800	}
 801
 
 802	spin_lock(&adev->gmc.invalidate_lock);
 803
 804	/*
 805	 * It may lose gpuvm invalidate acknowldege state across power-gating
 806	 * off cycle, add semaphore acquire before invalidation and semaphore
 807	 * release after invalidation to avoid entering power gated state
 808	 * to WA the Issue
 809	 */
 810
 811	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
 812	if (use_semaphore) {
 813		for (j = 0; j < adev->usec_timeout; j++) {
 814			/* a read return value of 1 means semaphore acquire */
 815			if (vmhub == AMDGPU_GFXHUB_0)
 816				tmp = RREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_sem + hub->eng_distance * eng);
 817			else
 818				tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_sem + hub->eng_distance * eng);
 819
 820			if (tmp & 0x1)
 821				break;
 822			udelay(1);
 823		}
 824
 825		if (j >= adev->usec_timeout)
 826			DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
 827	}
 828
 829	do {
 830		if (vmhub == AMDGPU_GFXHUB_0)
 831			WREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req);
 832		else
 833			WREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req);
 834
 835		/*
 836		 * Issue a dummy read to wait for the ACK register to
 837		 * be cleared to avoid a false ACK due to the new fast
 838		 * GRBM interface.
 839		 */
 840		if ((vmhub == AMDGPU_GFXHUB_0) &&
 841		    (adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 4, 2)))
 842			RREG32_NO_KIQ(hub->vm_inv_eng0_req +
 843				      hub->eng_distance * eng);
 844
 845		for (j = 0; j < adev->usec_timeout; j++) {
 846			if (vmhub == AMDGPU_GFXHUB_0)
 847				tmp = RREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_ack + hub->eng_distance * eng);
 848			else
 849				tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_ack + hub->eng_distance * eng);
 850
 851			if (tmp & (1 << vmid))
 852				break;
 853			udelay(1);
 854		}
 855
 856		inv_req = inv_req2;
 857		inv_req2 = 0;
 858	} while (inv_req);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 859
 860	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
 861	if (use_semaphore) {
 862		/*
 863		 * add semaphore release after invalidation,
 864		 * write with 0 means semaphore release
 865		 */
 866		if (vmhub == AMDGPU_GFXHUB_0)
 867			WREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_sem + hub->eng_distance * eng, 0);
 868		else
 869			WREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_sem + hub->eng_distance * eng, 0);
 870	}
 871
 872	spin_unlock(&adev->gmc.invalidate_lock);
 873
 874	if (j < adev->usec_timeout)
 875		return;
 876
 877	DRM_ERROR("Timeout waiting for VM flush ACK!\n");
 878}
 879
 880/**
 881 * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid
 882 *
 883 * @adev: amdgpu_device pointer
 884 * @pasid: pasid to be flush
 885 * @flush_type: the flush type
 886 * @all_hub: flush all hubs
 
 887 *
 888 * Flush the TLB for the requested pasid.
 889 */
 890static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
 891					uint16_t pasid, uint32_t flush_type,
 892					bool all_hub)
 893{
 894	int vmid, i;
 895	signed long r;
 896	uint32_t seq;
 897	uint16_t queried_pasid;
 898	bool ret;
 899	u32 usec_timeout = amdgpu_sriov_vf(adev) ? SRIOV_USEC_TIMEOUT : adev->usec_timeout;
 900	struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
 901	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
 902
 903	if (amdgpu_in_reset(adev))
 904		return -EIO;
 905
 906	if (ring->sched.ready && down_read_trylock(&adev->reset_domain->sem)) {
 907		/* Vega20+XGMI caches PTEs in TC and TLB. Add a
 908		 * heavy-weight TLB flush (type 2), which flushes
 909		 * both. Due to a race condition with concurrent
 910		 * memory accesses using the same TLB cache line, we
 911		 * still need a second TLB flush after this.
 912		 */
 913		bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes &&
 914				       adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0));
 915		/* 2 dwords flush + 8 dwords fence */
 916		unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8;
 917
 918		if (vega20_xgmi_wa)
 919			ndw += kiq->pmf->invalidate_tlbs_size;
 920
 921		spin_lock(&adev->gfx.kiq.ring_lock);
 922		/* 2 dwords flush + 8 dwords fence */
 923		amdgpu_ring_alloc(ring, ndw);
 924		if (vega20_xgmi_wa)
 925			kiq->pmf->kiq_invalidate_tlbs(ring,
 926						      pasid, 2, all_hub);
 927		kiq->pmf->kiq_invalidate_tlbs(ring,
 928					pasid, flush_type, all_hub);
 929		r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
 930		if (r) {
 931			amdgpu_ring_undo(ring);
 932			spin_unlock(&adev->gfx.kiq.ring_lock);
 933			up_read(&adev->reset_domain->sem);
 934			return -ETIME;
 935		}
 936
 937		amdgpu_ring_commit(ring);
 938		spin_unlock(&adev->gfx.kiq.ring_lock);
 939		r = amdgpu_fence_wait_polling(ring, seq, usec_timeout);
 940		if (r < 1) {
 941			dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r);
 942			up_read(&adev->reset_domain->sem);
 943			return -ETIME;
 944		}
 945		up_read(&adev->reset_domain->sem);
 946		return 0;
 947	}
 948
 949	for (vmid = 1; vmid < 16; vmid++) {
 
 950
 951		ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid,
 952				&queried_pasid);
 953		if (ret && queried_pasid == pasid) {
 954			if (all_hub) {
 955				for (i = 0; i < adev->num_vmhubs; i++)
 956					gmc_v9_0_flush_gpu_tlb(adev, vmid,
 957							i, flush_type);
 958			} else {
 959				gmc_v9_0_flush_gpu_tlb(adev, vmid,
 960						AMDGPU_GFXHUB_0, flush_type);
 961			}
 962			break;
 
 
 963		}
 964	}
 965
 966	return 0;
 967
 968}
 969
 970static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
 971					    unsigned vmid, uint64_t pd_addr)
 972{
 973	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub);
 974	struct amdgpu_device *adev = ring->adev;
 975	struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub];
 976	uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0);
 977	unsigned eng = ring->vm_inv_eng;
 978
 979	/*
 980	 * It may lose gpuvm invalidate acknowldege state across power-gating
 981	 * off cycle, add semaphore acquire before invalidation and semaphore
 982	 * release after invalidation to avoid entering power gated state
 983	 * to WA the Issue
 984	 */
 985
 986	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
 987	if (use_semaphore)
 988		/* a read return value of 1 means semaphore acuqire */
 989		amdgpu_ring_emit_reg_wait(ring,
 990					  hub->vm_inv_eng0_sem +
 991					  hub->eng_distance * eng, 0x1, 0x1);
 992
 993	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
 994			      (hub->ctx_addr_distance * vmid),
 995			      lower_32_bits(pd_addr));
 996
 997	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
 998			      (hub->ctx_addr_distance * vmid),
 999			      upper_32_bits(pd_addr));
1000
1001	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
1002					    hub->eng_distance * eng,
1003					    hub->vm_inv_eng0_ack +
1004					    hub->eng_distance * eng,
1005					    req, 1 << vmid);
1006
1007	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
1008	if (use_semaphore)
1009		/*
1010		 * add semaphore release after invalidation,
1011		 * write with 0 means semaphore release
1012		 */
1013		amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
1014				      hub->eng_distance * eng, 0);
1015
1016	return pd_addr;
1017}
1018
1019static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
1020					unsigned pasid)
1021{
1022	struct amdgpu_device *adev = ring->adev;
1023	uint32_t reg;
1024
1025	/* Do nothing because there's no lut register for mmhub1. */
1026	if (ring->funcs->vmhub == AMDGPU_MMHUB_1)
1027		return;
1028
1029	if (ring->funcs->vmhub == AMDGPU_GFXHUB_0)
1030		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
1031	else
1032		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
1033
1034	amdgpu_ring_emit_wreg(ring, reg, pasid);
1035}
1036
1037/*
1038 * PTE format on VEGA 10:
1039 * 63:59 reserved
1040 * 58:57 mtype
1041 * 56 F
1042 * 55 L
1043 * 54 P
1044 * 53 SW
1045 * 52 T
1046 * 50:48 reserved
1047 * 47:12 4k physical page base address
1048 * 11:7 fragment
1049 * 6 write
1050 * 5 read
1051 * 4 exe
1052 * 3 Z
1053 * 2 snooped
1054 * 1 system
1055 * 0 valid
1056 *
1057 * PDE format on VEGA 10:
1058 * 63:59 block fragment size
1059 * 58:55 reserved
1060 * 54 P
1061 * 53:48 reserved
1062 * 47:6 physical base address of PD or PTE
1063 * 5:3 reserved
1064 * 2 C
1065 * 1 system
1066 * 0 valid
1067 */
1068
1069static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
1070
1071{
1072	switch (flags) {
1073	case AMDGPU_VM_MTYPE_DEFAULT:
1074		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1075	case AMDGPU_VM_MTYPE_NC:
1076		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1077	case AMDGPU_VM_MTYPE_WC:
1078		return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC);
1079	case AMDGPU_VM_MTYPE_RW:
1080		return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW);
1081	case AMDGPU_VM_MTYPE_CC:
1082		return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC);
1083	case AMDGPU_VM_MTYPE_UC:
1084		return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC);
1085	default:
1086		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1087	}
1088}
1089
1090static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
1091				uint64_t *addr, uint64_t *flags)
1092{
1093	if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
1094		*addr = amdgpu_gmc_vram_mc2pa(adev, *addr);
1095	BUG_ON(*addr & 0xFFFF00000000003FULL);
1096
1097	if (!adev->gmc.translate_further)
1098		return;
1099
1100	if (level == AMDGPU_VM_PDB1) {
1101		/* Set the block fragment size */
1102		if (!(*flags & AMDGPU_PDE_PTE))
1103			*flags |= AMDGPU_PDE_BFS(0x9);
1104
1105	} else if (level == AMDGPU_VM_PDB0) {
1106		if (*flags & AMDGPU_PDE_PTE) {
1107			*flags &= ~AMDGPU_PDE_PTE;
1108			if (!(*flags & AMDGPU_PTE_VALID))
1109				*addr |= 1 << PAGE_SHIFT;
1110		} else {
1111			*flags |= AMDGPU_PTE_TF;
1112		}
1113	}
1114}
1115
1116static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev,
1117					 struct amdgpu_bo *bo,
1118					 struct amdgpu_bo_va_mapping *mapping,
1119					 uint64_t *flags)
1120{
1121	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
1122	bool is_vram = bo->tbo.resource->mem_type == TTM_PL_VRAM;
1123	bool coherent = bo->flags & AMDGPU_GEM_CREATE_COHERENT;
 
 
 
1124	bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED;
1125	unsigned int mtype;
 
1126	bool snoop = false;
 
1127
1128	switch (adev->ip_versions[GC_HWIP][0]) {
 
 
1129	case IP_VERSION(9, 4, 1):
1130	case IP_VERSION(9, 4, 2):
1131		if (is_vram) {
1132			if (bo_adev == adev) {
1133				if (uncached)
1134					mtype = MTYPE_UC;
1135				else if (coherent)
1136					mtype = MTYPE_CC;
1137				else
1138					mtype = MTYPE_RW;
1139				/* FIXME: is this still needed? Or does
1140				 * amdgpu_ttm_tt_pde_flags already handle this?
1141				 */
1142				if (adev->ip_versions[GC_HWIP][0] ==
1143					IP_VERSION(9, 4, 2) &&
 
 
1144				    adev->gmc.xgmi.connected_to_cpu)
1145					snoop = true;
1146			} else {
1147				if (uncached || coherent)
1148					mtype = MTYPE_UC;
1149				else
1150					mtype = MTYPE_NC;
1151				if (mapping->bo_va->is_xgmi)
1152					snoop = true;
1153			}
1154		} else {
1155			if (uncached || coherent)
1156				mtype = MTYPE_UC;
1157			else
1158				mtype = MTYPE_NC;
1159			/* FIXME: is this still needed? Or does
1160			 * amdgpu_ttm_tt_pde_flags already handle this?
1161			 */
1162			snoop = true;
1163		}
1164		break;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1165	default:
1166		if (uncached || coherent)
1167			mtype = MTYPE_UC;
1168		else
1169			mtype = MTYPE_NC;
1170
1171		/* FIXME: is this still needed? Or does
1172		 * amdgpu_ttm_tt_pde_flags already handle this?
1173		 */
1174		if (!is_vram)
1175			snoop = true;
1176	}
1177
1178	if (mtype != MTYPE_NC)
1179		*flags = (*flags & ~AMDGPU_PTE_MTYPE_VG10_MASK) |
1180			 AMDGPU_PTE_MTYPE_VG10(mtype);
1181	*flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;
1182}
1183
1184static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
1185				struct amdgpu_bo_va_mapping *mapping,
1186				uint64_t *flags)
1187{
1188	struct amdgpu_bo *bo = mapping->bo_va->base.bo;
1189
1190	*flags &= ~AMDGPU_PTE_EXECUTABLE;
1191	*flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1192
1193	*flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1194	*flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK;
1195
1196	if (mapping->flags & AMDGPU_PTE_PRT) {
1197		*flags |= AMDGPU_PTE_PRT;
1198		*flags &= ~AMDGPU_PTE_VALID;
1199	}
1200
1201	if (bo && bo->tbo.resource)
1202		gmc_v9_0_get_coherence_flags(adev, mapping->bo_va->base.bo,
1203					     mapping, flags);
1204}
1205
1206static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1207{
1208	u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
1209	unsigned size;
1210
1211	/* TODO move to DC so GMC doesn't need to hard-code DCN registers */
1212
1213	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
1214		size = AMDGPU_VBIOS_VGA_ALLOCATION;
1215	} else {
1216		u32 viewport;
1217
1218		switch (adev->ip_versions[DCE_HWIP][0]) {
1219		case IP_VERSION(1, 0, 0):
1220		case IP_VERSION(1, 0, 1):
1221			viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
1222			size = (REG_GET_FIELD(viewport,
1223					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1224				REG_GET_FIELD(viewport,
1225					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1226				4);
1227			break;
1228		case IP_VERSION(2, 1, 0):
1229			viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2);
1230			size = (REG_GET_FIELD(viewport,
1231					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1232				REG_GET_FIELD(viewport,
1233					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1234				4);
1235			break;
1236		default:
1237			viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
1238			size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
1239				REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
1240				4);
1241			break;
1242		}
1243	}
1244
1245	return size;
1246}
1247
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1248static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
1249	.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
1250	.flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid,
1251	.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
1252	.emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
1253	.map_mtype = gmc_v9_0_map_mtype,
1254	.get_vm_pde = gmc_v9_0_get_vm_pde,
1255	.get_vm_pte = gmc_v9_0_get_vm_pte,
 
1256	.get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
 
 
 
1257};
1258
1259static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
1260{
1261	adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
1262}
1263
1264static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
1265{
1266	switch (adev->ip_versions[UMC_HWIP][0]) {
1267	case IP_VERSION(6, 0, 0):
1268		adev->umc.funcs = &umc_v6_0_funcs;
1269		break;
1270	case IP_VERSION(6, 1, 1):
1271		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1272		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1273		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1274		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20;
 
1275		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1276		adev->umc.ras = &umc_v6_1_ras;
1277		break;
1278	case IP_VERSION(6, 1, 2):
1279		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1280		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1281		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1282		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT;
 
1283		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1284		adev->umc.ras = &umc_v6_1_ras;
1285		break;
1286	case IP_VERSION(6, 7, 0):
1287		adev->umc.max_ras_err_cnt_per_query =
1288			UMC_V6_7_TOTAL_CHANNEL_NUM * UMC_V6_7_BAD_PAGE_NUM_PER_CHANNEL;
1289		adev->umc.channel_inst_num = UMC_V6_7_CHANNEL_INSTANCE_NUM;
1290		adev->umc.umc_inst_num = UMC_V6_7_UMC_INSTANCE_NUM;
1291		adev->umc.channel_offs = UMC_V6_7_PER_CHANNEL_OFFSET;
 
1292		if (!adev->gmc.xgmi.connected_to_cpu)
1293			adev->umc.ras = &umc_v6_7_ras;
1294		if (1 & adev->smuio.funcs->get_die_id(adev))
1295			adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_first[0][0];
1296		else
1297			adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_second[0][0];
1298		break;
 
 
 
 
 
 
 
 
 
 
 
 
1299	default:
1300		break;
1301	}
1302
1303	if (adev->umc.ras) {
1304		amdgpu_ras_register_ras_block(adev, &adev->umc.ras->ras_block);
1305
1306		strcpy(adev->umc.ras->ras_block.ras_comm.name, "umc");
1307		adev->umc.ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__UMC;
1308		adev->umc.ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
1309		adev->umc.ras_if = &adev->umc.ras->ras_block.ras_comm;
1310
1311		/* If don't define special ras_late_init function, use default ras_late_init */
1312		if (!adev->umc.ras->ras_block.ras_late_init)
1313				adev->umc.ras->ras_block.ras_late_init = amdgpu_umc_ras_late_init;
1314
1315		/* If not defined special ras_cb function, use default ras_cb */
1316		if (!adev->umc.ras->ras_block.ras_cb)
1317			adev->umc.ras->ras_block.ras_cb = amdgpu_umc_process_ras_data_cb;
1318	}
1319}
1320
1321static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
1322{
1323	switch (adev->ip_versions[MMHUB_HWIP][0]) {
1324	case IP_VERSION(9, 4, 1):
1325		adev->mmhub.funcs = &mmhub_v9_4_funcs;
1326		break;
1327	case IP_VERSION(9, 4, 2):
1328		adev->mmhub.funcs = &mmhub_v1_7_funcs;
1329		break;
 
 
 
1330	default:
1331		adev->mmhub.funcs = &mmhub_v1_0_funcs;
1332		break;
1333	}
1334}
1335
1336static void gmc_v9_0_set_mmhub_ras_funcs(struct amdgpu_device *adev)
1337{
1338	switch (adev->ip_versions[MMHUB_HWIP][0]) {
1339	case IP_VERSION(9, 4, 0):
1340		adev->mmhub.ras = &mmhub_v1_0_ras;
1341		break;
1342	case IP_VERSION(9, 4, 1):
1343		adev->mmhub.ras = &mmhub_v9_4_ras;
1344		break;
1345	case IP_VERSION(9, 4, 2):
1346		adev->mmhub.ras = &mmhub_v1_7_ras;
1347		break;
 
 
 
1348	default:
1349		/* mmhub ras is not available */
1350		break;
1351	}
1352
1353	if (adev->mmhub.ras) {
1354		amdgpu_ras_register_ras_block(adev, &adev->mmhub.ras->ras_block);
1355
1356		strcpy(adev->mmhub.ras->ras_block.ras_comm.name, "mmhub");
1357		adev->mmhub.ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MMHUB;
1358		adev->mmhub.ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
1359		adev->mmhub.ras_if = &adev->mmhub.ras->ras_block.ras_comm;
1360	}
1361}
1362
1363static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev)
1364{
1365	adev->gfxhub.funcs = &gfxhub_v1_0_funcs;
 
 
 
 
1366}
1367
1368static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev)
1369{
1370	adev->hdp.ras = &hdp_v4_0_ras;
1371	amdgpu_ras_register_ras_block(adev, &adev->hdp.ras->ras_block);
1372	adev->hdp.ras_if = &adev->hdp.ras->ras_block.ras_comm;
1373}
1374
1375static void gmc_v9_0_set_mca_funcs(struct amdgpu_device *adev)
1376{
 
 
1377	/* is UMC the right IP to check for MCA?  Maybe DF? */
1378	switch (adev->ip_versions[UMC_HWIP][0]) {
1379	case IP_VERSION(6, 7, 0):
1380		if (!adev->gmc.xgmi.connected_to_cpu)
1381			adev->mca.funcs = &mca_v3_0_funcs;
 
 
 
1382		break;
1383	default:
1384		break;
1385	}
1386}
1387
1388static int gmc_v9_0_early_init(void *handle)
1389{
1390	int r;
1391	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 
 
 
 
1392
1393	/* ARCT and VEGA20 don't have XGMI defined in their IP discovery tables */
1394	if (adev->asic_type == CHIP_VEGA20 ||
1395	    adev->asic_type == CHIP_ARCTURUS)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1396		adev->gmc.xgmi.supported = true;
1397
1398	if (adev->ip_versions[XGMI_HWIP][0] == IP_VERSION(6, 1, 0)) {
1399		adev->gmc.xgmi.supported = true;
1400		adev->gmc.xgmi.connected_to_cpu =
1401			adev->smuio.funcs->is_host_gpu_xgmi_supported(adev);
1402	}
1403
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1404	gmc_v9_0_set_gmc_funcs(adev);
1405	gmc_v9_0_set_irq_funcs(adev);
1406	gmc_v9_0_set_umc_funcs(adev);
1407	gmc_v9_0_set_mmhub_funcs(adev);
1408	gmc_v9_0_set_mmhub_ras_funcs(adev);
1409	gmc_v9_0_set_gfxhub_funcs(adev);
1410	gmc_v9_0_set_hdp_ras_funcs(adev);
1411	gmc_v9_0_set_mca_funcs(adev);
 
1412
1413	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
1414	adev->gmc.shared_aperture_end =
1415		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
1416	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
1417	adev->gmc.private_aperture_end =
1418		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
1419
1420	r = amdgpu_gmc_ras_early_init(adev);
1421	if (r)
1422		return r;
1423
1424	return 0;
1425}
1426
1427static int gmc_v9_0_late_init(void *handle)
1428{
1429	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1430	int r;
1431
1432	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
1433	if (r)
1434		return r;
1435
1436	/*
1437	 * Workaround performance drop issue with VBIOS enables partial
1438	 * writes, while disables HBM ECC for vega10.
1439	 */
1440	if (!amdgpu_sriov_vf(adev) &&
1441	    (adev->ip_versions[UMC_HWIP][0] == IP_VERSION(6, 0, 0))) {
1442		if (!(adev->ras_enabled & (1 << AMDGPU_RAS_BLOCK__UMC))) {
1443			if (adev->df.funcs &&
1444			    adev->df.funcs->enable_ecc_force_par_wr_rmw)
1445				adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false);
1446		}
1447	}
1448
1449	if (!amdgpu_persistent_edc_harvesting_supported(adev)) {
1450		if (adev->mmhub.ras && adev->mmhub.ras->ras_block.hw_ops &&
1451		    adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
1452			adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(adev);
1453
1454		if (adev->hdp.ras && adev->hdp.ras->ras_block.hw_ops &&
1455		    adev->hdp.ras->ras_block.hw_ops->reset_ras_error_count)
1456			adev->hdp.ras->ras_block.hw_ops->reset_ras_error_count(adev);
1457	}
1458
1459	r = amdgpu_gmc_ras_late_init(adev);
1460	if (r)
1461		return r;
1462
1463	return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
1464}
1465
1466static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
1467					struct amdgpu_gmc *mc)
1468{
1469	u64 base = adev->mmhub.funcs->get_fb_location(adev);
1470
 
 
1471	/* add the xgmi offset of the physical node */
1472	base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1473	if (adev->gmc.xgmi.connected_to_cpu) {
1474		amdgpu_gmc_sysvm_location(adev, mc);
1475	} else {
1476		amdgpu_gmc_vram_location(adev, mc, base);
1477		amdgpu_gmc_gart_location(adev, mc);
1478		amdgpu_gmc_agp_location(adev, mc);
 
1479	}
1480	/* base offset of vram pages */
1481	adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);
1482
1483	/* XXX: add the xgmi offset of the physical node? */
1484	adev->vm_manager.vram_base_offset +=
1485		adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1486}
1487
1488/**
1489 * gmc_v9_0_mc_init - initialize the memory controller driver params
1490 *
1491 * @adev: amdgpu_device pointer
1492 *
1493 * Look up the amount of vram, vram width, and decide how to place
1494 * vram and gart within the GPU's physical address space.
1495 * Returns 0 for success.
1496 */
1497static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
1498{
1499	int r;
1500
1501	/* size in MB on si */
1502	adev->gmc.mc_vram_size =
1503		adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
 
 
 
 
 
1504	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
1505
1506	if (!(adev->flags & AMD_IS_APU) &&
1507	    !adev->gmc.xgmi.connected_to_cpu) {
1508		r = amdgpu_device_resize_fb_bar(adev);
1509		if (r)
1510			return r;
1511	}
1512	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
1513	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
1514
1515#ifdef CONFIG_X86_64
1516	/*
1517	 * AMD Accelerated Processing Platform (APP) supporting GPU-HOST xgmi
1518	 * interface can use VRAM through here as it appears system reserved
1519	 * memory in host address space.
1520	 *
1521	 * For APUs, VRAM is just the stolen system memory and can be accessed
1522	 * directly.
1523	 *
1524	 * Otherwise, use the legacy Host Data Path (HDP) through PCIe BAR.
1525	 */
1526
1527	/* check whether both host-gpu and gpu-gpu xgmi links exist */
1528	if (((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) ||
 
1529	    (adev->gmc.xgmi.supported &&
1530	     adev->gmc.xgmi.connected_to_cpu)) {
1531		adev->gmc.aper_base =
1532			adev->gfxhub.funcs->get_mc_fb_offset(adev) +
1533			adev->gmc.xgmi.physical_node_id *
1534			adev->gmc.xgmi.node_segment_size;
1535		adev->gmc.aper_size = adev->gmc.real_vram_size;
1536	}
1537
1538#endif
1539	/* In case the PCI BAR is larger than the actual amount of vram */
1540	adev->gmc.visible_vram_size = adev->gmc.aper_size;
1541	if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
1542		adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
1543
1544	/* set the gart size */
1545	if (amdgpu_gart_size == -1) {
1546		switch (adev->ip_versions[GC_HWIP][0]) {
1547		case IP_VERSION(9, 0, 1):  /* all engines support GPUVM */
1548		case IP_VERSION(9, 2, 1):  /* all engines support GPUVM */
1549		case IP_VERSION(9, 4, 0):
1550		case IP_VERSION(9, 4, 1):
1551		case IP_VERSION(9, 4, 2):
 
 
1552		default:
1553			adev->gmc.gart_size = 512ULL << 20;
1554			break;
1555		case IP_VERSION(9, 1, 0):   /* DCE SG support */
1556		case IP_VERSION(9, 2, 2):   /* DCE SG support */
1557		case IP_VERSION(9, 3, 0):
1558			adev->gmc.gart_size = 1024ULL << 20;
1559			break;
1560		}
1561	} else {
1562		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
1563	}
1564
1565	adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
1566
1567	gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
1568
1569	return 0;
1570}
1571
1572static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
1573{
1574	int r;
1575
1576	if (adev->gart.bo) {
1577		WARN(1, "VEGA10 PCIE GART already initialized\n");
1578		return 0;
1579	}
1580
1581	if (adev->gmc.xgmi.connected_to_cpu) {
1582		adev->gmc.vmid0_page_table_depth = 1;
1583		adev->gmc.vmid0_page_table_block_size = 12;
1584	} else {
1585		adev->gmc.vmid0_page_table_depth = 0;
1586		adev->gmc.vmid0_page_table_block_size = 0;
1587	}
1588
1589	/* Initialize common gart structure */
1590	r = amdgpu_gart_init(adev);
1591	if (r)
1592		return r;
1593	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
1594	adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) |
1595				 AMDGPU_PTE_EXECUTABLE;
1596
1597	r = amdgpu_gart_table_vram_alloc(adev);
1598	if (r)
1599		return r;
 
 
 
 
 
 
1600
1601	if (adev->gmc.xgmi.connected_to_cpu) {
1602		r = amdgpu_gmc_pdb0_alloc(adev);
1603	}
1604
1605	return r;
1606}
1607
1608/**
1609 * gmc_v9_0_save_registers - saves regs
1610 *
1611 * @adev: amdgpu_device pointer
1612 *
1613 * This saves potential register values that should be
1614 * restored upon resume
1615 */
1616static void gmc_v9_0_save_registers(struct amdgpu_device *adev)
1617{
1618	if ((adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 0)) ||
1619	    (adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 1)))
1620		adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0);
1621}
1622
1623static int gmc_v9_0_sw_init(void *handle)
1624{
1625	int r, vram_width = 0, vram_type = 0, vram_vendor = 0, dma_addr_bits;
1626	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
1627
1628	adev->gfxhub.funcs->init(adev);
1629
1630	adev->mmhub.funcs->init(adev);
1631	if (adev->mca.funcs)
1632		adev->mca.funcs->init(adev);
 
1633
1634	spin_lock_init(&adev->gmc.invalidate_lock);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1635
1636	r = amdgpu_atomfirmware_get_vram_info(adev,
1637		&vram_width, &vram_type, &vram_vendor);
1638	if (amdgpu_sriov_vf(adev))
1639		/* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN,
1640		 * and DF related registers is not readable, seems hardcord is the
1641		 * only way to set the correct vram_width
1642		 */
1643		adev->gmc.vram_width = 2048;
1644	else if (amdgpu_emu_mode != 1)
1645		adev->gmc.vram_width = vram_width;
1646
1647	if (!adev->gmc.vram_width) {
1648		int chansize, numchan;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1649
1650		/* hbm memory channel size */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1651		if (adev->flags & AMD_IS_APU)
1652			chansize = 64;
1653		else
1654			chansize = 128;
1655		if (adev->df.funcs &&
1656		    adev->df.funcs->get_hbm_channel_number) {
1657			numchan = adev->df.funcs->get_hbm_channel_number(adev);
1658			adev->gmc.vram_width = numchan * chansize;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1659		}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1660	}
1661
1662	adev->gmc.vram_type = vram_type;
1663	adev->gmc.vram_vendor = vram_vendor;
1664	switch (adev->ip_versions[GC_HWIP][0]) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1665	case IP_VERSION(9, 1, 0):
1666	case IP_VERSION(9, 2, 2):
1667		adev->num_vmhubs = 2;
 
1668
1669		if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
1670			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1671		} else {
1672			/* vm_size is 128TB + 512GB for legacy 3-level page support */
1673			amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);
1674			adev->gmc.translate_further =
1675				adev->vm_manager.num_level > 1;
1676		}
1677		break;
1678	case IP_VERSION(9, 0, 1):
1679	case IP_VERSION(9, 2, 1):
1680	case IP_VERSION(9, 4, 0):
1681	case IP_VERSION(9, 3, 0):
1682	case IP_VERSION(9, 4, 2):
1683		adev->num_vmhubs = 2;
1684
1685
1686		/*
1687		 * To fulfill 4-level page support,
1688		 * vm size is 256TB (48bit), maximum size of Vega10,
1689		 * block size 512 (9bit)
1690		 */
1691		/* sriov restrict max_pfn below AMDGPU_GMC_HOLE */
1692		if (amdgpu_sriov_vf(adev))
1693			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47);
1694		else
1695			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1696		if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
1697			adev->gmc.translate_further = adev->vm_manager.num_level > 1;
1698		break;
1699	case IP_VERSION(9, 4, 1):
1700		adev->num_vmhubs = 3;
 
 
1701
1702		/* Keep the vm size same with Vega20 */
1703		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1704		adev->gmc.translate_further = adev->vm_manager.num_level > 1;
1705		break;
 
 
 
 
 
 
 
 
 
 
 
1706	default:
1707		break;
1708	}
1709
1710	/* This interrupt is VMC page fault.*/
1711	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,
1712				&adev->gmc.vm_fault);
1713	if (r)
1714		return r;
1715
1716	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1)) {
1717		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT,
1718					&adev->gmc.vm_fault);
1719		if (r)
1720			return r;
1721	}
1722
1723	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,
1724				&adev->gmc.vm_fault);
1725
1726	if (r)
1727		return r;
1728
1729	if (!amdgpu_sriov_vf(adev) &&
1730	    !adev->gmc.xgmi.connected_to_cpu) {
 
1731		/* interrupt sent to DF. */
1732		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0,
1733				      &adev->gmc.ecc_irq);
1734		if (r)
1735			return r;
1736	}
1737
1738	/* Set the internal MC address mask
1739	 * This is the max address of the GPU's
1740	 * internal address space.
1741	 */
1742	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
1743
1744	dma_addr_bits = adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ? 48:44;
 
 
 
1745	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(dma_addr_bits));
1746	if (r) {
1747		printk(KERN_WARNING "amdgpu: No suitable DMA available.\n");
1748		return r;
1749	}
1750	adev->need_swiotlb = drm_need_swiotlb(dma_addr_bits);
1751
1752	r = gmc_v9_0_mc_init(adev);
1753	if (r)
1754		return r;
1755
1756	amdgpu_gmc_get_vbios_allocations(adev);
1757
 
 
 
 
 
 
 
1758	/* Memory manager */
1759	r = amdgpu_bo_init(adev);
1760	if (r)
1761		return r;
1762
1763	r = gmc_v9_0_gart_init(adev);
1764	if (r)
1765		return r;
1766
 
1767	/*
1768	 * number of VMs
1769	 * VMID 0 is reserved for System
1770	 * amdgpu graphics/compute will use VMIDs 1..n-1
1771	 * amdkfd will use VMIDs n..15
1772	 *
1773	 * The first KFD VMID is 8 for GPUs with graphics, 3 for
1774	 * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs
1775	 * for video processing.
1776	 */
1777	adev->vm_manager.first_kfd_vmid =
1778		(adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) ||
1779		 adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) ? 3 : 8;
 
 
 
 
1780
1781	amdgpu_vm_manager_init(adev);
1782
1783	gmc_v9_0_save_registers(adev);
1784
 
 
 
 
 
 
 
 
1785	return 0;
1786}
1787
1788static int gmc_v9_0_sw_fini(void *handle)
1789{
1790	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 
 
 
1791
1792	amdgpu_gmc_ras_fini(adev);
1793	amdgpu_gem_force_release(adev);
1794	amdgpu_vm_manager_fini(adev);
1795	amdgpu_gart_table_vram_free(adev);
 
 
 
 
 
1796	amdgpu_bo_free_kernel(&adev->gmc.pdb0_bo, NULL, &adev->gmc.ptr_pdb0);
1797	amdgpu_bo_fini(adev);
1798
 
 
 
1799	return 0;
1800}
1801
1802static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
1803{
1804
1805	switch (adev->ip_versions[MMHUB_HWIP][0]) {
1806	case IP_VERSION(9, 0, 0):
1807		if (amdgpu_sriov_vf(adev))
1808			break;
1809		fallthrough;
1810	case IP_VERSION(9, 4, 0):
1811		soc15_program_register_sequence(adev,
1812						golden_settings_mmhub_1_0_0,
1813						ARRAY_SIZE(golden_settings_mmhub_1_0_0));
1814		soc15_program_register_sequence(adev,
1815						golden_settings_athub_1_0_0,
1816						ARRAY_SIZE(golden_settings_athub_1_0_0));
1817		break;
1818	case IP_VERSION(9, 1, 0):
1819	case IP_VERSION(9, 2, 0):
1820		/* TODO for renoir */
1821		soc15_program_register_sequence(adev,
1822						golden_settings_athub_1_0_0,
1823						ARRAY_SIZE(golden_settings_athub_1_0_0));
1824		break;
1825	default:
1826		break;
1827	}
1828}
1829
1830/**
1831 * gmc_v9_0_restore_registers - restores regs
1832 *
1833 * @adev: amdgpu_device pointer
1834 *
1835 * This restores register values, saved at suspend.
1836 */
1837void gmc_v9_0_restore_registers(struct amdgpu_device *adev)
1838{
1839	if ((adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 0)) ||
1840	    (adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 1))) {
1841		WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register);
1842		WARN_ON(adev->gmc.sdpif_register !=
1843			RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0));
1844	}
1845}
1846
1847/**
1848 * gmc_v9_0_gart_enable - gart enable
1849 *
1850 * @adev: amdgpu_device pointer
1851 */
1852static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
1853{
1854	int r;
1855
1856	if (adev->gmc.xgmi.connected_to_cpu)
1857		amdgpu_gmc_init_pdb0(adev);
1858
1859	if (adev->gart.bo == NULL) {
1860		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
1861		return -EINVAL;
1862	}
1863
1864	amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr);
1865	r = adev->gfxhub.funcs->gart_enable(adev);
1866	if (r)
1867		return r;
 
 
 
1868
1869	r = adev->mmhub.funcs->gart_enable(adev);
1870	if (r)
1871		return r;
1872
1873	DRM_INFO("PCIE GART of %uM enabled.\n",
1874		 (unsigned)(adev->gmc.gart_size >> 20));
1875	if (adev->gmc.pdb0_bo)
1876		DRM_INFO("PDB0 located at 0x%016llX\n",
1877				(unsigned long long)amdgpu_bo_gpu_offset(adev->gmc.pdb0_bo));
1878	DRM_INFO("PTB located at 0x%016llX\n",
1879			(unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
1880
1881	return 0;
1882}
1883
1884static int gmc_v9_0_hw_init(void *handle)
1885{
1886	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1887	bool value;
1888	int i, r;
1889
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1890	/* The sequence of these two function calls matters.*/
1891	gmc_v9_0_init_golden_registers(adev);
1892
1893	if (adev->mode_info.num_crtc) {
1894		/* Lockout access through VGA aperture*/
1895		WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
1896		/* disable VGA render */
1897		WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
1898	}
1899
1900	if (adev->mmhub.funcs->update_power_gating)
1901		adev->mmhub.funcs->update_power_gating(adev, true);
1902
1903	adev->hdp.funcs->init_registers(adev);
1904
1905	/* After HDP is initialized, flush HDP.*/
1906	adev->hdp.funcs->flush_hdp(adev, NULL);
1907
1908	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
1909		value = false;
1910	else
1911		value = true;
1912
1913	if (!amdgpu_sriov_vf(adev)) {
1914		adev->gfxhub.funcs->set_fault_enable_default(adev, value);
 
1915		adev->mmhub.funcs->set_fault_enable_default(adev, value);
1916	}
1917	for (i = 0; i < adev->num_vmhubs; ++i)
 
 
1918		gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0);
 
1919
1920	if (adev->umc.funcs && adev->umc.funcs->init_registers)
1921		adev->umc.funcs->init_registers(adev);
1922
1923	r = gmc_v9_0_gart_enable(adev);
1924	if (r)
1925		return r;
1926
1927	if (amdgpu_emu_mode == 1)
1928		return amdgpu_gmc_vram_checking(adev);
1929	else
1930		return r;
1931}
1932
1933/**
1934 * gmc_v9_0_gart_disable - gart disable
1935 *
1936 * @adev: amdgpu_device pointer
1937 *
1938 * This disables all VM page table.
1939 */
1940static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
1941{
1942	adev->gfxhub.funcs->gart_disable(adev);
 
1943	adev->mmhub.funcs->gart_disable(adev);
1944}
1945
1946static int gmc_v9_0_hw_fini(void *handle)
1947{
1948	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1949
1950	gmc_v9_0_gart_disable(adev);
1951
1952	if (amdgpu_sriov_vf(adev)) {
1953		/* full access mode, so don't touch any GMC register */
1954		DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
1955		return 0;
1956	}
1957
1958	/*
1959	 * Pair the operations did in gmc_v9_0_hw_init and thus maintain
1960	 * a correct cached state for GMC. Otherwise, the "gate" again
1961	 * operation on S3 resuming will fail due to wrong cached state.
1962	 */
1963	if (adev->mmhub.funcs->update_power_gating)
1964		adev->mmhub.funcs->update_power_gating(adev, false);
1965
1966	amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0);
1967	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
 
 
 
 
 
 
 
 
 
1968
1969	return 0;
1970}
1971
1972static int gmc_v9_0_suspend(void *handle)
1973{
1974	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1975
1976	return gmc_v9_0_hw_fini(adev);
1977}
1978
1979static int gmc_v9_0_resume(void *handle)
1980{
 
1981	int r;
1982	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1983
1984	r = gmc_v9_0_hw_init(adev);
 
 
 
 
 
 
 
 
1985	if (r)
1986		return r;
1987
1988	amdgpu_vmid_reset_all(adev);
1989
1990	return 0;
1991}
1992
1993static bool gmc_v9_0_is_idle(void *handle)
1994{
1995	/* MC is always ready in GMC v9.*/
1996	return true;
1997}
1998
1999static int gmc_v9_0_wait_for_idle(void *handle)
2000{
2001	/* There is no need to wait for MC idle in GMC v9.*/
2002	return 0;
2003}
2004
2005static int gmc_v9_0_soft_reset(void *handle)
2006{
2007	/* XXX for emulation.*/
2008	return 0;
2009}
2010
2011static int gmc_v9_0_set_clockgating_state(void *handle,
2012					enum amd_clockgating_state state)
2013{
2014	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2015
2016	adev->mmhub.funcs->set_clockgating(adev, state);
2017
2018	athub_v1_0_set_clockgating(adev, state);
2019
2020	return 0;
2021}
2022
2023static void gmc_v9_0_get_clockgating_state(void *handle, u64 *flags)
2024{
2025	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2026
2027	adev->mmhub.funcs->get_clockgating(adev, flags);
2028
2029	athub_v1_0_get_clockgating(adev, flags);
2030}
2031
2032static int gmc_v9_0_set_powergating_state(void *handle,
2033					enum amd_powergating_state state)
2034{
2035	return 0;
2036}
2037
2038const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
2039	.name = "gmc_v9_0",
2040	.early_init = gmc_v9_0_early_init,
2041	.late_init = gmc_v9_0_late_init,
2042	.sw_init = gmc_v9_0_sw_init,
2043	.sw_fini = gmc_v9_0_sw_fini,
2044	.hw_init = gmc_v9_0_hw_init,
2045	.hw_fini = gmc_v9_0_hw_fini,
2046	.suspend = gmc_v9_0_suspend,
2047	.resume = gmc_v9_0_resume,
2048	.is_idle = gmc_v9_0_is_idle,
2049	.wait_for_idle = gmc_v9_0_wait_for_idle,
2050	.soft_reset = gmc_v9_0_soft_reset,
2051	.set_clockgating_state = gmc_v9_0_set_clockgating_state,
2052	.set_powergating_state = gmc_v9_0_set_powergating_state,
2053	.get_clockgating_state = gmc_v9_0_get_clockgating_state,
2054};
2055
2056const struct amdgpu_ip_block_version gmc_v9_0_ip_block =
2057{
2058	.type = AMD_IP_BLOCK_TYPE_GMC,
2059	.major = 9,
2060	.minor = 0,
2061	.rev = 0,
2062	.funcs = &gmc_v9_0_ip_funcs,
2063};