1/*
   2 * Copyright 2016 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/firmware.h>
  25#include <linux/pci.h>
  26
  27#include <drm/drm_cache.h>
  28
  29#include "amdgpu.h"
  30#include "gmc_v9_0.h"
  31#include "amdgpu_atomfirmware.h"
  32#include "amdgpu_gem.h"
  33
  34#include "gc/gc_9_0_sh_mask.h"
  35#include "dce/dce_12_0_offset.h"
  36#include "dce/dce_12_0_sh_mask.h"
  37#include "vega10_enum.h"
  38#include "mmhub/mmhub_1_0_offset.h"
  39#include "athub/athub_1_0_sh_mask.h"
  40#include "athub/athub_1_0_offset.h"
  41#include "oss/osssys_4_0_offset.h"
  42
  43#include "soc15.h"
  44#include "soc15d.h"
  45#include "soc15_common.h"
  46#include "umc/umc_6_0_sh_mask.h"
  47
  48#include "gfxhub_v1_0.h"
  49#include "mmhub_v1_0.h"
  50#include "athub_v1_0.h"
  51#include "gfxhub_v1_1.h"
  52#include "gfxhub_v1_2.h"
  53#include "mmhub_v9_4.h"
  54#include "mmhub_v1_7.h"
  55#include "mmhub_v1_8.h"
  56#include "umc_v6_1.h"
  57#include "umc_v6_0.h"
  58#include "umc_v6_7.h"
  59#include "umc_v12_0.h"
  60#include "hdp_v4_0.h"
  61#include "mca_v3_0.h"
  62
  63#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
  64
  65#include "amdgpu_ras.h"
  66#include "amdgpu_xgmi.h"
  67
  68/* add these here since we already include dce12 headers and these are for DCN */
  69#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x055d
  70#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
  71#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT                                        0x0
  72#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT                                       0x10
  73#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
  74#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
  75#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0                                                                  0x049d
  76#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX                                                         2
  77
  78#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2                                                          0x05ea
  79#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2_BASE_IDX                                                 2
  80
  81#define MAX_MEM_RANGES 8
  82
  83static const char * const gfxhub_client_ids[] = {
  84	"CB",
  85	"DB",
  86	"IA",
  87	"WD",
  88	"CPF",
  89	"CPC",
  90	"CPG",
  91	"RLC",
  92	"TCP",
  93	"SQC (inst)",
  94	"SQC (data)",
  95	"SQG",
  96	"PA",
  97};
  98
  99static const char *mmhub_client_ids_raven[][2] = {
 100	[0][0] = "MP1",
 101	[1][0] = "MP0",
 102	[2][0] = "VCN",
 103	[3][0] = "VCNU",
 104	[4][0] = "HDP",
 105	[5][0] = "DCE",
 106	[13][0] = "UTCL2",
 107	[19][0] = "TLS",
 108	[26][0] = "OSS",
 109	[27][0] = "SDMA0",
 110	[0][1] = "MP1",
 111	[1][1] = "MP0",
 112	[2][1] = "VCN",
 113	[3][1] = "VCNU",
 114	[4][1] = "HDP",
 115	[5][1] = "XDP",
 116	[6][1] = "DBGU0",
 117	[7][1] = "DCE",
 118	[8][1] = "DCEDWB0",
 119	[9][1] = "DCEDWB1",
 120	[26][1] = "OSS",
 121	[27][1] = "SDMA0",
 122};
 123
 124static const char *mmhub_client_ids_renoir[][2] = {
 125	[0][0] = "MP1",
 126	[1][0] = "MP0",
 127	[2][0] = "HDP",
 128	[4][0] = "DCEDMC",
 129	[5][0] = "DCEVGA",
 130	[13][0] = "UTCL2",
 131	[19][0] = "TLS",
 132	[26][0] = "OSS",
 133	[27][0] = "SDMA0",
 134	[28][0] = "VCN",
 135	[29][0] = "VCNU",
 136	[30][0] = "JPEG",
 137	[0][1] = "MP1",
 138	[1][1] = "MP0",
 139	[2][1] = "HDP",
 140	[3][1] = "XDP",
 141	[6][1] = "DBGU0",
 142	[7][1] = "DCEDMC",
 143	[8][1] = "DCEVGA",
 144	[9][1] = "DCEDWB",
 145	[26][1] = "OSS",
 146	[27][1] = "SDMA0",
 147	[28][1] = "VCN",
 148	[29][1] = "VCNU",
 149	[30][1] = "JPEG",
 150};
 151
 152static const char *mmhub_client_ids_vega10[][2] = {
 153	[0][0] = "MP0",
 154	[1][0] = "UVD",
 155	[2][0] = "UVDU",
 156	[3][0] = "HDP",
 157	[13][0] = "UTCL2",
 158	[14][0] = "OSS",
 159	[15][0] = "SDMA1",
 160	[32+0][0] = "VCE0",
 161	[32+1][0] = "VCE0U",
 162	[32+2][0] = "XDMA",
 163	[32+3][0] = "DCE",
 164	[32+4][0] = "MP1",
 165	[32+14][0] = "SDMA0",
 166	[0][1] = "MP0",
 167	[1][1] = "UVD",
 168	[2][1] = "UVDU",
 169	[3][1] = "DBGU0",
 170	[4][1] = "HDP",
 171	[5][1] = "XDP",
 172	[14][1] = "OSS",
 173	[15][1] = "SDMA0",
 174	[32+0][1] = "VCE0",
 175	[32+1][1] = "VCE0U",
 176	[32+2][1] = "XDMA",
 177	[32+3][1] = "DCE",
 178	[32+4][1] = "DCEDWB",
 179	[32+5][1] = "MP1",
 180	[32+6][1] = "DBGU1",
 181	[32+14][1] = "SDMA1",
 182};
 183
 184static const char *mmhub_client_ids_vega12[][2] = {
 185	[0][0] = "MP0",
 186	[1][0] = "VCE0",
 187	[2][0] = "VCE0U",
 188	[3][0] = "HDP",
 189	[13][0] = "UTCL2",
 190	[14][0] = "OSS",
 191	[15][0] = "SDMA1",
 192	[32+0][0] = "DCE",
 193	[32+1][0] = "XDMA",
 194	[32+2][0] = "UVD",
 195	[32+3][0] = "UVDU",
 196	[32+4][0] = "MP1",
 197	[32+15][0] = "SDMA0",
 198	[0][1] = "MP0",
 199	[1][1] = "VCE0",
 200	[2][1] = "VCE0U",
 201	[3][1] = "DBGU0",
 202	[4][1] = "HDP",
 203	[5][1] = "XDP",
 204	[14][1] = "OSS",
 205	[15][1] = "SDMA0",
 206	[32+0][1] = "DCE",
 207	[32+1][1] = "DCEDWB",
 208	[32+2][1] = "XDMA",
 209	[32+3][1] = "UVD",
 210	[32+4][1] = "UVDU",
 211	[32+5][1] = "MP1",
 212	[32+6][1] = "DBGU1",
 213	[32+15][1] = "SDMA1",
 214};
 215
 216static const char *mmhub_client_ids_vega20[][2] = {
 217	[0][0] = "XDMA",
 218	[1][0] = "DCE",
 219	[2][0] = "VCE0",
 220	[3][0] = "VCE0U",
 221	[4][0] = "UVD",
 222	[5][0] = "UVD1U",
 223	[13][0] = "OSS",
 224	[14][0] = "HDP",
 225	[15][0] = "SDMA0",
 226	[32+0][0] = "UVD",
 227	[32+1][0] = "UVDU",
 228	[32+2][0] = "MP1",
 229	[32+3][0] = "MP0",
 230	[32+12][0] = "UTCL2",
 231	[32+14][0] = "SDMA1",
 232	[0][1] = "XDMA",
 233	[1][1] = "DCE",
 234	[2][1] = "DCEDWB",
 235	[3][1] = "VCE0",
 236	[4][1] = "VCE0U",
 237	[5][1] = "UVD1",
 238	[6][1] = "UVD1U",
 239	[7][1] = "DBGU0",
 240	[8][1] = "XDP",
 241	[13][1] = "OSS",
 242	[14][1] = "HDP",
 243	[15][1] = "SDMA0",
 244	[32+0][1] = "UVD",
 245	[32+1][1] = "UVDU",
 246	[32+2][1] = "DBGU1",
 247	[32+3][1] = "MP1",
 248	[32+4][1] = "MP0",
 249	[32+14][1] = "SDMA1",
 250};
 251
 252static const char *mmhub_client_ids_arcturus[][2] = {
 253	[0][0] = "DBGU1",
 254	[1][0] = "XDP",
 255	[2][0] = "MP1",
 256	[14][0] = "HDP",
 257	[171][0] = "JPEG",
 258	[172][0] = "VCN",
 259	[173][0] = "VCNU",
 260	[203][0] = "JPEG1",
 261	[204][0] = "VCN1",
 262	[205][0] = "VCN1U",
 263	[256][0] = "SDMA0",
 264	[257][0] = "SDMA1",
 265	[258][0] = "SDMA2",
 266	[259][0] = "SDMA3",
 267	[260][0] = "SDMA4",
 268	[261][0] = "SDMA5",
 269	[262][0] = "SDMA6",
 270	[263][0] = "SDMA7",
 271	[384][0] = "OSS",
 272	[0][1] = "DBGU1",
 273	[1][1] = "XDP",
 274	[2][1] = "MP1",
 275	[14][1] = "HDP",
 276	[171][1] = "JPEG",
 277	[172][1] = "VCN",
 278	[173][1] = "VCNU",
 279	[203][1] = "JPEG1",
 280	[204][1] = "VCN1",
 281	[205][1] = "VCN1U",
 282	[256][1] = "SDMA0",
 283	[257][1] = "SDMA1",
 284	[258][1] = "SDMA2",
 285	[259][1] = "SDMA3",
 286	[260][1] = "SDMA4",
 287	[261][1] = "SDMA5",
 288	[262][1] = "SDMA6",
 289	[263][1] = "SDMA7",
 290	[384][1] = "OSS",
 291};
 292
 293static const char *mmhub_client_ids_aldebaran[][2] = {
 294	[2][0] = "MP1",
 295	[3][0] = "MP0",
 296	[32+1][0] = "DBGU_IO0",
 297	[32+2][0] = "DBGU_IO2",
 298	[32+4][0] = "MPIO",
 299	[96+11][0] = "JPEG0",
 300	[96+12][0] = "VCN0",
 301	[96+13][0] = "VCNU0",
 302	[128+11][0] = "JPEG1",
 303	[128+12][0] = "VCN1",
 304	[128+13][0] = "VCNU1",
 305	[160+1][0] = "XDP",
 306	[160+14][0] = "HDP",
 307	[256+0][0] = "SDMA0",
 308	[256+1][0] = "SDMA1",
 309	[256+2][0] = "SDMA2",
 310	[256+3][0] = "SDMA3",
 311	[256+4][0] = "SDMA4",
 312	[384+0][0] = "OSS",
 313	[2][1] = "MP1",
 314	[3][1] = "MP0",
 315	[32+1][1] = "DBGU_IO0",
 316	[32+2][1] = "DBGU_IO2",
 317	[32+4][1] = "MPIO",
 318	[96+11][1] = "JPEG0",
 319	[96+12][1] = "VCN0",
 320	[96+13][1] = "VCNU0",
 321	[128+11][1] = "JPEG1",
 322	[128+12][1] = "VCN1",
 323	[128+13][1] = "VCNU1",
 324	[160+1][1] = "XDP",
 325	[160+14][1] = "HDP",
 326	[256+0][1] = "SDMA0",
 327	[256+1][1] = "SDMA1",
 328	[256+2][1] = "SDMA2",
 329	[256+3][1] = "SDMA3",
 330	[256+4][1] = "SDMA4",
 331	[384+0][1] = "OSS",
 332};
 333
 334static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = {
 
 335	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
 336	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
 337};
 338
 339static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = {
 
 340	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
 341	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
 342};
 343
 344static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = {
 345	(0x000143c0 + 0x00000000),
 346	(0x000143c0 + 0x00000800),
 347	(0x000143c0 + 0x00001000),
 348	(0x000143c0 + 0x00001800),
 349	(0x000543c0 + 0x00000000),
 350	(0x000543c0 + 0x00000800),
 351	(0x000543c0 + 0x00001000),
 352	(0x000543c0 + 0x00001800),
 353	(0x000943c0 + 0x00000000),
 354	(0x000943c0 + 0x00000800),
 355	(0x000943c0 + 0x00001000),
 356	(0x000943c0 + 0x00001800),
 357	(0x000d43c0 + 0x00000000),
 358	(0x000d43c0 + 0x00000800),
 359	(0x000d43c0 + 0x00001000),
 360	(0x000d43c0 + 0x00001800),
 361	(0x001143c0 + 0x00000000),
 362	(0x001143c0 + 0x00000800),
 363	(0x001143c0 + 0x00001000),
 364	(0x001143c0 + 0x00001800),
 365	(0x001543c0 + 0x00000000),
 366	(0x001543c0 + 0x00000800),
 367	(0x001543c0 + 0x00001000),
 368	(0x001543c0 + 0x00001800),
 369	(0x001943c0 + 0x00000000),
 370	(0x001943c0 + 0x00000800),
 371	(0x001943c0 + 0x00001000),
 372	(0x001943c0 + 0x00001800),
 373	(0x001d43c0 + 0x00000000),
 374	(0x001d43c0 + 0x00000800),
 375	(0x001d43c0 + 0x00001000),
 376	(0x001d43c0 + 0x00001800),
 377};
 378
 379static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = {
 380	(0x000143e0 + 0x00000000),
 381	(0x000143e0 + 0x00000800),
 382	(0x000143e0 + 0x00001000),
 383	(0x000143e0 + 0x00001800),
 384	(0x000543e0 + 0x00000000),
 385	(0x000543e0 + 0x00000800),
 386	(0x000543e0 + 0x00001000),
 387	(0x000543e0 + 0x00001800),
 388	(0x000943e0 + 0x00000000),
 389	(0x000943e0 + 0x00000800),
 390	(0x000943e0 + 0x00001000),
 391	(0x000943e0 + 0x00001800),
 392	(0x000d43e0 + 0x00000000),
 393	(0x000d43e0 + 0x00000800),
 394	(0x000d43e0 + 0x00001000),
 395	(0x000d43e0 + 0x00001800),
 396	(0x001143e0 + 0x00000000),
 397	(0x001143e0 + 0x00000800),
 398	(0x001143e0 + 0x00001000),
 399	(0x001143e0 + 0x00001800),
 400	(0x001543e0 + 0x00000000),
 401	(0x001543e0 + 0x00000800),
 402	(0x001543e0 + 0x00001000),
 403	(0x001543e0 + 0x00001800),
 404	(0x001943e0 + 0x00000000),
 405	(0x001943e0 + 0x00000800),
 406	(0x001943e0 + 0x00001000),
 407	(0x001943e0 + 0x00001800),
 408	(0x001d43e0 + 0x00000000),
 409	(0x001d43e0 + 0x00000800),
 410	(0x001d43e0 + 0x00001000),
 411	(0x001d43e0 + 0x00001800),
 412};
 413
 414static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev,
 415		struct amdgpu_irq_src *src,
 416		unsigned int type,
 417		enum amdgpu_interrupt_state state)
 418{
 419	u32 bits, i, tmp, reg;
 420
 421	/* Devices newer then VEGA10/12 shall have these programming
 422	 * sequences performed by PSP BL
 423	 */
 424	if (adev->asic_type >= CHIP_VEGA20)
 425		return 0;
 426
 427	bits = 0x7f;
 428
 429	switch (state) {
 430	case AMDGPU_IRQ_STATE_DISABLE:
 431		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
 432			reg = ecc_umc_mcumc_ctrl_addrs[i];
 433			tmp = RREG32(reg);
 434			tmp &= ~bits;
 435			WREG32(reg, tmp);
 436		}
 437		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
 438			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
 439			tmp = RREG32(reg);
 440			tmp &= ~bits;
 441			WREG32(reg, tmp);
 442		}
 443		break;
 444	case AMDGPU_IRQ_STATE_ENABLE:
 445		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
 446			reg = ecc_umc_mcumc_ctrl_addrs[i];
 447			tmp = RREG32(reg);
 448			tmp |= bits;
 449			WREG32(reg, tmp);
 450		}
 451		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
 452			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
 453			tmp = RREG32(reg);
 454			tmp |= bits;
 455			WREG32(reg, tmp);
 456		}
 457		break;
 458	default:
 459		break;
 460	}
 461
 462	return 0;
 463}
 464
 465static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
 466					struct amdgpu_irq_src *src,
 467					unsigned int type,
 468					enum amdgpu_interrupt_state state)
 469{
 470	struct amdgpu_vmhub *hub;
 471	u32 tmp, reg, bits, i, j;
 472
 473	bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
 474		VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
 475		VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
 476		VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
 477		VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
 478		VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
 479		VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
 480
 481	switch (state) {
 482	case AMDGPU_IRQ_STATE_DISABLE:
 483		for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
 484			hub = &adev->vmhub[j];
 485			for (i = 0; i < 16; i++) {
 486				reg = hub->vm_context0_cntl + i;
 487
 488				/* This works because this interrupt is only
 489				 * enabled at init/resume and disabled in
 490				 * fini/suspend, so the overall state doesn't
 491				 * change over the course of suspend/resume.
 492				 */
 493				if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
 494					continue;
 495
 496				if (j >= AMDGPU_MMHUB0(0))
 497					tmp = RREG32_SOC15_IP(MMHUB, reg);
 498				else
 499					tmp = RREG32_SOC15_IP(GC, reg);
 500
 501				tmp &= ~bits;
 502
 503				if (j >= AMDGPU_MMHUB0(0))
 504					WREG32_SOC15_IP(MMHUB, reg, tmp);
 505				else
 506					WREG32_SOC15_IP(GC, reg, tmp);
 507			}
 508		}
 509		break;
 510	case AMDGPU_IRQ_STATE_ENABLE:
 511		for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
 512			hub = &adev->vmhub[j];
 513			for (i = 0; i < 16; i++) {
 514				reg = hub->vm_context0_cntl + i;
 515
 516				/* This works because this interrupt is only
 517				 * enabled at init/resume and disabled in
 518				 * fini/suspend, so the overall state doesn't
 519				 * change over the course of suspend/resume.
 520				 */
 521				if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
 522					continue;
 523
 524				if (j >= AMDGPU_MMHUB0(0))
 525					tmp = RREG32_SOC15_IP(MMHUB, reg);
 526				else
 527					tmp = RREG32_SOC15_IP(GC, reg);
 528
 529				tmp |= bits;
 530
 531				if (j >= AMDGPU_MMHUB0(0))
 532					WREG32_SOC15_IP(MMHUB, reg, tmp);
 533				else
 534					WREG32_SOC15_IP(GC, reg, tmp);
 535			}
 536		}
 537		break;
 538	default:
 539		break;
 540	}
 541
 542	return 0;
 543}
 544
 545static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
 546				      struct amdgpu_irq_src *source,
 547				      struct amdgpu_iv_entry *entry)
 548{
 549	bool retry_fault = !!(entry->src_data[1] & 0x80);
 550	bool write_fault = !!(entry->src_data[1] & 0x20);
 551	uint32_t status = 0, cid = 0, rw = 0;
 552	struct amdgpu_task_info task_info;
 553	struct amdgpu_vmhub *hub;
 554	const char *mmhub_cid;
 555	const char *hub_name;
 556	unsigned int vmhub;
 557	u64 addr;
 558	uint32_t cam_index = 0;
 559	int ret, xcc_id = 0;
 560	uint32_t node_id;
 561
 562	node_id = entry->node_id;
 563
 564	addr = (u64)entry->src_data[0] << 12;
 565	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
 566
 567	if (entry->client_id == SOC15_IH_CLIENTID_VMC) {
 568		hub_name = "mmhub0";
 569		vmhub = AMDGPU_MMHUB0(node_id / 4);
 570	} else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) {
 571		hub_name = "mmhub1";
 572		vmhub = AMDGPU_MMHUB1(0);
 573	} else {
 574		hub_name = "gfxhub0";
 575		if (adev->gfx.funcs->ih_node_to_logical_xcc) {
 576			xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev,
 577				node_id);
 578			if (xcc_id < 0)
 579				xcc_id = 0;
 580		}
 581		vmhub = xcc_id;
 582	}
 583	hub = &adev->vmhub[vmhub];
 584
 585	if (retry_fault) {
 586		if (adev->irq.retry_cam_enabled) {
 587			/* Delegate it to a different ring if the hardware hasn't
 588			 * already done it.
 589			 */
 590			if (entry->ih == &adev->irq.ih) {
 591				amdgpu_irq_delegate(adev, entry, 8);
 592				return 1;
 593			}
 594
 595			cam_index = entry->src_data[2] & 0x3ff;
 596
 597			ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
 598						     addr, write_fault);
 599			WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index);
 600			if (ret)
 601				return 1;
 602		} else {
 603			/* Process it onyl if it's the first fault for this address */
 604			if (entry->ih != &adev->irq.ih_soft &&
 605			    amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
 606					     entry->timestamp))
 607				return 1;
 608
 609			/* Delegate it to a different ring if the hardware hasn't
 610			 * already done it.
 611			 */
 612			if (entry->ih == &adev->irq.ih) {
 613				amdgpu_irq_delegate(adev, entry, 8);
 614				return 1;
 615			}
 616
 617			/* Try to handle the recoverable page faults by filling page
 618			 * tables
 619			 */
 620			if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
 621						   addr, write_fault))
 622				return 1;
 623		}
 
 
 
 
 
 
 624	}
 625
 626	if (!printk_ratelimit())
 627		return 0;
 628
 
 
 
 
 
 
 
 
 
 
 
 629	memset(&task_info, 0, sizeof(struct amdgpu_task_info));
 630	amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
 631
 632	dev_err(adev->dev,
 633		"[%s] %s page fault (src_id:%u ring:%u vmid:%u pasid:%u, for process %s pid %d thread %s pid %d)\n",
 
 634		hub_name, retry_fault ? "retry" : "no-retry",
 635		entry->src_id, entry->ring_id, entry->vmid,
 636		entry->pasid, task_info.process_name, task_info.tgid,
 637		task_info.task_name, task_info.pid);
 638	dev_err(adev->dev, "  in page starting at address 0x%016llx from IH client 0x%x (%s)\n",
 639		addr, entry->client_id,
 640		soc15_ih_clientid_name[entry->client_id]);
 641
 642	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3))
 643		dev_err(adev->dev, "  cookie node_id %d fault from die %s%d%s\n",
 644			node_id, node_id % 4 == 3 ? "RSV" : "AID", node_id / 4,
 645			node_id % 4 == 1 ? ".XCD0" : node_id % 4 == 2 ? ".XCD1" : "");
 646
 647	if (amdgpu_sriov_vf(adev))
 648		return 0;
 649
 650	/*
 651	 * Issue a dummy read to wait for the status register to
 652	 * be updated to avoid reading an incorrect value due to
 653	 * the new fast GRBM interface.
 654	 */
 655	if ((entry->vmid_src == AMDGPU_GFXHUB(0)) &&
 656	    (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(9, 4, 2)))
 657		RREG32(hub->vm_l2_pro_fault_status);
 658
 659	status = RREG32(hub->vm_l2_pro_fault_status);
 660	cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID);
 661	rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW);
 662	WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
 663
 664	amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, vmhub);
 665
 666	dev_err(adev->dev,
 667		"VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
 668		status);
 669	if (entry->vmid_src == AMDGPU_GFXHUB(0)) {
 670		dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
 671			cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" :
 672			gfxhub_client_ids[cid],
 673			cid);
 674	} else {
 675		switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
 676		case IP_VERSION(9, 0, 0):
 677			mmhub_cid = mmhub_client_ids_vega10[cid][rw];
 678			break;
 679		case IP_VERSION(9, 3, 0):
 680			mmhub_cid = mmhub_client_ids_vega12[cid][rw];
 681			break;
 682		case IP_VERSION(9, 4, 0):
 683			mmhub_cid = mmhub_client_ids_vega20[cid][rw];
 684			break;
 685		case IP_VERSION(9, 4, 1):
 686			mmhub_cid = mmhub_client_ids_arcturus[cid][rw];
 687			break;
 688		case IP_VERSION(9, 1, 0):
 689		case IP_VERSION(9, 2, 0):
 690			mmhub_cid = mmhub_client_ids_raven[cid][rw];
 691			break;
 692		case IP_VERSION(1, 5, 0):
 693		case IP_VERSION(2, 4, 0):
 694			mmhub_cid = mmhub_client_ids_renoir[cid][rw];
 695			break;
 696		case IP_VERSION(1, 8, 0):
 697		case IP_VERSION(9, 4, 2):
 698			mmhub_cid = mmhub_client_ids_aldebaran[cid][rw];
 699			break;
 700		default:
 701			mmhub_cid = NULL;
 702			break;
 703		}
 704		dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
 705			mmhub_cid ? mmhub_cid : "unknown", cid);
 706	}
 707	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
 708		REG_GET_FIELD(status,
 709		VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
 710	dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
 711		REG_GET_FIELD(status,
 712		VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
 713	dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
 714		REG_GET_FIELD(status,
 715		VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
 716	dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
 717		REG_GET_FIELD(status,
 718		VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
 719	dev_err(adev->dev, "\t RW: 0x%x\n", rw);
 720	return 0;
 721}
 722
 723static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
 724	.set = gmc_v9_0_vm_fault_interrupt_state,
 725	.process = gmc_v9_0_process_interrupt,
 726};
 727
 728
 729static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = {
 730	.set = gmc_v9_0_ecc_interrupt_state,
 731	.process = amdgpu_umc_process_ecc_irq,
 732};
 733
 734static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
 735{
 736	adev->gmc.vm_fault.num_types = 1;
 737	adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
 738
 739	if (!amdgpu_sriov_vf(adev) &&
 740	    !adev->gmc.xgmi.connected_to_cpu &&
 741	    !adev->gmc.is_app_apu) {
 742		adev->gmc.ecc_irq.num_types = 1;
 743		adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs;
 744	}
 745}
 746
 747static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid,
 748					uint32_t flush_type)
 749{
 750	u32 req = 0;
 751
 752	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
 753			    PER_VMID_INVALIDATE_REQ, 1 << vmid);
 754	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
 755	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
 756	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
 757	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
 758	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
 759	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
 760	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
 761			    CLEAR_PROTECTION_FAULT_STATUS_ADDR,	0);
 762
 763	return req;
 764}
 765
 766/**
 767 * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore
 768 *
 769 * @adev: amdgpu_device pointer
 770 * @vmhub: vmhub type
 771 *
 772 */
 773static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev,
 774				       uint32_t vmhub)
 775{
 776	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
 777	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3))
 778		return false;
 779
 780	return ((vmhub == AMDGPU_MMHUB0(0) ||
 781		 vmhub == AMDGPU_MMHUB1(0)) &&
 782		(!amdgpu_sriov_vf(adev)) &&
 783		(!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) &&
 784		   (adev->apu_flags & AMD_APU_IS_PICASSO))));
 785}
 786
 787static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
 788					uint8_t vmid, uint16_t *p_pasid)
 789{
 790	uint32_t value;
 791
 792	value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
 793		     + vmid);
 794	*p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
 795
 796	return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
 797}
 798
 799/*
 800 * GART
 801 * VMID 0 is the physical GPU addresses as used by the kernel.
 802 * VMIDs 1-15 are used for userspace clients and are handled
 803 * by the amdgpu vm/hsa code.
 804 */
 805
 806/**
 807 * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type
 808 *
 809 * @adev: amdgpu_device pointer
 810 * @vmid: vm instance to flush
 811 * @vmhub: which hub to flush
 812 * @flush_type: the flush type
 813 *
 814 * Flush the TLB for the requested page table using certain type.
 815 */
 816static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 817					uint32_t vmhub, uint32_t flush_type)
 818{
 819	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub);
 820	u32 j, inv_req, tmp, sem, req, ack, inst;
 821	const unsigned int eng = 17;
 822	struct amdgpu_vmhub *hub;
 823
 824	BUG_ON(vmhub >= AMDGPU_MAX_VMHUBS);
 825
 826	hub = &adev->vmhub[vmhub];
 827	inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type);
 828	sem = hub->vm_inv_eng0_sem + hub->eng_distance * eng;
 829	req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
 830	ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
 
 
 
 
 
 
 
 
 
 
 831
 832	/* This is necessary for a HW workaround under SRIOV as well
 833	 * as GFXOFF under bare metal
 834	 */
 835	if (vmhub >= AMDGPU_MMHUB0(0))
 836		inst = GET_INST(GC, 0);
 837	else
 838		inst = vmhub;
 839	if (adev->gfx.kiq[inst].ring.sched.ready &&
 840	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) {
 841		uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
 842		uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
 843
 844		amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req,
 845						   1 << vmid, inst);
 
 846		return;
 847	}
 848
 849	spin_lock(&adev->gmc.invalidate_lock);
 850
 851	/*
 852	 * It may lose gpuvm invalidate acknowldege state across power-gating
 853	 * off cycle, add semaphore acquire before invalidation and semaphore
 854	 * release after invalidation to avoid entering power gated state
 855	 * to WA the Issue
 856	 */
 857
 858	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
 859	if (use_semaphore) {
 860		for (j = 0; j < adev->usec_timeout; j++) {
 861			/* a read return value of 1 means semaphore acquire */
 862			if (vmhub >= AMDGPU_MMHUB0(0))
 863				tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, sem, inst);
 864			else
 865				tmp = RREG32_SOC15_IP_NO_KIQ(GC, sem, inst);
 866			if (tmp & 0x1)
 867				break;
 868			udelay(1);
 869		}
 870
 871		if (j >= adev->usec_timeout)
 872			DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
 873	}
 874
 875	if (vmhub >= AMDGPU_MMHUB0(0))
 876		WREG32_SOC15_IP_NO_KIQ(MMHUB, req, inv_req, inst);
 877	else
 878		WREG32_SOC15_IP_NO_KIQ(GC, req, inv_req, inst);
 879
 880	/*
 881	 * Issue a dummy read to wait for the ACK register to
 882	 * be cleared to avoid a false ACK due to the new fast
 883	 * GRBM interface.
 884	 */
 885	if ((vmhub == AMDGPU_GFXHUB(0)) &&
 886	    (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(9, 4, 2)))
 887		RREG32_NO_KIQ(req);
 888
 889	for (j = 0; j < adev->usec_timeout; j++) {
 890		if (vmhub >= AMDGPU_MMHUB0(0))
 891			tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, ack, inst);
 892		else
 893			tmp = RREG32_SOC15_IP_NO_KIQ(GC, ack, inst);
 894		if (tmp & (1 << vmid))
 895			break;
 896		udelay(1);
 897	}
 
 
 
 898
 899	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
 900	if (use_semaphore) {
 901		/*
 902		 * add semaphore release after invalidation,
 903		 * write with 0 means semaphore release
 904		 */
 905		if (vmhub >= AMDGPU_MMHUB0(0))
 906			WREG32_SOC15_IP_NO_KIQ(MMHUB, sem, 0, inst);
 907		else
 908			WREG32_SOC15_IP_NO_KIQ(GC, sem, 0, inst);
 909	}
 910
 911	spin_unlock(&adev->gmc.invalidate_lock);
 912
 913	if (j < adev->usec_timeout)
 914		return;
 915
 916	DRM_ERROR("Timeout waiting for VM flush ACK!\n");
 917}
 918
 919/**
 920 * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid
 921 *
 922 * @adev: amdgpu_device pointer
 923 * @pasid: pasid to be flush
 924 * @flush_type: the flush type
 925 * @all_hub: flush all hubs
 926 * @inst: is used to select which instance of KIQ to use for the invalidation
 927 *
 928 * Flush the TLB for the requested pasid.
 929 */
 930static void gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
 931					 uint16_t pasid, uint32_t flush_type,
 932					 bool all_hub, uint32_t inst)
 933{
 934	uint16_t queried;
 935	int i, vmid;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 936
 937	for (vmid = 1; vmid < 16; vmid++) {
 938		bool valid;
 939
 940		valid = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid,
 941								 &queried);
 942		if (!valid || queried != pasid)
 943			continue;
 944
 945		if (all_hub) {
 946			for_each_set_bit(i, adev->vmhubs_mask,
 947					 AMDGPU_MAX_VMHUBS)
 948				gmc_v9_0_flush_gpu_tlb(adev, vmid, i,
 949						       flush_type);
 950		} else {
 951			gmc_v9_0_flush_gpu_tlb(adev, vmid,
 952					       AMDGPU_GFXHUB(0),
 953					       flush_type);
 954		}
 955	}
 
 
 
 956}
 957
 958static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
 959					    unsigned int vmid, uint64_t pd_addr)
 960{
 961	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->vm_hub);
 962	struct amdgpu_device *adev = ring->adev;
 963	struct amdgpu_vmhub *hub = &adev->vmhub[ring->vm_hub];
 964	uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0);
 965	unsigned int eng = ring->vm_inv_eng;
 966
 967	/*
 968	 * It may lose gpuvm invalidate acknowldege state across power-gating
 969	 * off cycle, add semaphore acquire before invalidation and semaphore
 970	 * release after invalidation to avoid entering power gated state
 971	 * to WA the Issue
 972	 */
 973
 974	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
 975	if (use_semaphore)
 976		/* a read return value of 1 means semaphore acuqire */
 977		amdgpu_ring_emit_reg_wait(ring,
 978					  hub->vm_inv_eng0_sem +
 979					  hub->eng_distance * eng, 0x1, 0x1);
 980
 981	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
 982			      (hub->ctx_addr_distance * vmid),
 983			      lower_32_bits(pd_addr));
 984
 985	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
 986			      (hub->ctx_addr_distance * vmid),
 987			      upper_32_bits(pd_addr));
 988
 989	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
 990					    hub->eng_distance * eng,
 991					    hub->vm_inv_eng0_ack +
 992					    hub->eng_distance * eng,
 993					    req, 1 << vmid);
 994
 995	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
 996	if (use_semaphore)
 997		/*
 998		 * add semaphore release after invalidation,
 999		 * write with 0 means semaphore release
1000		 */
1001		amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
1002				      hub->eng_distance * eng, 0);
1003
1004	return pd_addr;
1005}
1006
1007static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int vmid,
1008					unsigned int pasid)
1009{
1010	struct amdgpu_device *adev = ring->adev;
1011	uint32_t reg;
1012
1013	/* Do nothing because there's no lut register for mmhub1. */
1014	if (ring->vm_hub == AMDGPU_MMHUB1(0))
1015		return;
1016
1017	if (ring->vm_hub == AMDGPU_GFXHUB(0))
1018		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
1019	else
1020		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
1021
1022	amdgpu_ring_emit_wreg(ring, reg, pasid);
1023}
1024
1025/*
1026 * PTE format on VEGA 10:
1027 * 63:59 reserved
1028 * 58:57 mtype
1029 * 56 F
1030 * 55 L
1031 * 54 P
1032 * 53 SW
1033 * 52 T
1034 * 50:48 reserved
1035 * 47:12 4k physical page base address
1036 * 11:7 fragment
1037 * 6 write
1038 * 5 read
1039 * 4 exe
1040 * 3 Z
1041 * 2 snooped
1042 * 1 system
1043 * 0 valid
1044 *
1045 * PDE format on VEGA 10:
1046 * 63:59 block fragment size
1047 * 58:55 reserved
1048 * 54 P
1049 * 53:48 reserved
1050 * 47:6 physical base address of PD or PTE
1051 * 5:3 reserved
1052 * 2 C
1053 * 1 system
1054 * 0 valid
1055 */
1056
1057static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
1058
1059{
1060	switch (flags) {
1061	case AMDGPU_VM_MTYPE_DEFAULT:
1062		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1063	case AMDGPU_VM_MTYPE_NC:
1064		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1065	case AMDGPU_VM_MTYPE_WC:
1066		return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC);
1067	case AMDGPU_VM_MTYPE_RW:
1068		return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW);
1069	case AMDGPU_VM_MTYPE_CC:
1070		return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC);
1071	case AMDGPU_VM_MTYPE_UC:
1072		return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC);
1073	default:
1074		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1075	}
1076}
1077
1078static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
1079				uint64_t *addr, uint64_t *flags)
1080{
1081	if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
1082		*addr = amdgpu_gmc_vram_mc2pa(adev, *addr);
1083	BUG_ON(*addr & 0xFFFF00000000003FULL);
1084
1085	if (!adev->gmc.translate_further)
1086		return;
1087
1088	if (level == AMDGPU_VM_PDB1) {
1089		/* Set the block fragment size */
1090		if (!(*flags & AMDGPU_PDE_PTE))
1091			*flags |= AMDGPU_PDE_BFS(0x9);
1092
1093	} else if (level == AMDGPU_VM_PDB0) {
1094		if (*flags & AMDGPU_PDE_PTE) {
1095			*flags &= ~AMDGPU_PDE_PTE;
1096			if (!(*flags & AMDGPU_PTE_VALID))
1097				*addr |= 1 << PAGE_SHIFT;
1098		} else {
1099			*flags |= AMDGPU_PTE_TF;
1100		}
1101	}
1102}
1103
1104static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev,
1105					 struct amdgpu_bo *bo,
1106					 struct amdgpu_bo_va_mapping *mapping,
1107					 uint64_t *flags)
1108{
1109	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
1110	bool is_vram = bo->tbo.resource->mem_type == TTM_PL_VRAM;
1111	bool coherent = bo->flags & (AMDGPU_GEM_CREATE_COHERENT | AMDGPU_GEM_CREATE_EXT_COHERENT);
1112	bool ext_coherent = bo->flags & AMDGPU_GEM_CREATE_EXT_COHERENT;
1113	bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED;
1114	struct amdgpu_vm *vm = mapping->bo_va->base.vm;
1115	unsigned int mtype_local, mtype;
1116	bool snoop = false;
1117	bool is_local;
1118
1119	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1120	case IP_VERSION(9, 4, 1):
1121	case IP_VERSION(9, 4, 2):
1122		if (is_vram) {
1123			if (bo_adev == adev) {
1124				if (uncached)
1125					mtype = MTYPE_UC;
1126				else if (coherent)
1127					mtype = MTYPE_CC;
1128				else
1129					mtype = MTYPE_RW;
1130				/* FIXME: is this still needed? Or does
1131				 * amdgpu_ttm_tt_pde_flags already handle this?
1132				 */
1133				if ((amdgpu_ip_version(adev, GC_HWIP, 0) ==
1134					     IP_VERSION(9, 4, 2) ||
1135				     amdgpu_ip_version(adev, GC_HWIP, 0) ==
1136					     IP_VERSION(9, 4, 3)) &&
1137				    adev->gmc.xgmi.connected_to_cpu)
1138					snoop = true;
1139			} else {
1140				if (uncached || coherent)
1141					mtype = MTYPE_UC;
1142				else
1143					mtype = MTYPE_NC;
1144				if (mapping->bo_va->is_xgmi)
1145					snoop = true;
1146			}
1147		} else {
1148			if (uncached || coherent)
1149				mtype = MTYPE_UC;
1150			else
1151				mtype = MTYPE_NC;
1152			/* FIXME: is this still needed? Or does
1153			 * amdgpu_ttm_tt_pde_flags already handle this?
1154			 */
1155			snoop = true;
1156		}
1157		break;
1158	case IP_VERSION(9, 4, 3):
1159		/* Only local VRAM BOs or system memory on non-NUMA APUs
1160		 * can be assumed to be local in their entirety. Choose
1161		 * MTYPE_NC as safe fallback for all system memory BOs on
1162		 * NUMA systems. Their MTYPE can be overridden per-page in
1163		 * gmc_v9_0_override_vm_pte_flags.
1164		 */
1165		mtype_local = MTYPE_RW;
1166		if (amdgpu_mtype_local == 1) {
1167			DRM_INFO_ONCE("Using MTYPE_NC for local memory\n");
1168			mtype_local = MTYPE_NC;
1169		} else if (amdgpu_mtype_local == 2) {
1170			DRM_INFO_ONCE("Using MTYPE_CC for local memory\n");
1171			mtype_local = MTYPE_CC;
1172		} else {
1173			DRM_INFO_ONCE("Using MTYPE_RW for local memory\n");
1174		}
1175		is_local = (!is_vram && (adev->flags & AMD_IS_APU) &&
1176			    num_possible_nodes() <= 1) ||
1177			   (is_vram && adev == bo_adev &&
1178			    KFD_XCP_MEM_ID(adev, bo->xcp_id) == vm->mem_id);
1179		snoop = true;
1180		if (uncached) {
1181			mtype = MTYPE_UC;
1182		} else if (ext_coherent) {
1183			if (adev->rev_id)
1184				mtype = is_local ? MTYPE_CC : MTYPE_UC;
1185			else
1186				mtype = MTYPE_UC;
1187		} else if (adev->flags & AMD_IS_APU) {
1188			mtype = is_local ? mtype_local : MTYPE_NC;
1189		} else {
1190			/* dGPU */
1191			if (is_local)
1192				mtype = mtype_local;
1193			else if (is_vram)
1194				mtype = MTYPE_NC;
1195			else
1196				mtype = MTYPE_UC;
1197		}
1198
1199		break;
1200	default:
1201		if (uncached || coherent)
1202			mtype = MTYPE_UC;
1203		else
1204			mtype = MTYPE_NC;
1205
1206		/* FIXME: is this still needed? Or does
1207		 * amdgpu_ttm_tt_pde_flags already handle this?
1208		 */
1209		if (!is_vram)
1210			snoop = true;
1211	}
1212
1213	if (mtype != MTYPE_NC)
1214		*flags = (*flags & ~AMDGPU_PTE_MTYPE_VG10_MASK) |
1215			 AMDGPU_PTE_MTYPE_VG10(mtype);
1216	*flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;
1217}
1218
1219static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
1220				struct amdgpu_bo_va_mapping *mapping,
1221				uint64_t *flags)
1222{
1223	struct amdgpu_bo *bo = mapping->bo_va->base.bo;
1224
1225	*flags &= ~AMDGPU_PTE_EXECUTABLE;
1226	*flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1227
1228	*flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1229	*flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK;
1230
1231	if (mapping->flags & AMDGPU_PTE_PRT) {
1232		*flags |= AMDGPU_PTE_PRT;
1233		*flags &= ~AMDGPU_PTE_VALID;
1234	}
1235
1236	if (bo && bo->tbo.resource)
1237		gmc_v9_0_get_coherence_flags(adev, mapping->bo_va->base.bo,
1238					     mapping, flags);
1239}
1240
1241static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
1242					   struct amdgpu_vm *vm,
1243					   uint64_t addr, uint64_t *flags)
1244{
1245	int local_node, nid;
1246
1247	/* Only GFX 9.4.3 APUs associate GPUs with NUMA nodes. Local system
1248	 * memory can use more efficient MTYPEs.
1249	 */
1250	if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 3))
1251		return;
1252
1253	/* Only direct-mapped memory allows us to determine the NUMA node from
1254	 * the DMA address.
1255	 */
1256	if (!adev->ram_is_direct_mapped) {
1257		dev_dbg_ratelimited(adev->dev, "RAM is not direct mapped\n");
1258		return;
1259	}
1260
1261	/* MTYPE_NC is the same default and can be overridden.
1262	 * MTYPE_UC will be present if the memory is extended-coherent
1263	 * and can also be overridden.
1264	 */
1265	if ((*flags & AMDGPU_PTE_MTYPE_VG10_MASK) !=
1266	    AMDGPU_PTE_MTYPE_VG10(MTYPE_NC) &&
1267	    (*flags & AMDGPU_PTE_MTYPE_VG10_MASK) !=
1268	    AMDGPU_PTE_MTYPE_VG10(MTYPE_UC)) {
1269		dev_dbg_ratelimited(adev->dev, "MTYPE is not NC or UC\n");
1270		return;
1271	}
1272
1273	/* FIXME: Only supported on native mode for now. For carve-out, the
1274	 * NUMA affinity of the GPU/VM needs to come from the PCI info because
1275	 * memory partitions are not associated with different NUMA nodes.
1276	 */
1277	if (adev->gmc.is_app_apu && vm->mem_id >= 0) {
1278		local_node = adev->gmc.mem_partitions[vm->mem_id].numa.node;
1279	} else {
1280		dev_dbg_ratelimited(adev->dev, "Only native mode APU is supported.\n");
1281		return;
1282	}
1283
1284	/* Only handle real RAM. Mappings of PCIe resources don't have struct
1285	 * page or NUMA nodes.
1286	 */
1287	if (!page_is_ram(addr >> PAGE_SHIFT)) {
1288		dev_dbg_ratelimited(adev->dev, "Page is not RAM.\n");
1289		return;
1290	}
1291	nid = pfn_to_nid(addr >> PAGE_SHIFT);
1292	dev_dbg_ratelimited(adev->dev, "vm->mem_id=%d, local_node=%d, nid=%d\n",
1293			    vm->mem_id, local_node, nid);
1294	if (nid == local_node) {
1295		uint64_t old_flags = *flags;
1296		if ((*flags & AMDGPU_PTE_MTYPE_VG10_MASK) ==
1297			AMDGPU_PTE_MTYPE_VG10(MTYPE_NC)) {
1298			unsigned int mtype_local = MTYPE_RW;
1299
1300			if (amdgpu_mtype_local == 1)
1301				mtype_local = MTYPE_NC;
1302			else if (amdgpu_mtype_local == 2)
1303				mtype_local = MTYPE_CC;
1304
1305			*flags = (*flags & ~AMDGPU_PTE_MTYPE_VG10_MASK) |
1306				 AMDGPU_PTE_MTYPE_VG10(mtype_local);
1307		} else if (adev->rev_id) {
1308			/* MTYPE_UC case */
1309			*flags = (*flags & ~AMDGPU_PTE_MTYPE_VG10_MASK) |
1310				 AMDGPU_PTE_MTYPE_VG10(MTYPE_CC);
1311		}
1312
1313		dev_dbg_ratelimited(adev->dev, "flags updated from %llx to %llx\n",
1314				    old_flags, *flags);
1315	}
1316}
1317
1318static unsigned int gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
1319{
1320	u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
1321	unsigned int size;
1322
1323	/* TODO move to DC so GMC doesn't need to hard-code DCN registers */
1324
1325	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
1326		size = AMDGPU_VBIOS_VGA_ALLOCATION;
1327	} else {
1328		u32 viewport;
1329
1330		switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
1331		case IP_VERSION(1, 0, 0):
1332		case IP_VERSION(1, 0, 1):
1333			viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
1334			size = (REG_GET_FIELD(viewport,
1335					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1336				REG_GET_FIELD(viewport,
1337					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1338				4);
1339			break;
1340		case IP_VERSION(2, 1, 0):
1341			viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2);
1342			size = (REG_GET_FIELD(viewport,
1343					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1344				REG_GET_FIELD(viewport,
1345					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1346				4);
1347			break;
1348		default:
1349			viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
1350			size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
1351				REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
1352				4);
1353			break;
1354		}
1355	}
1356
1357	return size;
1358}
1359
1360static enum amdgpu_memory_partition
1361gmc_v9_0_get_memory_partition(struct amdgpu_device *adev, u32 *supp_modes)
1362{
1363	enum amdgpu_memory_partition mode = UNKNOWN_MEMORY_PARTITION_MODE;
1364
1365	if (adev->nbio.funcs->get_memory_partition_mode)
1366		mode = adev->nbio.funcs->get_memory_partition_mode(adev,
1367								   supp_modes);
1368
1369	return mode;
1370}
1371
1372static enum amdgpu_memory_partition
1373gmc_v9_0_query_memory_partition(struct amdgpu_device *adev)
1374{
1375	if (amdgpu_sriov_vf(adev))
1376		return AMDGPU_NPS1_PARTITION_MODE;
1377
1378	return gmc_v9_0_get_memory_partition(adev, NULL);
1379}
1380
1381static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
1382	.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
1383	.flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid,
1384	.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
1385	.emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
1386	.map_mtype = gmc_v9_0_map_mtype,
1387	.get_vm_pde = gmc_v9_0_get_vm_pde,
1388	.get_vm_pte = gmc_v9_0_get_vm_pte,
1389	.override_vm_pte_flags = gmc_v9_0_override_vm_pte_flags,
1390	.get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
1391	.query_mem_partition_mode = &gmc_v9_0_query_memory_partition,
1392};
1393
1394static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
1395{
1396	adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
1397}
1398
1399static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
1400{
1401	switch (amdgpu_ip_version(adev, UMC_HWIP, 0)) {
1402	case IP_VERSION(6, 0, 0):
1403		adev->umc.funcs = &umc_v6_0_funcs;
1404		break;
1405	case IP_VERSION(6, 1, 1):
1406		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1407		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1408		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1409		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20;
1410		adev->umc.retire_unit = 1;
1411		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1412		adev->umc.ras = &umc_v6_1_ras;
1413		break;
1414	case IP_VERSION(6, 1, 2):
1415		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1416		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1417		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1418		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT;
1419		adev->umc.retire_unit = 1;
1420		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1421		adev->umc.ras = &umc_v6_1_ras;
1422		break;
1423	case IP_VERSION(6, 7, 0):
1424		adev->umc.max_ras_err_cnt_per_query =
1425			UMC_V6_7_TOTAL_CHANNEL_NUM * UMC_V6_7_BAD_PAGE_NUM_PER_CHANNEL;
1426		adev->umc.channel_inst_num = UMC_V6_7_CHANNEL_INSTANCE_NUM;
1427		adev->umc.umc_inst_num = UMC_V6_7_UMC_INSTANCE_NUM;
1428		adev->umc.channel_offs = UMC_V6_7_PER_CHANNEL_OFFSET;
1429		adev->umc.retire_unit = (UMC_V6_7_NA_MAP_PA_NUM * 2);
1430		if (!adev->gmc.xgmi.connected_to_cpu)
1431			adev->umc.ras = &umc_v6_7_ras;
1432		if (1 & adev->smuio.funcs->get_die_id(adev))
1433			adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_first[0][0];
1434		else
1435			adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_second[0][0];
1436		break;
1437	case IP_VERSION(12, 0, 0):
1438		adev->umc.max_ras_err_cnt_per_query =
1439			UMC_V12_0_TOTAL_CHANNEL_NUM(adev) * UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL;
1440		adev->umc.channel_inst_num = UMC_V12_0_CHANNEL_INSTANCE_NUM;
1441		adev->umc.umc_inst_num = UMC_V12_0_UMC_INSTANCE_NUM;
1442		adev->umc.node_inst_num /= UMC_V12_0_UMC_INSTANCE_NUM;
1443		adev->umc.channel_offs = UMC_V12_0_PER_CHANNEL_OFFSET;
1444		adev->umc.active_mask = adev->aid_mask;
1445		adev->umc.retire_unit = UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL;
1446		adev->umc.channel_idx_tbl = &umc_v12_0_channel_idx_tbl[0][0][0];
1447		if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu)
1448			adev->umc.ras = &umc_v12_0_ras;
1449		break;
1450	default:
1451		break;
1452	}
1453}
1454
1455static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
1456{
1457	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
1458	case IP_VERSION(9, 4, 1):
1459		adev->mmhub.funcs = &mmhub_v9_4_funcs;
1460		break;
1461	case IP_VERSION(9, 4, 2):
1462		adev->mmhub.funcs = &mmhub_v1_7_funcs;
1463		break;
1464	case IP_VERSION(1, 8, 0):
1465		adev->mmhub.funcs = &mmhub_v1_8_funcs;
1466		break;
1467	default:
1468		adev->mmhub.funcs = &mmhub_v1_0_funcs;
1469		break;
1470	}
1471}
1472
1473static void gmc_v9_0_set_mmhub_ras_funcs(struct amdgpu_device *adev)
1474{
1475	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
1476	case IP_VERSION(9, 4, 0):
1477		adev->mmhub.ras = &mmhub_v1_0_ras;
1478		break;
1479	case IP_VERSION(9, 4, 1):
1480		adev->mmhub.ras = &mmhub_v9_4_ras;
1481		break;
1482	case IP_VERSION(9, 4, 2):
1483		adev->mmhub.ras = &mmhub_v1_7_ras;
1484		break;
1485	case IP_VERSION(1, 8, 0):
1486		adev->mmhub.ras = &mmhub_v1_8_ras;
1487		break;
1488	default:
1489		/* mmhub ras is not available */
1490		break;
1491	}
1492}
1493
1494static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev)
1495{
1496	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3))
1497		adev->gfxhub.funcs = &gfxhub_v1_2_funcs;
1498	else
1499		adev->gfxhub.funcs = &gfxhub_v1_0_funcs;
1500}
1501
1502static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev)
1503{
1504	adev->hdp.ras = &hdp_v4_0_ras;
1505}
1506
1507static void gmc_v9_0_set_mca_ras_funcs(struct amdgpu_device *adev)
1508{
1509	struct amdgpu_mca *mca = &adev->mca;
1510
1511	/* is UMC the right IP to check for MCA?  Maybe DF? */
1512	switch (amdgpu_ip_version(adev, UMC_HWIP, 0)) {
1513	case IP_VERSION(6, 7, 0):
1514		if (!adev->gmc.xgmi.connected_to_cpu) {
1515			mca->mp0.ras = &mca_v3_0_mp0_ras;
1516			mca->mp1.ras = &mca_v3_0_mp1_ras;
1517			mca->mpio.ras = &mca_v3_0_mpio_ras;
1518		}
1519		break;
1520	default:
1521		break;
1522	}
1523}
1524
1525static void gmc_v9_0_set_xgmi_ras_funcs(struct amdgpu_device *adev)
1526{
1527	if (!adev->gmc.xgmi.connected_to_cpu)
1528		adev->gmc.xgmi.ras = &xgmi_ras;
1529}
1530
1531static int gmc_v9_0_early_init(void *handle)
1532{
1533	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1534
1535	/*
1536	 * 9.4.0, 9.4.1 and 9.4.3 don't have XGMI defined
1537	 * in their IP discovery tables
1538	 */
1539	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 0) ||
1540	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
1541	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3))
1542		adev->gmc.xgmi.supported = true;
1543
1544	if (amdgpu_ip_version(adev, XGMI_HWIP, 0) == IP_VERSION(6, 1, 0)) {
1545		adev->gmc.xgmi.supported = true;
1546		adev->gmc.xgmi.connected_to_cpu =
1547			adev->smuio.funcs->is_host_gpu_xgmi_supported(adev);
1548	}
1549
1550	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3)) {
1551		enum amdgpu_pkg_type pkg_type =
1552			adev->smuio.funcs->get_pkg_type(adev);
1553		/* On GFXIP 9.4.3. APU, there is no physical VRAM domain present
1554		 * and the APU, can be in used two possible modes:
1555		 *  - carveout mode
1556		 *  - native APU mode
1557		 * "is_app_apu" can be used to identify the APU in the native
1558		 * mode.
1559		 */
1560		adev->gmc.is_app_apu = (pkg_type == AMDGPU_PKG_TYPE_APU &&
1561					!pci_resource_len(adev->pdev, 0));
1562	}
1563
1564	gmc_v9_0_set_gmc_funcs(adev);
1565	gmc_v9_0_set_irq_funcs(adev);
1566	gmc_v9_0_set_umc_funcs(adev);
1567	gmc_v9_0_set_mmhub_funcs(adev);
1568	gmc_v9_0_set_mmhub_ras_funcs(adev);
1569	gmc_v9_0_set_gfxhub_funcs(adev);
1570	gmc_v9_0_set_hdp_ras_funcs(adev);
1571	gmc_v9_0_set_mca_ras_funcs(adev);
1572	gmc_v9_0_set_xgmi_ras_funcs(adev);
1573
1574	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
1575	adev->gmc.shared_aperture_end =
1576		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
1577	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
1578	adev->gmc.private_aperture_end =
1579		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
1580	adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
1581
1582	return 0;
1583}
1584
1585static int gmc_v9_0_late_init(void *handle)
1586{
1587	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1588	int r;
1589
1590	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
1591	if (r)
1592		return r;
1593
1594	/*
1595	 * Workaround performance drop issue with VBIOS enables partial
1596	 * writes, while disables HBM ECC for vega10.
1597	 */
1598	if (!amdgpu_sriov_vf(adev) &&
1599	    (amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(6, 0, 0))) {
1600		if (!(adev->ras_enabled & (1 << AMDGPU_RAS_BLOCK__UMC))) {
1601			if (adev->df.funcs &&
1602			    adev->df.funcs->enable_ecc_force_par_wr_rmw)
1603				adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false);
1604		}
1605	}
1606
1607	if (!amdgpu_persistent_edc_harvesting_supported(adev)) {
1608		amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB);
1609		amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__HDP);
 
 
 
 
 
1610	}
1611
1612	r = amdgpu_gmc_ras_late_init(adev);
1613	if (r)
1614		return r;
1615
1616	return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
1617}
1618
1619static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
1620					struct amdgpu_gmc *mc)
1621{
1622	u64 base = adev->mmhub.funcs->get_fb_location(adev);
1623
1624	amdgpu_gmc_set_agp_default(adev, mc);
1625
1626	/* add the xgmi offset of the physical node */
1627	base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1628	if (adev->gmc.xgmi.connected_to_cpu) {
1629		amdgpu_gmc_sysvm_location(adev, mc);
1630	} else {
1631		amdgpu_gmc_vram_location(adev, mc, base);
1632		amdgpu_gmc_gart_location(adev, mc, AMDGPU_GART_PLACEMENT_BEST_FIT);
1633		if (!amdgpu_sriov_vf(adev) && (amdgpu_agp == 1))
1634			amdgpu_gmc_agp_location(adev, mc);
1635	}
1636	/* base offset of vram pages */
1637	adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);
1638
1639	/* XXX: add the xgmi offset of the physical node? */
1640	adev->vm_manager.vram_base_offset +=
1641		adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1642}
1643
1644/**
1645 * gmc_v9_0_mc_init - initialize the memory controller driver params
1646 *
1647 * @adev: amdgpu_device pointer
1648 *
1649 * Look up the amount of vram, vram width, and decide how to place
1650 * vram and gart within the GPU's physical address space.
1651 * Returns 0 for success.
1652 */
1653static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
1654{
1655	int r;
1656
1657	/* size in MB on si */
1658	if (!adev->gmc.is_app_apu) {
1659		adev->gmc.mc_vram_size =
1660			adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
1661	} else {
1662		DRM_DEBUG("Set mc_vram_size = 0 for APP APU\n");
1663		adev->gmc.mc_vram_size = 0;
1664	}
1665	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
1666
1667	if (!(adev->flags & AMD_IS_APU) &&
1668	    !adev->gmc.xgmi.connected_to_cpu) {
1669		r = amdgpu_device_resize_fb_bar(adev);
1670		if (r)
1671			return r;
1672	}
1673	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
1674	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
1675
1676#ifdef CONFIG_X86_64
1677	/*
1678	 * AMD Accelerated Processing Platform (APP) supporting GPU-HOST xgmi
1679	 * interface can use VRAM through here as it appears system reserved
1680	 * memory in host address space.
1681	 *
1682	 * For APUs, VRAM is just the stolen system memory and can be accessed
1683	 * directly.
1684	 *
1685	 * Otherwise, use the legacy Host Data Path (HDP) through PCIe BAR.
1686	 */
1687
1688	/* check whether both host-gpu and gpu-gpu xgmi links exist */
1689	if ((!amdgpu_sriov_vf(adev) &&
1690		(adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) ||
1691	    (adev->gmc.xgmi.supported &&
1692	     adev->gmc.xgmi.connected_to_cpu)) {
1693		adev->gmc.aper_base =
1694			adev->gfxhub.funcs->get_mc_fb_offset(adev) +
1695			adev->gmc.xgmi.physical_node_id *
1696			adev->gmc.xgmi.node_segment_size;
1697		adev->gmc.aper_size = adev->gmc.real_vram_size;
1698	}
1699
1700#endif
 
1701	adev->gmc.visible_vram_size = adev->gmc.aper_size;
 
 
1702
1703	/* set the gart size */
1704	if (amdgpu_gart_size == -1) {
1705		switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1706		case IP_VERSION(9, 0, 1):  /* all engines support GPUVM */
1707		case IP_VERSION(9, 2, 1):  /* all engines support GPUVM */
1708		case IP_VERSION(9, 4, 0):
1709		case IP_VERSION(9, 4, 1):
1710		case IP_VERSION(9, 4, 2):
1711		case IP_VERSION(9, 4, 3):
1712		default:
1713			adev->gmc.gart_size = 512ULL << 20;
1714			break;
1715		case IP_VERSION(9, 1, 0):   /* DCE SG support */
1716		case IP_VERSION(9, 2, 2):   /* DCE SG support */
1717		case IP_VERSION(9, 3, 0):
1718			adev->gmc.gart_size = 1024ULL << 20;
1719			break;
1720		}
1721	} else {
1722		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
1723	}
1724
1725	adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
1726
1727	gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
1728
1729	return 0;
1730}
1731
1732static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
1733{
1734	int r;
1735
1736	if (adev->gart.bo) {
1737		WARN(1, "VEGA10 PCIE GART already initialized\n");
1738		return 0;
1739	}
1740
1741	if (adev->gmc.xgmi.connected_to_cpu) {
1742		adev->gmc.vmid0_page_table_depth = 1;
1743		adev->gmc.vmid0_page_table_block_size = 12;
1744	} else {
1745		adev->gmc.vmid0_page_table_depth = 0;
1746		adev->gmc.vmid0_page_table_block_size = 0;
1747	}
1748
1749	/* Initialize common gart structure */
1750	r = amdgpu_gart_init(adev);
1751	if (r)
1752		return r;
1753	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
1754	adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) |
1755				 AMDGPU_PTE_EXECUTABLE;
1756
1757	if (!adev->gmc.real_vram_size) {
1758		dev_info(adev->dev, "Put GART in system memory for APU\n");
1759		r = amdgpu_gart_table_ram_alloc(adev);
1760		if (r)
1761			dev_err(adev->dev, "Failed to allocate GART in system memory\n");
1762	} else {
1763		r = amdgpu_gart_table_vram_alloc(adev);
1764		if (r)
1765			return r;
1766
1767		if (adev->gmc.xgmi.connected_to_cpu)
1768			r = amdgpu_gmc_pdb0_alloc(adev);
1769	}
1770
1771	return r;
1772}
1773
1774/**
1775 * gmc_v9_0_save_registers - saves regs
1776 *
1777 * @adev: amdgpu_device pointer
1778 *
1779 * This saves potential register values that should be
1780 * restored upon resume
1781 */
1782static void gmc_v9_0_save_registers(struct amdgpu_device *adev)
1783{
1784	if ((amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(1, 0, 0)) ||
1785	    (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(1, 0, 1)))
1786		adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0);
1787}
1788
1789static bool gmc_v9_0_validate_partition_info(struct amdgpu_device *adev)
1790{
1791	enum amdgpu_memory_partition mode;
1792	u32 supp_modes;
1793	bool valid;
1794
1795	mode = gmc_v9_0_get_memory_partition(adev, &supp_modes);
1796
1797	/* Mode detected by hardware not present in supported modes */
1798	if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) &&
1799	    !(BIT(mode - 1) & supp_modes))
1800		return false;
1801
1802	switch (mode) {
1803	case UNKNOWN_MEMORY_PARTITION_MODE:
1804	case AMDGPU_NPS1_PARTITION_MODE:
1805		valid = (adev->gmc.num_mem_partitions == 1);
1806		break;
1807	case AMDGPU_NPS2_PARTITION_MODE:
1808		valid = (adev->gmc.num_mem_partitions == 2);
1809		break;
1810	case AMDGPU_NPS4_PARTITION_MODE:
1811		valid = (adev->gmc.num_mem_partitions == 3 ||
1812			 adev->gmc.num_mem_partitions == 4);
1813		break;
1814	default:
1815		valid = false;
1816	}
1817
1818	return valid;
1819}
1820
1821static bool gmc_v9_0_is_node_present(int *node_ids, int num_ids, int nid)
1822{
1823	int i;
1824
1825	/* Check if node with id 'nid' is present in 'node_ids' array */
1826	for (i = 0; i < num_ids; ++i)
1827		if (node_ids[i] == nid)
1828			return true;
1829
1830	return false;
1831}
1832
1833static void
1834gmc_v9_0_init_acpi_mem_ranges(struct amdgpu_device *adev,
1835			      struct amdgpu_mem_partition_info *mem_ranges)
1836{
1837	struct amdgpu_numa_info numa_info;
1838	int node_ids[MAX_MEM_RANGES];
1839	int num_ranges = 0, ret;
1840	int num_xcc, xcc_id;
1841	uint32_t xcc_mask;
1842
1843	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1844	xcc_mask = (1U << num_xcc) - 1;
1845
1846	for_each_inst(xcc_id, xcc_mask)	{
1847		ret = amdgpu_acpi_get_mem_info(adev, xcc_id, &numa_info);
1848		if (ret)
1849			continue;
1850
1851		if (numa_info.nid == NUMA_NO_NODE) {
1852			mem_ranges[0].size = numa_info.size;
1853			mem_ranges[0].numa.node = numa_info.nid;
1854			num_ranges = 1;
1855			break;
1856		}
1857
1858		if (gmc_v9_0_is_node_present(node_ids, num_ranges,
1859					     numa_info.nid))
1860			continue;
1861
1862		node_ids[num_ranges] = numa_info.nid;
1863		mem_ranges[num_ranges].numa.node = numa_info.nid;
1864		mem_ranges[num_ranges].size = numa_info.size;
1865		++num_ranges;
1866	}
1867
1868	adev->gmc.num_mem_partitions = num_ranges;
1869}
1870
1871static void
1872gmc_v9_0_init_sw_mem_ranges(struct amdgpu_device *adev,
1873			    struct amdgpu_mem_partition_info *mem_ranges)
1874{
1875	enum amdgpu_memory_partition mode;
1876	u32 start_addr = 0, size;
1877	int i;
1878
1879	mode = gmc_v9_0_query_memory_partition(adev);
1880
1881	switch (mode) {
1882	case UNKNOWN_MEMORY_PARTITION_MODE:
1883	case AMDGPU_NPS1_PARTITION_MODE:
1884		adev->gmc.num_mem_partitions = 1;
1885		break;
1886	case AMDGPU_NPS2_PARTITION_MODE:
1887		adev->gmc.num_mem_partitions = 2;
1888		break;
1889	case AMDGPU_NPS4_PARTITION_MODE:
1890		if (adev->flags & AMD_IS_APU)
1891			adev->gmc.num_mem_partitions = 3;
1892		else
1893			adev->gmc.num_mem_partitions = 4;
1894		break;
1895	default:
1896		adev->gmc.num_mem_partitions = 1;
1897		break;
1898	}
1899
1900	size = adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT;
1901	size /= adev->gmc.num_mem_partitions;
1902
1903	for (i = 0; i < adev->gmc.num_mem_partitions; ++i) {
1904		mem_ranges[i].range.fpfn = start_addr;
1905		mem_ranges[i].size = ((u64)size << AMDGPU_GPU_PAGE_SHIFT);
1906		mem_ranges[i].range.lpfn = start_addr + size - 1;
1907		start_addr += size;
1908	}
1909
1910	/* Adjust the last one */
1911	mem_ranges[adev->gmc.num_mem_partitions - 1].range.lpfn =
1912		(adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT) - 1;
1913	mem_ranges[adev->gmc.num_mem_partitions - 1].size =
1914		adev->gmc.real_vram_size -
1915		((u64)mem_ranges[adev->gmc.num_mem_partitions - 1].range.fpfn
1916		 << AMDGPU_GPU_PAGE_SHIFT);
1917}
1918
1919static int gmc_v9_0_init_mem_ranges(struct amdgpu_device *adev)
1920{
1921	bool valid;
1922
1923	adev->gmc.mem_partitions = kcalloc(MAX_MEM_RANGES,
1924					   sizeof(struct amdgpu_mem_partition_info),
1925					   GFP_KERNEL);
1926	if (!adev->gmc.mem_partitions)
1927		return -ENOMEM;
1928
1929	/* TODO : Get the range from PSP/Discovery for dGPU */
1930	if (adev->gmc.is_app_apu)
1931		gmc_v9_0_init_acpi_mem_ranges(adev, adev->gmc.mem_partitions);
1932	else
1933		gmc_v9_0_init_sw_mem_ranges(adev, adev->gmc.mem_partitions);
1934
1935	if (amdgpu_sriov_vf(adev))
1936		valid = true;
1937	else
1938		valid = gmc_v9_0_validate_partition_info(adev);
1939	if (!valid) {
1940		/* TODO: handle invalid case */
1941		dev_WARN(adev->dev,
1942			 "Mem ranges not matching with hardware config");
1943	}
1944
1945	return 0;
1946}
1947
1948static void gmc_v9_4_3_init_vram_info(struct amdgpu_device *adev)
1949{
1950	adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM;
1951	adev->gmc.vram_width = 128 * 64;
1952}
1953
1954static int gmc_v9_0_sw_init(void *handle)
1955{
1956	int r, vram_width = 0, vram_type = 0, vram_vendor = 0, dma_addr_bits;
1957	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1958	unsigned long inst_mask = adev->aid_mask;
1959
1960	adev->gfxhub.funcs->init(adev);
1961
1962	adev->mmhub.funcs->init(adev);
1963
1964	spin_lock_init(&adev->gmc.invalidate_lock);
1965
1966	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3)) {
1967		gmc_v9_4_3_init_vram_info(adev);
1968	} else if (!adev->bios) {
1969		if (adev->flags & AMD_IS_APU) {
1970			adev->gmc.vram_type = AMDGPU_VRAM_TYPE_DDR4;
1971			adev->gmc.vram_width = 64 * 64;
1972		} else {
1973			adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM;
1974			adev->gmc.vram_width = 128 * 64;
1975		}
1976	} else {
1977		r = amdgpu_atomfirmware_get_vram_info(adev,
1978			&vram_width, &vram_type, &vram_vendor);
1979		if (amdgpu_sriov_vf(adev))
1980			/* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN,
1981			 * and DF related registers is not readable, seems hardcord is the
1982			 * only way to set the correct vram_width
1983			 */
1984			adev->gmc.vram_width = 2048;
1985		else if (amdgpu_emu_mode != 1)
1986			adev->gmc.vram_width = vram_width;
1987
1988		if (!adev->gmc.vram_width) {
1989			int chansize, numchan;
1990
1991			/* hbm memory channel size */
1992			if (adev->flags & AMD_IS_APU)
1993				chansize = 64;
1994			else
1995				chansize = 128;
1996			if (adev->df.funcs &&
1997			    adev->df.funcs->get_hbm_channel_number) {
1998				numchan = adev->df.funcs->get_hbm_channel_number(adev);
1999				adev->gmc.vram_width = numchan * chansize;
2000			}
2001		}
2002
2003		adev->gmc.vram_type = vram_type;
2004		adev->gmc.vram_vendor = vram_vendor;
 
 
 
 
 
 
2005	}
2006	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2007	case IP_VERSION(9, 1, 0):
2008	case IP_VERSION(9, 2, 2):
2009		set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
2010		set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);
 
2011
2012		if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
2013			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
2014		} else {
2015			/* vm_size is 128TB + 512GB for legacy 3-level page support */
2016			amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);
2017			adev->gmc.translate_further =
2018				adev->vm_manager.num_level > 1;
2019		}
2020		break;
2021	case IP_VERSION(9, 0, 1):
2022	case IP_VERSION(9, 2, 1):
2023	case IP_VERSION(9, 4, 0):
2024	case IP_VERSION(9, 3, 0):
2025	case IP_VERSION(9, 4, 2):
2026		set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
2027		set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);
2028
2029		/*
2030		 * To fulfill 4-level page support,
2031		 * vm size is 256TB (48bit), maximum size of Vega10,
2032		 * block size 512 (9bit)
2033		 */
2034
2035		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
2036		if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
2037			adev->gmc.translate_further = adev->vm_manager.num_level > 1;
 
2038		break;
2039	case IP_VERSION(9, 4, 1):
2040		set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
2041		set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);
2042		set_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask);
2043
2044		/* Keep the vm size same with Vega20 */
2045		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
2046		adev->gmc.translate_further = adev->vm_manager.num_level > 1;
2047		break;
2048	case IP_VERSION(9, 4, 3):
2049		bitmap_set(adev->vmhubs_mask, AMDGPU_GFXHUB(0),
2050				  NUM_XCC(adev->gfx.xcc_mask));
2051
2052		inst_mask <<= AMDGPU_MMHUB0(0);
2053		bitmap_or(adev->vmhubs_mask, adev->vmhubs_mask, &inst_mask, 32);
2054
2055		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
2056		adev->gmc.translate_further = adev->vm_manager.num_level > 1;
2057		break;
2058	default:
2059		break;
2060	}
2061
2062	/* This interrupt is VMC page fault.*/
2063	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,
2064				&adev->gmc.vm_fault);
2065	if (r)
2066		return r;
2067
2068	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) {
2069		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT,
2070					&adev->gmc.vm_fault);
2071		if (r)
2072			return r;
2073	}
2074
2075	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,
2076				&adev->gmc.vm_fault);
2077
2078	if (r)
2079		return r;
2080
2081	if (!amdgpu_sriov_vf(adev) &&
2082	    !adev->gmc.xgmi.connected_to_cpu &&
2083	    !adev->gmc.is_app_apu) {
2084		/* interrupt sent to DF. */
2085		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0,
2086				      &adev->gmc.ecc_irq);
2087		if (r)
2088			return r;
2089	}
2090
2091	/* Set the internal MC address mask
2092	 * This is the max address of the GPU's
2093	 * internal address space.
2094	 */
2095	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
2096
2097	dma_addr_bits = amdgpu_ip_version(adev, GC_HWIP, 0) >=
2098					IP_VERSION(9, 4, 2) ?
2099				48 :
2100				44;
2101	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(dma_addr_bits));
2102	if (r) {
2103		dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n");
2104		return r;
2105	}
2106	adev->need_swiotlb = drm_need_swiotlb(dma_addr_bits);
 
 
 
 
 
 
2107
2108	r = gmc_v9_0_mc_init(adev);
2109	if (r)
2110		return r;
2111
2112	amdgpu_gmc_get_vbios_allocations(adev);
2113
2114	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3)) {
2115		r = gmc_v9_0_init_mem_ranges(adev);
2116		if (r)
2117			return r;
2118	}
2119
2120	/* Memory manager */
2121	r = amdgpu_bo_init(adev);
2122	if (r)
2123		return r;
2124
2125	r = gmc_v9_0_gart_init(adev);
2126	if (r)
2127		return r;
2128
2129	/*
2130	 * number of VMs
2131	 * VMID 0 is reserved for System
2132	 * amdgpu graphics/compute will use VMIDs 1..n-1
2133	 * amdkfd will use VMIDs n..15
2134	 *
2135	 * The first KFD VMID is 8 for GPUs with graphics, 3 for
2136	 * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs
2137	 * for video processing.
2138	 */
2139	adev->vm_manager.first_kfd_vmid =
2140		(amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
2141		 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
2142		 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3)) ?
2143			3 :
2144			8;
2145
2146	amdgpu_vm_manager_init(adev);
2147
2148	gmc_v9_0_save_registers(adev);
2149
2150	r = amdgpu_gmc_ras_sw_init(adev);
2151	if (r)
2152		return r;
2153
2154	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3))
2155		amdgpu_gmc_sysfs_init(adev);
2156
2157	return 0;
2158}
2159
2160static int gmc_v9_0_sw_fini(void *handle)
2161{
2162	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2163
2164	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3))
2165		amdgpu_gmc_sysfs_fini(adev);
2166
2167	amdgpu_gmc_ras_fini(adev);
2168	amdgpu_gem_force_release(adev);
2169	amdgpu_vm_manager_fini(adev);
2170	if (!adev->gmc.real_vram_size) {
2171		dev_info(adev->dev, "Put GART in system memory for APU free\n");
2172		amdgpu_gart_table_ram_free(adev);
2173	} else {
2174		amdgpu_gart_table_vram_free(adev);
2175	}
2176	amdgpu_bo_free_kernel(&adev->gmc.pdb0_bo, NULL, &adev->gmc.ptr_pdb0);
2177	amdgpu_bo_fini(adev);
2178
2179	adev->gmc.num_mem_partitions = 0;
2180	kfree(adev->gmc.mem_partitions);
2181
2182	return 0;
2183}
2184
2185static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
2186{
2187	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
2188	case IP_VERSION(9, 0, 0):
 
2189		if (amdgpu_sriov_vf(adev))
2190			break;
2191		fallthrough;
2192	case IP_VERSION(9, 4, 0):
2193		soc15_program_register_sequence(adev,
2194						golden_settings_mmhub_1_0_0,
2195						ARRAY_SIZE(golden_settings_mmhub_1_0_0));
2196		soc15_program_register_sequence(adev,
2197						golden_settings_athub_1_0_0,
2198						ARRAY_SIZE(golden_settings_athub_1_0_0));
2199		break;
2200	case IP_VERSION(9, 1, 0):
2201	case IP_VERSION(9, 2, 0):
 
2202		/* TODO for renoir */
2203		soc15_program_register_sequence(adev,
2204						golden_settings_athub_1_0_0,
2205						ARRAY_SIZE(golden_settings_athub_1_0_0));
2206		break;
2207	default:
2208		break;
2209	}
2210}
2211
2212/**
2213 * gmc_v9_0_restore_registers - restores regs
2214 *
2215 * @adev: amdgpu_device pointer
2216 *
2217 * This restores register values, saved at suspend.
2218 */
2219void gmc_v9_0_restore_registers(struct amdgpu_device *adev)
2220{
2221	if ((amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(1, 0, 0)) ||
2222	    (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(1, 0, 1))) {
2223		WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register);
2224		WARN_ON(adev->gmc.sdpif_register !=
2225			RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0));
2226	}
2227}
2228
2229/**
2230 * gmc_v9_0_gart_enable - gart enable
2231 *
2232 * @adev: amdgpu_device pointer
2233 */
2234static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
2235{
2236	int r;
2237
2238	if (adev->gmc.xgmi.connected_to_cpu)
2239		amdgpu_gmc_init_pdb0(adev);
2240
2241	if (adev->gart.bo == NULL) {
2242		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
2243		return -EINVAL;
2244	}
2245
2246	amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr);
 
 
2247
2248	if (!adev->in_s0ix) {
2249		r = adev->gfxhub.funcs->gart_enable(adev);
2250		if (r)
2251			return r;
2252	}
2253
2254	r = adev->mmhub.funcs->gart_enable(adev);
2255	if (r)
2256		return r;
2257
2258	DRM_INFO("PCIE GART of %uM enabled.\n",
2259		 (unsigned int)(adev->gmc.gart_size >> 20));
2260	if (adev->gmc.pdb0_bo)
2261		DRM_INFO("PDB0 located at 0x%016llX\n",
2262				(unsigned long long)amdgpu_bo_gpu_offset(adev->gmc.pdb0_bo));
2263	DRM_INFO("PTB located at 0x%016llX\n",
2264			(unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
2265
 
2266	return 0;
2267}
2268
2269static int gmc_v9_0_hw_init(void *handle)
2270{
2271	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2272	bool value;
2273	int i, r;
2274
2275	adev->gmc.flush_pasid_uses_kiq = true;
2276
2277	/* Vega20+XGMI caches PTEs in TC and TLB. Add a heavy-weight TLB flush
2278	 * (type 2), which flushes both. Due to a race condition with
2279	 * concurrent memory accesses using the same TLB cache line, we still
2280	 * need a second TLB flush after this.
2281	 */
2282	adev->gmc.flush_tlb_needs_extra_type_2 =
2283		amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 0) &&
2284		adev->gmc.xgmi.num_physical_nodes;
2285	/*
2286	 * TODO: This workaround is badly documented and had a buggy
2287	 * implementation. We should probably verify what we do here.
2288	 */
2289	adev->gmc.flush_tlb_needs_extra_type_0 =
2290		amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) &&
2291		adev->rev_id == 0;
2292
2293	/* The sequence of these two function calls matters.*/
2294	gmc_v9_0_init_golden_registers(adev);
2295
2296	if (adev->mode_info.num_crtc) {
2297		/* Lockout access through VGA aperture*/
2298		WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
2299		/* disable VGA render */
2300		WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
2301	}
2302
2303	if (adev->mmhub.funcs->update_power_gating)
2304		adev->mmhub.funcs->update_power_gating(adev, true);
2305
2306	adev->hdp.funcs->init_registers(adev);
2307
2308	/* After HDP is initialized, flush HDP.*/
2309	adev->hdp.funcs->flush_hdp(adev, NULL);
2310
2311	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
2312		value = false;
2313	else
2314		value = true;
2315
2316	if (!amdgpu_sriov_vf(adev)) {
2317		if (!adev->in_s0ix)
2318			adev->gfxhub.funcs->set_fault_enable_default(adev, value);
2319		adev->mmhub.funcs->set_fault_enable_default(adev, value);
2320	}
2321	for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
2322		if (adev->in_s0ix && (i == AMDGPU_GFXHUB(0)))
2323			continue;
2324		gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0);
2325	}
2326
2327	if (adev->umc.funcs && adev->umc.funcs->init_registers)
2328		adev->umc.funcs->init_registers(adev);
2329
2330	r = gmc_v9_0_gart_enable(adev);
2331	if (r)
2332		return r;
2333
2334	if (amdgpu_emu_mode == 1)
2335		return amdgpu_gmc_vram_checking(adev);
2336
2337	return 0;
2338}
2339
2340/**
2341 * gmc_v9_0_gart_disable - gart disable
2342 *
2343 * @adev: amdgpu_device pointer
2344 *
2345 * This disables all VM page table.
2346 */
2347static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
2348{
2349	if (!adev->in_s0ix)
2350		adev->gfxhub.funcs->gart_disable(adev);
2351	adev->mmhub.funcs->gart_disable(adev);
 
2352}
2353
2354static int gmc_v9_0_hw_fini(void *handle)
2355{
2356	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2357
2358	gmc_v9_0_gart_disable(adev);
2359
2360	if (amdgpu_sriov_vf(adev)) {
2361		/* full access mode, so don't touch any GMC register */
2362		DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
2363		return 0;
2364	}
2365
2366	/*
2367	 * Pair the operations did in gmc_v9_0_hw_init and thus maintain
2368	 * a correct cached state for GMC. Otherwise, the "gate" again
2369	 * operation on S3 resuming will fail due to wrong cached state.
2370	 */
2371	if (adev->mmhub.funcs->update_power_gating)
2372		adev->mmhub.funcs->update_power_gating(adev, false);
2373
2374	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
2375
2376	if (adev->gmc.ecc_irq.funcs &&
2377		amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC))
2378		amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0);
2379
2380	return 0;
2381}
2382
2383static int gmc_v9_0_suspend(void *handle)
2384{
2385	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2386
2387	return gmc_v9_0_hw_fini(adev);
2388}
2389
2390static int gmc_v9_0_resume(void *handle)
2391{
2392	int r;
2393	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2394
2395	r = gmc_v9_0_hw_init(adev);
2396	if (r)
2397		return r;
2398
2399	amdgpu_vmid_reset_all(adev);
2400
2401	return 0;
2402}
2403
2404static bool gmc_v9_0_is_idle(void *handle)
2405{
2406	/* MC is always ready in GMC v9.*/
2407	return true;
2408}
2409
2410static int gmc_v9_0_wait_for_idle(void *handle)
2411{
2412	/* There is no need to wait for MC idle in GMC v9.*/
2413	return 0;
2414}
2415
2416static int gmc_v9_0_soft_reset(void *handle)
2417{
2418	/* XXX for emulation.*/
2419	return 0;
2420}
2421
2422static int gmc_v9_0_set_clockgating_state(void *handle,
2423					enum amd_clockgating_state state)
2424{
2425	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2426
2427	adev->mmhub.funcs->set_clockgating(adev, state);
2428
2429	athub_v1_0_set_clockgating(adev, state);
2430
2431	return 0;
2432}
2433
2434static void gmc_v9_0_get_clockgating_state(void *handle, u64 *flags)
2435{
2436	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2437
2438	adev->mmhub.funcs->get_clockgating(adev, flags);
2439
2440	athub_v1_0_get_clockgating(adev, flags);
2441}
2442
2443static int gmc_v9_0_set_powergating_state(void *handle,
2444					enum amd_powergating_state state)
2445{
2446	return 0;
2447}
2448
2449const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
2450	.name = "gmc_v9_0",
2451	.early_init = gmc_v9_0_early_init,
2452	.late_init = gmc_v9_0_late_init,
2453	.sw_init = gmc_v9_0_sw_init,
2454	.sw_fini = gmc_v9_0_sw_fini,
2455	.hw_init = gmc_v9_0_hw_init,
2456	.hw_fini = gmc_v9_0_hw_fini,
2457	.suspend = gmc_v9_0_suspend,
2458	.resume = gmc_v9_0_resume,
2459	.is_idle = gmc_v9_0_is_idle,
2460	.wait_for_idle = gmc_v9_0_wait_for_idle,
2461	.soft_reset = gmc_v9_0_soft_reset,
2462	.set_clockgating_state = gmc_v9_0_set_clockgating_state,
2463	.set_powergating_state = gmc_v9_0_set_powergating_state,
2464	.get_clockgating_state = gmc_v9_0_get_clockgating_state,
2465};
2466
2467const struct amdgpu_ip_block_version gmc_v9_0_ip_block = {
 
2468	.type = AMD_IP_BLOCK_TYPE_GMC,
2469	.major = 9,
2470	.minor = 0,
2471	.rev = 0,
2472	.funcs = &gmc_v9_0_ip_funcs,
2473};