1/*
   2 * Copyright 2018 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 *
  23 */
  24#include <linux/debugfs.h>
  25#include <linux/list.h>
  26#include <linux/module.h>
  27#include <linux/uaccess.h>
  28#include <linux/reboot.h>
  29#include <linux/syscalls.h>
  30#include <linux/pm_runtime.h>
  31
  32#include "amdgpu.h"
  33#include "amdgpu_ras.h"
  34#include "amdgpu_atomfirmware.h"
  35#include "amdgpu_xgmi.h"
  36#include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
  37#include "atom.h"
  38#include "amdgpu_reset.h"
  39
  40#ifdef CONFIG_X86_MCE_AMD
  41#include <asm/mce.h>
  42
  43static bool notifier_registered;
  44#endif
  45static const char *RAS_FS_NAME = "ras";
  46
  47const char *ras_error_string[] = {
  48	"none",
  49	"parity",
  50	"single_correctable",
  51	"multi_uncorrectable",
  52	"poison",
  53};
  54
  55const char *ras_block_string[] = {
  56	"umc",
  57	"sdma",
  58	"gfx",
  59	"mmhub",
  60	"athub",
  61	"pcie_bif",
  62	"hdp",
  63	"xgmi_wafl",
  64	"df",
  65	"smn",
  66	"sem",
  67	"mp0",
  68	"mp1",
  69	"fuse",
  70	"mca",
  71	"vcn",
  72	"jpeg",
  73};
  74
  75const char *ras_mca_block_string[] = {
  76	"mca_mp0",
  77	"mca_mp1",
  78	"mca_mpio",
  79	"mca_iohc",
  80};
  81
  82struct amdgpu_ras_block_list {
  83	/* ras block link */
  84	struct list_head node;
  85
  86	struct amdgpu_ras_block_object *ras_obj;
  87};
  88
  89const char *get_ras_block_str(struct ras_common_if *ras_block)
  90{
  91	if (!ras_block)
  92		return "NULL";
  93
  94	if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT)
  95		return "OUT OF RANGE";
  96
  97	if (ras_block->block == AMDGPU_RAS_BLOCK__MCA)
  98		return ras_mca_block_string[ras_block->sub_block_index];
  99
 100	return ras_block_string[ras_block->block];
 101}
 102
 103#define ras_block_str(_BLOCK_) \
 104	(((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range")
 105
 106#define ras_err_str(i) (ras_error_string[ffs(i)])
 107
 108#define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
 109
 110/* inject address is 52 bits */
 111#define	RAS_UMC_INJECT_ADDR_LIMIT	(0x1ULL << 52)
 112
 113/* typical ECC bad page rate is 1 bad page per 100MB VRAM */
 114#define RAS_BAD_PAGE_COVER              (100 * 1024 * 1024ULL)
 115
 116enum amdgpu_ras_retire_page_reservation {
 117	AMDGPU_RAS_RETIRE_PAGE_RESERVED,
 118	AMDGPU_RAS_RETIRE_PAGE_PENDING,
 119	AMDGPU_RAS_RETIRE_PAGE_FAULT,
 120};
 121
 122atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0);
 123
 124static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
 125				uint64_t addr);
 126static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
 127				uint64_t addr);
 128#ifdef CONFIG_X86_MCE_AMD
 129static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev);
 130struct mce_notifier_adev_list {
 131	struct amdgpu_device *devs[MAX_GPU_INSTANCE];
 132	int num_gpu;
 133};
 134static struct mce_notifier_adev_list mce_adev_list;
 135#endif
 136
 137void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready)
 138{
 139	if (adev && amdgpu_ras_get_context(adev))
 140		amdgpu_ras_get_context(adev)->error_query_ready = ready;
 141}
 142
 143static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev)
 144{
 145	if (adev && amdgpu_ras_get_context(adev))
 146		return amdgpu_ras_get_context(adev)->error_query_ready;
 147
 148	return false;
 149}
 150
 151static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address)
 152{
 153	struct ras_err_data err_data = {0, 0, 0, NULL};
 154	struct eeprom_table_record err_rec;
 155
 156	if ((address >= adev->gmc.mc_vram_size) ||
 157	    (address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
 158		dev_warn(adev->dev,
 159		         "RAS WARN: input address 0x%llx is invalid.\n",
 160		         address);
 161		return -EINVAL;
 162	}
 163
 164	if (amdgpu_ras_check_bad_page(adev, address)) {
 165		dev_warn(adev->dev,
 166			 "RAS WARN: 0x%llx has already been marked as bad page!\n",
 167			 address);
 168		return 0;
 169	}
 170
 171	memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
 172	err_data.err_addr = &err_rec;
 173	amdgpu_umc_fill_error_record(&err_data, address,
 174			(address >> AMDGPU_GPU_PAGE_SHIFT), 0, 0);
 175
 176	if (amdgpu_bad_page_threshold != 0) {
 177		amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
 178					 err_data.err_addr_cnt);
 179		amdgpu_ras_save_bad_pages(adev);
 180	}
 181
 182	dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n");
 183	dev_warn(adev->dev, "Clear EEPROM:\n");
 184	dev_warn(adev->dev, "    echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n");
 185
 186	return 0;
 187}
 188
 189static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
 190					size_t size, loff_t *pos)
 191{
 192	struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private;
 193	struct ras_query_if info = {
 194		.head = obj->head,
 195	};
 196	ssize_t s;
 197	char val[128];
 198
 199	if (amdgpu_ras_query_error_status(obj->adev, &info))
 200		return -EINVAL;
 201
 202	/* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */
 203	if (obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
 204	    obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
 205		if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
 206			dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
 207	}
 208
 209	s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n",
 210			"ue", info.ue_count,
 211			"ce", info.ce_count);
 212	if (*pos >= s)
 213		return 0;
 214
 215	s -= *pos;
 216	s = min_t(u64, s, size);
 217
 218
 219	if (copy_to_user(buf, &val[*pos], s))
 220		return -EINVAL;
 221
 222	*pos += s;
 223
 224	return s;
 225}
 226
 227static const struct file_operations amdgpu_ras_debugfs_ops = {
 228	.owner = THIS_MODULE,
 229	.read = amdgpu_ras_debugfs_read,
 230	.write = NULL,
 231	.llseek = default_llseek
 232};
 233
 234static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id)
 235{
 236	int i;
 237
 238	for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) {
 239		*block_id = i;
 240		if (strcmp(name, ras_block_string[i]) == 0)
 241			return 0;
 242	}
 243	return -EINVAL;
 244}
 245
 246static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
 247		const char __user *buf, size_t size,
 248		loff_t *pos, struct ras_debug_if *data)
 249{
 250	ssize_t s = min_t(u64, 64, size);
 251	char str[65];
 252	char block_name[33];
 253	char err[9] = "ue";
 254	int op = -1;
 255	int block_id;
 256	uint32_t sub_block;
 257	u64 address, value;
 258
 259	if (*pos)
 260		return -EINVAL;
 261	*pos = size;
 262
 263	memset(str, 0, sizeof(str));
 264	memset(data, 0, sizeof(*data));
 265
 266	if (copy_from_user(str, buf, s))
 267		return -EINVAL;
 268
 269	if (sscanf(str, "disable %32s", block_name) == 1)
 270		op = 0;
 271	else if (sscanf(str, "enable %32s %8s", block_name, err) == 2)
 272		op = 1;
 273	else if (sscanf(str, "inject %32s %8s", block_name, err) == 2)
 274		op = 2;
 275	else if (strstr(str, "retire_page") != NULL)
 276		op = 3;
 277	else if (str[0] && str[1] && str[2] && str[3])
 278		/* ascii string, but commands are not matched. */
 279		return -EINVAL;
 280
 281	if (op != -1) {
 282		if (op == 3) {
 283			if (sscanf(str, "%*s 0x%llx", &address) != 1 &&
 284			    sscanf(str, "%*s %llu", &address) != 1)
 285				return -EINVAL;
 286
 287			data->op = op;
 288			data->inject.address = address;
 289
 290			return 0;
 291		}
 292
 293		if (amdgpu_ras_find_block_id_by_name(block_name, &block_id))
 294			return -EINVAL;
 295
 296		data->head.block = block_id;
 297		/* only ue and ce errors are supported */
 298		if (!memcmp("ue", err, 2))
 299			data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
 300		else if (!memcmp("ce", err, 2))
 301			data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE;
 302		else
 303			return -EINVAL;
 304
 305		data->op = op;
 306
 307		if (op == 2) {
 308			if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx",
 309				   &sub_block, &address, &value) != 3 &&
 310			    sscanf(str, "%*s %*s %*s %u %llu %llu",
 311				   &sub_block, &address, &value) != 3)
 312				return -EINVAL;
 313			data->head.sub_block_index = sub_block;
 314			data->inject.address = address;
 315			data->inject.value = value;
 316		}
 317	} else {
 318		if (size < sizeof(*data))
 319			return -EINVAL;
 320
 321		if (copy_from_user(data, buf, sizeof(*data)))
 322			return -EINVAL;
 323	}
 324
 325	return 0;
 326}
 327
 328/**
 329 * DOC: AMDGPU RAS debugfs control interface
 330 *
 331 * The control interface accepts struct ras_debug_if which has two members.
 332 *
 333 * First member: ras_debug_if::head or ras_debug_if::inject.
 334 *
 335 * head is used to indicate which IP block will be under control.
 336 *
 337 * head has four members, they are block, type, sub_block_index, name.
 338 * block: which IP will be under control.
 339 * type: what kind of error will be enabled/disabled/injected.
 340 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA.
 341 * name: the name of IP.
 342 *
 343 * inject has two more members than head, they are address, value.
 344 * As their names indicate, inject operation will write the
 345 * value to the address.
 346 *
 347 * The second member: struct ras_debug_if::op.
 348 * It has three kinds of operations.
 349 *
 350 * - 0: disable RAS on the block. Take ::head as its data.
 351 * - 1: enable RAS on the block. Take ::head as its data.
 352 * - 2: inject errors on the block. Take ::inject as its data.
 353 *
 354 * How to use the interface?
 355 *
 356 * In a program
 357 *
 358 * Copy the struct ras_debug_if in your code and initialize it.
 359 * Write the struct to the control interface.
 360 *
 361 * From shell
 362 *
 363 * .. code-block:: bash
 364 *
 365 *	echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
 366 *	echo "enable  <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
 367 *	echo "inject  <block> <error> <sub-block> <address> <value> > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
 368 *
 369 * Where N, is the card which you want to affect.
 370 *
 371 * "disable" requires only the block.
 372 * "enable" requires the block and error type.
 373 * "inject" requires the block, error type, address, and value.
 374 *
 375 * The block is one of: umc, sdma, gfx, etc.
 376 *	see ras_block_string[] for details
 377 *
 378 * The error type is one of: ue, ce, where,
 379 *	ue is multi-uncorrectable
 380 *	ce is single-correctable
 381 *
 382 * The sub-block is a the sub-block index, pass 0 if there is no sub-block.
 383 * The address and value are hexadecimal numbers, leading 0x is optional.
 384 *
 385 * For instance,
 386 *
 387 * .. code-block:: bash
 388 *
 389 *	echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
 390 *	echo inject umc ce 0 0 0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
 391 *	echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl
 392 *
 393 * How to check the result of the operation?
 394 *
 395 * To check disable/enable, see "ras" features at,
 396 * /sys/class/drm/card[0/1/2...]/device/ras/features
 397 *
 398 * To check inject, see the corresponding error count at,
 399 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count
 400 *
 401 * .. note::
 402 *	Operations are only allowed on blocks which are supported.
 403 *	Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask
 404 *	to see which blocks support RAS on a particular asic.
 405 *
 406 */
 407static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f,
 408					     const char __user *buf,
 409					     size_t size, loff_t *pos)
 410{
 411	struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
 412	struct ras_debug_if data;
 413	int ret = 0;
 414
 415	if (!amdgpu_ras_get_error_query_ready(adev)) {
 416		dev_warn(adev->dev, "RAS WARN: error injection "
 417				"currently inaccessible\n");
 418		return size;
 419	}
 420
 421	ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data);
 422	if (ret)
 423		return ret;
 424
 425	if (data.op == 3) {
 426		ret = amdgpu_reserve_page_direct(adev, data.inject.address);
 427		if (!ret)
 428			return size;
 429		else
 430			return ret;
 431	}
 432
 433	if (!amdgpu_ras_is_supported(adev, data.head.block))
 434		return -EINVAL;
 435
 436	switch (data.op) {
 437	case 0:
 438		ret = amdgpu_ras_feature_enable(adev, &data.head, 0);
 439		break;
 440	case 1:
 441		ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
 442		break;
 443	case 2:
 444		if ((data.inject.address >= adev->gmc.mc_vram_size) ||
 445		    (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
 446			dev_warn(adev->dev, "RAS WARN: input address "
 447					"0x%llx is invalid.",
 448					data.inject.address);
 449			ret = -EINVAL;
 450			break;
 451		}
 452
 453		/* umc ce/ue error injection for a bad page is not allowed */
 454		if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) &&
 455		    amdgpu_ras_check_bad_page(adev, data.inject.address)) {
 456			dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has "
 457				 "already been marked as bad!\n",
 458				 data.inject.address);
 459			break;
 460		}
 461
 462		/* data.inject.address is offset instead of absolute gpu address */
 463		ret = amdgpu_ras_error_inject(adev, &data.inject);
 464		break;
 465	default:
 466		ret = -EINVAL;
 467		break;
 468	}
 469
 470	if (ret)
 471		return ret;
 472
 473	return size;
 474}
 475
 476/**
 477 * DOC: AMDGPU RAS debugfs EEPROM table reset interface
 478 *
 479 * Some boards contain an EEPROM which is used to persistently store a list of
 480 * bad pages which experiences ECC errors in vram.  This interface provides
 481 * a way to reset the EEPROM, e.g., after testing error injection.
 482 *
 483 * Usage:
 484 *
 485 * .. code-block:: bash
 486 *
 487 *	echo 1 > ../ras/ras_eeprom_reset
 488 *
 489 * will reset EEPROM table to 0 entries.
 490 *
 491 */
 492static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f,
 493					       const char __user *buf,
 494					       size_t size, loff_t *pos)
 495{
 496	struct amdgpu_device *adev =
 497		(struct amdgpu_device *)file_inode(f)->i_private;
 498	int ret;
 499
 500	ret = amdgpu_ras_eeprom_reset_table(
 501		&(amdgpu_ras_get_context(adev)->eeprom_control));
 502
 503	if (!ret) {
 504		/* Something was written to EEPROM.
 505		 */
 506		amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS;
 507		return size;
 508	} else {
 509		return ret;
 510	}
 511}
 512
 513static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = {
 514	.owner = THIS_MODULE,
 515	.read = NULL,
 516	.write = amdgpu_ras_debugfs_ctrl_write,
 517	.llseek = default_llseek
 518};
 519
 520static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = {
 521	.owner = THIS_MODULE,
 522	.read = NULL,
 523	.write = amdgpu_ras_debugfs_eeprom_write,
 524	.llseek = default_llseek
 525};
 526
 527/**
 528 * DOC: AMDGPU RAS sysfs Error Count Interface
 529 *
 530 * It allows the user to read the error count for each IP block on the gpu through
 531 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
 532 *
 533 * It outputs the multiple lines which report the uncorrected (ue) and corrected
 534 * (ce) error counts.
 535 *
 536 * The format of one line is below,
 537 *
 538 * [ce|ue]: count
 539 *
 540 * Example:
 541 *
 542 * .. code-block:: bash
 543 *
 544 *	ue: 0
 545 *	ce: 1
 546 *
 547 */
 548static ssize_t amdgpu_ras_sysfs_read(struct device *dev,
 549		struct device_attribute *attr, char *buf)
 550{
 551	struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr);
 552	struct ras_query_if info = {
 553		.head = obj->head,
 554	};
 555
 556	if (!amdgpu_ras_get_error_query_ready(obj->adev))
 557		return sysfs_emit(buf, "Query currently inaccessible\n");
 558
 559	if (amdgpu_ras_query_error_status(obj->adev, &info))
 560		return -EINVAL;
 561
 562	if (obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
 563	    obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
 564		if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
 565			dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
 566	}
 567
 568	return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count,
 569			  "ce", info.ce_count);
 570}
 571
 572/* obj begin */
 573
 574#define get_obj(obj) do { (obj)->use++; } while (0)
 575#define alive_obj(obj) ((obj)->use)
 576
 577static inline void put_obj(struct ras_manager *obj)
 578{
 579	if (obj && (--obj->use == 0))
 580		list_del(&obj->node);
 581	if (obj && (obj->use < 0))
 582		DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head));
 583}
 584
 585/* make one obj and return it. */
 586static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev,
 587		struct ras_common_if *head)
 588{
 589	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 590	struct ras_manager *obj;
 591
 592	if (!adev->ras_enabled || !con)
 593		return NULL;
 594
 595	if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
 596		return NULL;
 597
 598	if (head->block == AMDGPU_RAS_BLOCK__MCA) {
 599		if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
 600			return NULL;
 601
 602		obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
 603	} else
 604		obj = &con->objs[head->block];
 605
 606	/* already exist. return obj? */
 607	if (alive_obj(obj))
 608		return NULL;
 609
 610	obj->head = *head;
 611	obj->adev = adev;
 612	list_add(&obj->node, &con->head);
 613	get_obj(obj);
 614
 615	return obj;
 616}
 617
 618/* return an obj equal to head, or the first when head is NULL */
 619struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev,
 620		struct ras_common_if *head)
 621{
 622	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 623	struct ras_manager *obj;
 624	int i;
 625
 626	if (!adev->ras_enabled || !con)
 627		return NULL;
 628
 629	if (head) {
 630		if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
 631			return NULL;
 632
 633		if (head->block == AMDGPU_RAS_BLOCK__MCA) {
 634			if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
 635				return NULL;
 636
 637			obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
 638		} else
 639			obj = &con->objs[head->block];
 640
 641		if (alive_obj(obj))
 642			return obj;
 643	} else {
 644		for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
 645			obj = &con->objs[i];
 646			if (alive_obj(obj))
 647				return obj;
 648		}
 649	}
 650
 651	return NULL;
 652}
 653/* obj end */
 654
 655/* feature ctl begin */
 656static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
 657					 struct ras_common_if *head)
 658{
 659	return adev->ras_hw_enabled & BIT(head->block);
 660}
 661
 662static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
 663		struct ras_common_if *head)
 664{
 665	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 666
 667	return con->features & BIT(head->block);
 668}
 669
 670/*
 671 * if obj is not created, then create one.
 672 * set feature enable flag.
 673 */
 674static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev,
 675		struct ras_common_if *head, int enable)
 676{
 677	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 678	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
 679
 680	/* If hardware does not support ras, then do not create obj.
 681	 * But if hardware support ras, we can create the obj.
 682	 * Ras framework checks con->hw_supported to see if it need do
 683	 * corresponding initialization.
 684	 * IP checks con->support to see if it need disable ras.
 685	 */
 686	if (!amdgpu_ras_is_feature_allowed(adev, head))
 687		return 0;
 688
 689	if (enable) {
 690		if (!obj) {
 691			obj = amdgpu_ras_create_obj(adev, head);
 692			if (!obj)
 693				return -EINVAL;
 694		} else {
 695			/* In case we create obj somewhere else */
 696			get_obj(obj);
 697		}
 698		con->features |= BIT(head->block);
 699	} else {
 700		if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
 701			con->features &= ~BIT(head->block);
 702			put_obj(obj);
 703		}
 704	}
 705
 706	return 0;
 707}
 708
 709/* wrapper of psp_ras_enable_features */
 710int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
 711		struct ras_common_if *head, bool enable)
 712{
 713	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 714	union ta_ras_cmd_input *info;
 715	int ret;
 716
 717	if (!con)
 718		return -EINVAL;
 719
 720	if (head->block == AMDGPU_RAS_BLOCK__GFX) {
 721		info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL);
 722		if (!info)
 723			return -ENOMEM;
 724
 725		if (!enable) {
 726			info->disable_features = (struct ta_ras_disable_features_input) {
 727				.block_id =  amdgpu_ras_block_to_ta(head->block),
 728				.error_type = amdgpu_ras_error_to_ta(head->type),
 729			};
 730		} else {
 731			info->enable_features = (struct ta_ras_enable_features_input) {
 732				.block_id =  amdgpu_ras_block_to_ta(head->block),
 733				.error_type = amdgpu_ras_error_to_ta(head->type),
 734			};
 735		}
 736	}
 737
 738	/* Do not enable if it is not allowed. */
 739	WARN_ON(enable && !amdgpu_ras_is_feature_allowed(adev, head));
 740
 741	/* Only enable ras feature operation handle on host side */
 742	if (head->block == AMDGPU_RAS_BLOCK__GFX &&
 743		!amdgpu_sriov_vf(adev) &&
 744		!amdgpu_ras_intr_triggered()) {
 745		ret = psp_ras_enable_features(&adev->psp, info, enable);
 746		if (ret) {
 747			dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n",
 748				enable ? "enable":"disable",
 749				get_ras_block_str(head),
 750				amdgpu_ras_is_poison_mode_supported(adev), ret);
 751			goto out;
 752		}
 753	}
 754
 755	/* setup the obj */
 756	__amdgpu_ras_feature_enable(adev, head, enable);
 757	ret = 0;
 758out:
 759	if (head->block == AMDGPU_RAS_BLOCK__GFX)
 760		kfree(info);
 761	return ret;
 762}
 763
 764/* Only used in device probe stage and called only once. */
 765int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
 766		struct ras_common_if *head, bool enable)
 767{
 768	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 769	int ret;
 770
 771	if (!con)
 772		return -EINVAL;
 773
 774	if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
 775		if (enable) {
 776			/* There is no harm to issue a ras TA cmd regardless of
 777			 * the currecnt ras state.
 778			 * If current state == target state, it will do nothing
 779			 * But sometimes it requests driver to reset and repost
 780			 * with error code -EAGAIN.
 781			 */
 782			ret = amdgpu_ras_feature_enable(adev, head, 1);
 783			/* With old ras TA, we might fail to enable ras.
 784			 * Log it and just setup the object.
 785			 * TODO need remove this WA in the future.
 786			 */
 787			if (ret == -EINVAL) {
 788				ret = __amdgpu_ras_feature_enable(adev, head, 1);
 789				if (!ret)
 790					dev_info(adev->dev,
 791						"RAS INFO: %s setup object\n",
 792						get_ras_block_str(head));
 793			}
 794		} else {
 795			/* setup the object then issue a ras TA disable cmd.*/
 796			ret = __amdgpu_ras_feature_enable(adev, head, 1);
 797			if (ret)
 798				return ret;
 799
 800			/* gfx block ras dsiable cmd must send to ras-ta */
 801			if (head->block == AMDGPU_RAS_BLOCK__GFX)
 802				con->features |= BIT(head->block);
 803
 804			ret = amdgpu_ras_feature_enable(adev, head, 0);
 805
 806			/* clean gfx block ras features flag */
 807			if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX)
 808				con->features &= ~BIT(head->block);
 809		}
 810	} else
 811		ret = amdgpu_ras_feature_enable(adev, head, enable);
 812
 813	return ret;
 814}
 815
 816static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev,
 817		bool bypass)
 818{
 819	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 820	struct ras_manager *obj, *tmp;
 821
 822	list_for_each_entry_safe(obj, tmp, &con->head, node) {
 823		/* bypass psp.
 824		 * aka just release the obj and corresponding flags
 825		 */
 826		if (bypass) {
 827			if (__amdgpu_ras_feature_enable(adev, &obj->head, 0))
 828				break;
 829		} else {
 830			if (amdgpu_ras_feature_enable(adev, &obj->head, 0))
 831				break;
 832		}
 833	}
 834
 835	return con->features;
 836}
 837
 838static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev,
 839		bool bypass)
 840{
 841	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 842	int i;
 843	const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE;
 844
 845	for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) {
 846		struct ras_common_if head = {
 847			.block = i,
 848			.type = default_ras_type,
 849			.sub_block_index = 0,
 850		};
 851
 852		if (i == AMDGPU_RAS_BLOCK__MCA)
 853			continue;
 854
 855		if (bypass) {
 856			/*
 857			 * bypass psp. vbios enable ras for us.
 858			 * so just create the obj
 859			 */
 860			if (__amdgpu_ras_feature_enable(adev, &head, 1))
 861				break;
 862		} else {
 863			if (amdgpu_ras_feature_enable(adev, &head, 1))
 864				break;
 865		}
 866	}
 867
 868	for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
 869		struct ras_common_if head = {
 870			.block = AMDGPU_RAS_BLOCK__MCA,
 871			.type = default_ras_type,
 872			.sub_block_index = i,
 873		};
 874
 875		if (bypass) {
 876			/*
 877			 * bypass psp. vbios enable ras for us.
 878			 * so just create the obj
 879			 */
 880			if (__amdgpu_ras_feature_enable(adev, &head, 1))
 881				break;
 882		} else {
 883			if (amdgpu_ras_feature_enable(adev, &head, 1))
 884				break;
 885		}
 886	}
 887
 888	return con->features;
 889}
 890/* feature ctl end */
 891
 892static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj,
 893		enum amdgpu_ras_block block)
 894{
 895	if (!block_obj)
 896		return -EINVAL;
 897
 898	if (block_obj->ras_comm.block == block)
 899		return 0;
 900
 901	return -EINVAL;
 902}
 903
 904static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev,
 905					enum amdgpu_ras_block block, uint32_t sub_block_index)
 906{
 907	struct amdgpu_ras_block_list *node, *tmp;
 908	struct amdgpu_ras_block_object *obj;
 909
 910	if (block >= AMDGPU_RAS_BLOCK__LAST)
 911		return NULL;
 912
 913	if (!amdgpu_ras_is_supported(adev, block))
 914		return NULL;
 915
 916	list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
 917		if (!node->ras_obj) {
 918			dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
 919			continue;
 920		}
 921
 922		obj = node->ras_obj;
 923		if (obj->ras_block_match) {
 924			if (obj->ras_block_match(obj, block, sub_block_index) == 0)
 925				return obj;
 926		} else {
 927			if (amdgpu_ras_block_match_default(obj, block) == 0)
 928				return obj;
 929		}
 930	}
 931
 932	return NULL;
 933}
 934
 935static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data)
 936{
 937	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 938	int ret = 0;
 939
 940	/*
 941	 * choosing right query method according to
 942	 * whether smu support query error information
 943	 */
 944	ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc));
 945	if (ret == -EOPNOTSUPP) {
 946		if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
 947			adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
 948			adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
 949
 950		/* umc query_ras_error_address is also responsible for clearing
 951		 * error status
 952		 */
 953		if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
 954		    adev->umc.ras->ras_block.hw_ops->query_ras_error_address)
 955			adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data);
 956	} else if (!ret) {
 957		if (adev->umc.ras &&
 958			adev->umc.ras->ecc_info_query_ras_error_count)
 959			adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data);
 960
 961		if (adev->umc.ras &&
 962			adev->umc.ras->ecc_info_query_ras_error_address)
 963			adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data);
 964	}
 965}
 966
 967/* query/inject/cure begin */
 968int amdgpu_ras_query_error_status(struct amdgpu_device *adev,
 969				  struct ras_query_if *info)
 970{
 971	struct amdgpu_ras_block_object *block_obj = NULL;
 972	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
 973	struct ras_err_data err_data = {0, 0, 0, NULL};
 974
 975	if (!obj)
 976		return -EINVAL;
 977
 978	if (info->head.block == AMDGPU_RAS_BLOCK__UMC) {
 979		amdgpu_ras_get_ecc_info(adev, &err_data);
 980	} else {
 981		block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0);
 982		if (!block_obj || !block_obj->hw_ops)   {
 983			dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
 984				     get_ras_block_str(&info->head));
 985			return -EINVAL;
 986		}
 987
 988		if (block_obj->hw_ops->query_ras_error_count)
 989			block_obj->hw_ops->query_ras_error_count(adev, &err_data);
 990
 991		if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) ||
 992		    (info->head.block == AMDGPU_RAS_BLOCK__GFX) ||
 993		    (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) {
 994				if (block_obj->hw_ops->query_ras_error_status)
 995					block_obj->hw_ops->query_ras_error_status(adev);
 996			}
 997	}
 998
 999	obj->err_data.ue_count += err_data.ue_count;
1000	obj->err_data.ce_count += err_data.ce_count;
1001
1002	info->ue_count = obj->err_data.ue_count;
1003	info->ce_count = obj->err_data.ce_count;
1004
1005	if (err_data.ce_count) {
1006		if (adev->smuio.funcs &&
1007		    adev->smuio.funcs->get_socket_id &&
1008		    adev->smuio.funcs->get_die_id) {
1009			dev_info(adev->dev, "socket: %d, die: %d "
1010					"%ld correctable hardware errors "
1011					"detected in %s block, no user "
1012					"action is needed.\n",
1013					adev->smuio.funcs->get_socket_id(adev),
1014					adev->smuio.funcs->get_die_id(adev),
1015					obj->err_data.ce_count,
1016					get_ras_block_str(&info->head));
1017		} else {
1018			dev_info(adev->dev, "%ld correctable hardware errors "
1019					"detected in %s block, no user "
1020					"action is needed.\n",
1021					obj->err_data.ce_count,
1022					get_ras_block_str(&info->head));
1023		}
1024	}
1025	if (err_data.ue_count) {
1026		if (adev->smuio.funcs &&
1027		    adev->smuio.funcs->get_socket_id &&
1028		    adev->smuio.funcs->get_die_id) {
1029			dev_info(adev->dev, "socket: %d, die: %d "
1030					"%ld uncorrectable hardware errors "
1031					"detected in %s block\n",
1032					adev->smuio.funcs->get_socket_id(adev),
1033					adev->smuio.funcs->get_die_id(adev),
1034					obj->err_data.ue_count,
1035					get_ras_block_str(&info->head));
1036		} else {
1037			dev_info(adev->dev, "%ld uncorrectable hardware errors "
1038					"detected in %s block\n",
1039					obj->err_data.ue_count,
1040					get_ras_block_str(&info->head));
1041		}
1042	}
1043
1044	return 0;
1045}
1046
1047int amdgpu_ras_reset_error_status(struct amdgpu_device *adev,
1048		enum amdgpu_ras_block block)
1049{
1050	struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
1051
1052	if (!amdgpu_ras_is_supported(adev, block))
1053		return -EINVAL;
1054
1055	if (!block_obj || !block_obj->hw_ops)   {
1056		dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1057			     ras_block_str(block));
1058		return -EINVAL;
1059	}
1060
1061	if (block_obj->hw_ops->reset_ras_error_count)
1062		block_obj->hw_ops->reset_ras_error_count(adev);
1063
1064	if ((block == AMDGPU_RAS_BLOCK__GFX) ||
1065	    (block == AMDGPU_RAS_BLOCK__MMHUB)) {
1066		if (block_obj->hw_ops->reset_ras_error_status)
1067			block_obj->hw_ops->reset_ras_error_status(adev);
1068	}
1069
1070	return 0;
1071}
1072
1073/* wrapper of psp_ras_trigger_error */
1074int amdgpu_ras_error_inject(struct amdgpu_device *adev,
1075		struct ras_inject_if *info)
1076{
1077	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1078	struct ta_ras_trigger_error_input block_info = {
1079		.block_id =  amdgpu_ras_block_to_ta(info->head.block),
1080		.inject_error_type = amdgpu_ras_error_to_ta(info->head.type),
1081		.sub_block_index = info->head.sub_block_index,
1082		.address = info->address,
1083		.value = info->value,
1084	};
1085	int ret = -EINVAL;
1086	struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev,
1087							info->head.block,
1088							info->head.sub_block_index);
1089
1090	if (!obj)
1091		return -EINVAL;
1092
1093	if (!block_obj || !block_obj->hw_ops)	{
1094		dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1095			     get_ras_block_str(&info->head));
1096		return -EINVAL;
1097	}
1098
1099	/* Calculate XGMI relative offset */
1100	if (adev->gmc.xgmi.num_physical_nodes > 1) {
1101		block_info.address =
1102			amdgpu_xgmi_get_relative_phy_addr(adev,
1103							  block_info.address);
1104	}
1105
1106	if (info->head.block == AMDGPU_RAS_BLOCK__GFX) {
1107		if (block_obj->hw_ops->ras_error_inject)
1108			ret = block_obj->hw_ops->ras_error_inject(adev, info);
1109	} else {
1110		/* If defined special ras_error_inject(e.g: xgmi), implement special ras_error_inject */
1111		if (block_obj->hw_ops->ras_error_inject)
1112			ret = block_obj->hw_ops->ras_error_inject(adev, &block_info);
1113		else  /*If not defined .ras_error_inject, use default ras_error_inject*/
1114			ret = psp_ras_trigger_error(&adev->psp, &block_info);
1115	}
1116
1117	if (ret)
1118		dev_err(adev->dev, "ras inject %s failed %d\n",
1119			get_ras_block_str(&info->head), ret);
1120
1121	return ret;
1122}
1123
1124/**
1125 * amdgpu_ras_query_error_count -- Get error counts of all IPs
1126 * @adev: pointer to AMD GPU device
1127 * @ce_count: pointer to an integer to be set to the count of correctible errors.
1128 * @ue_count: pointer to an integer to be set to the count of uncorrectible
1129 * errors.
1130 *
1131 * If set, @ce_count or @ue_count, count and return the corresponding
1132 * error counts in those integer pointers. Return 0 if the device
1133 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS.
1134 */
1135int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
1136				 unsigned long *ce_count,
1137				 unsigned long *ue_count)
1138{
1139	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1140	struct ras_manager *obj;
1141	unsigned long ce, ue;
1142
1143	if (!adev->ras_enabled || !con)
1144		return -EOPNOTSUPP;
1145
1146	/* Don't count since no reporting.
1147	 */
1148	if (!ce_count && !ue_count)
1149		return 0;
1150
1151	ce = 0;
1152	ue = 0;
1153	list_for_each_entry(obj, &con->head, node) {
1154		struct ras_query_if info = {
1155			.head = obj->head,
1156		};
1157		int res;
1158
1159		res = amdgpu_ras_query_error_status(adev, &info);
1160		if (res)
1161			return res;
1162
1163		if (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
1164		    adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
1165			if (amdgpu_ras_reset_error_status(adev, info.head.block))
1166				dev_warn(adev->dev, "Failed to reset error counter and error status");
1167		}
1168
1169		ce += info.ce_count;
1170		ue += info.ue_count;
1171	}
1172
1173	if (ce_count)
1174		*ce_count = ce;
1175
1176	if (ue_count)
1177		*ue_count = ue;
1178
1179	return 0;
1180}
1181/* query/inject/cure end */
1182
1183
1184/* sysfs begin */
1185
1186static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1187		struct ras_badpage **bps, unsigned int *count);
1188
1189static char *amdgpu_ras_badpage_flags_str(unsigned int flags)
1190{
1191	switch (flags) {
1192	case AMDGPU_RAS_RETIRE_PAGE_RESERVED:
1193		return "R";
1194	case AMDGPU_RAS_RETIRE_PAGE_PENDING:
1195		return "P";
1196	case AMDGPU_RAS_RETIRE_PAGE_FAULT:
1197	default:
1198		return "F";
1199	}
1200}
1201
1202/**
1203 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface
1204 *
1205 * It allows user to read the bad pages of vram on the gpu through
1206 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages
1207 *
1208 * It outputs multiple lines, and each line stands for one gpu page.
1209 *
1210 * The format of one line is below,
1211 * gpu pfn : gpu page size : flags
1212 *
1213 * gpu pfn and gpu page size are printed in hex format.
1214 * flags can be one of below character,
1215 *
1216 * R: reserved, this gpu page is reserved and not able to use.
1217 *
1218 * P: pending for reserve, this gpu page is marked as bad, will be reserved
1219 * in next window of page_reserve.
1220 *
1221 * F: unable to reserve. this gpu page can't be reserved due to some reasons.
1222 *
1223 * Examples:
1224 *
1225 * .. code-block:: bash
1226 *
1227 *	0x00000001 : 0x00001000 : R
1228 *	0x00000002 : 0x00001000 : P
1229 *
1230 */
1231
1232static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f,
1233		struct kobject *kobj, struct bin_attribute *attr,
1234		char *buf, loff_t ppos, size_t count)
1235{
1236	struct amdgpu_ras *con =
1237		container_of(attr, struct amdgpu_ras, badpages_attr);
1238	struct amdgpu_device *adev = con->adev;
1239	const unsigned int element_size =
1240		sizeof("0xabcdabcd : 0x12345678 : R\n") - 1;
1241	unsigned int start = div64_ul(ppos + element_size - 1, element_size);
1242	unsigned int end = div64_ul(ppos + count - 1, element_size);
1243	ssize_t s = 0;
1244	struct ras_badpage *bps = NULL;
1245	unsigned int bps_count = 0;
1246
1247	memset(buf, 0, count);
1248
1249	if (amdgpu_ras_badpages_read(adev, &bps, &bps_count))
1250		return 0;
1251
1252	for (; start < end && start < bps_count; start++)
1253		s += scnprintf(&buf[s], element_size + 1,
1254				"0x%08x : 0x%08x : %1s\n",
1255				bps[start].bp,
1256				bps[start].size,
1257				amdgpu_ras_badpage_flags_str(bps[start].flags));
1258
1259	kfree(bps);
1260
1261	return s;
1262}
1263
1264static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
1265		struct device_attribute *attr, char *buf)
1266{
1267	struct amdgpu_ras *con =
1268		container_of(attr, struct amdgpu_ras, features_attr);
1269
1270	return sysfs_emit(buf, "feature mask: 0x%x\n", con->features);
1271}
1272
1273static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev)
1274{
1275	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1276
1277	sysfs_remove_file_from_group(&adev->dev->kobj,
1278				&con->badpages_attr.attr,
1279				RAS_FS_NAME);
1280}
1281
1282static int amdgpu_ras_sysfs_remove_feature_node(struct amdgpu_device *adev)
1283{
1284	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1285	struct attribute *attrs[] = {
1286		&con->features_attr.attr,
1287		NULL
1288	};
1289	struct attribute_group group = {
1290		.name = RAS_FS_NAME,
1291		.attrs = attrs,
1292	};
1293
1294	sysfs_remove_group(&adev->dev->kobj, &group);
1295
1296	return 0;
1297}
1298
1299int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
1300		struct ras_common_if *head)
1301{
1302	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1303
1304	if (!obj || obj->attr_inuse)
1305		return -EINVAL;
1306
1307	get_obj(obj);
1308
1309	snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name),
1310		"%s_err_count", head->name);
1311
1312	obj->sysfs_attr = (struct device_attribute){
1313		.attr = {
1314			.name = obj->fs_data.sysfs_name,
1315			.mode = S_IRUGO,
1316		},
1317			.show = amdgpu_ras_sysfs_read,
1318	};
1319	sysfs_attr_init(&obj->sysfs_attr.attr);
1320
1321	if (sysfs_add_file_to_group(&adev->dev->kobj,
1322				&obj->sysfs_attr.attr,
1323				RAS_FS_NAME)) {
1324		put_obj(obj);
1325		return -EINVAL;
1326	}
1327
1328	obj->attr_inuse = 1;
1329
1330	return 0;
1331}
1332
1333int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
1334		struct ras_common_if *head)
1335{
1336	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1337
1338	if (!obj || !obj->attr_inuse)
1339		return -EINVAL;
1340
1341	sysfs_remove_file_from_group(&adev->dev->kobj,
1342				&obj->sysfs_attr.attr,
1343				RAS_FS_NAME);
1344	obj->attr_inuse = 0;
1345	put_obj(obj);
1346
1347	return 0;
1348}
1349
1350static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
1351{
1352	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1353	struct ras_manager *obj, *tmp;
1354
1355	list_for_each_entry_safe(obj, tmp, &con->head, node) {
1356		amdgpu_ras_sysfs_remove(adev, &obj->head);
1357	}
1358
1359	if (amdgpu_bad_page_threshold != 0)
1360		amdgpu_ras_sysfs_remove_bad_page_node(adev);
1361
1362	amdgpu_ras_sysfs_remove_feature_node(adev);
1363
1364	return 0;
1365}
1366/* sysfs end */
1367
1368/**
1369 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors
1370 *
1371 * Normally when there is an uncorrectable error, the driver will reset
1372 * the GPU to recover.  However, in the event of an unrecoverable error,
1373 * the driver provides an interface to reboot the system automatically
1374 * in that event.
1375 *
1376 * The following file in debugfs provides that interface:
1377 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot
1378 *
1379 * Usage:
1380 *
1381 * .. code-block:: bash
1382 *
1383 *	echo true > .../ras/auto_reboot
1384 *
1385 */
1386/* debugfs begin */
1387static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
1388{
1389	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1390	struct drm_minor  *minor = adev_to_drm(adev)->primary;
1391	struct dentry     *dir;
1392
1393	dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root);
1394	debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev,
1395			    &amdgpu_ras_debugfs_ctrl_ops);
1396	debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev,
1397			    &amdgpu_ras_debugfs_eeprom_ops);
1398	debugfs_create_u32("bad_page_cnt_threshold", 0444, dir,
1399			   &con->bad_page_cnt_threshold);
1400	debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled);
1401	debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled);
1402	debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev,
1403			    &amdgpu_ras_debugfs_eeprom_size_ops);
1404	con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table",
1405						       S_IRUGO, dir, adev,
1406						       &amdgpu_ras_debugfs_eeprom_table_ops);
1407	amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control);
1408
1409	/*
1410	 * After one uncorrectable error happens, usually GPU recovery will
1411	 * be scheduled. But due to the known problem in GPU recovery failing
1412	 * to bring GPU back, below interface provides one direct way to
1413	 * user to reboot system automatically in such case within
1414	 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine
1415	 * will never be called.
1416	 */
1417	debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot);
1418
1419	/*
1420	 * User could set this not to clean up hardware's error count register
1421	 * of RAS IPs during ras recovery.
1422	 */
1423	debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir,
1424			    &con->disable_ras_err_cnt_harvest);
1425	return dir;
1426}
1427
1428static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
1429				      struct ras_fs_if *head,
1430				      struct dentry *dir)
1431{
1432	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
1433
1434	if (!obj || !dir)
1435		return;
1436
1437	get_obj(obj);
1438
1439	memcpy(obj->fs_data.debugfs_name,
1440			head->debugfs_name,
1441			sizeof(obj->fs_data.debugfs_name));
1442
1443	debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir,
1444			    obj, &amdgpu_ras_debugfs_ops);
1445}
1446
1447void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
1448{
1449	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1450	struct dentry *dir;
1451	struct ras_manager *obj;
1452	struct ras_fs_if fs_info;
1453
1454	/*
1455	 * it won't be called in resume path, no need to check
1456	 * suspend and gpu reset status
1457	 */
1458	if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con)
1459		return;
1460
1461	dir = amdgpu_ras_debugfs_create_ctrl_node(adev);
1462
1463	list_for_each_entry(obj, &con->head, node) {
1464		if (amdgpu_ras_is_supported(adev, obj->head.block) &&
1465			(obj->attr_inuse == 1)) {
1466			sprintf(fs_info.debugfs_name, "%s_err_inject",
1467					get_ras_block_str(&obj->head));
1468			fs_info.head = obj->head;
1469			amdgpu_ras_debugfs_create(adev, &fs_info, dir);
1470		}
1471	}
1472}
1473
1474/* debugfs end */
1475
1476/* ras fs */
1477static BIN_ATTR(gpu_vram_bad_pages, S_IRUGO,
1478		amdgpu_ras_sysfs_badpages_read, NULL, 0);
1479static DEVICE_ATTR(features, S_IRUGO,
1480		amdgpu_ras_sysfs_features_read, NULL);
1481static int amdgpu_ras_fs_init(struct amdgpu_device *adev)
1482{
1483	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1484	struct attribute_group group = {
1485		.name = RAS_FS_NAME,
1486	};
1487	struct attribute *attrs[] = {
1488		&con->features_attr.attr,
1489		NULL
1490	};
1491	struct bin_attribute *bin_attrs[] = {
1492		NULL,
1493		NULL,
1494	};
1495	int r;
1496
1497	/* add features entry */
1498	con->features_attr = dev_attr_features;
1499	group.attrs = attrs;
1500	sysfs_attr_init(attrs[0]);
1501
1502	if (amdgpu_bad_page_threshold != 0) {
1503		/* add bad_page_features entry */
1504		bin_attr_gpu_vram_bad_pages.private = NULL;
1505		con->badpages_attr = bin_attr_gpu_vram_bad_pages;
1506		bin_attrs[0] = &con->badpages_attr;
1507		group.bin_attrs = bin_attrs;
1508		sysfs_bin_attr_init(bin_attrs[0]);
1509	}
1510
1511	r = sysfs_create_group(&adev->dev->kobj, &group);
1512	if (r)
1513		dev_err(adev->dev, "Failed to create RAS sysfs group!");
1514
1515	return 0;
1516}
1517
1518static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
1519{
1520	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1521	struct ras_manager *con_obj, *ip_obj, *tmp;
1522
1523	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
1524		list_for_each_entry_safe(con_obj, tmp, &con->head, node) {
1525			ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head);
1526			if (ip_obj)
1527				put_obj(ip_obj);
1528		}
1529	}
1530
1531	amdgpu_ras_sysfs_remove_all(adev);
1532	return 0;
1533}
1534/* ras fs end */
1535
1536/* ih begin */
1537
1538/* For the hardware that cannot enable bif ring for both ras_controller_irq
1539 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status
1540 * register to check whether the interrupt is triggered or not, and properly
1541 * ack the interrupt if it is there
1542 */
1543void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev)
1544{
1545	/* Fatal error events are handled on host side */
1546	if (amdgpu_sriov_vf(adev) ||
1547		!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__PCIE_BIF))
1548		return;
1549
1550	if (adev->nbio.ras &&
1551	    adev->nbio.ras->handle_ras_controller_intr_no_bifring)
1552		adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev);
1553
1554	if (adev->nbio.ras &&
1555	    adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring)
1556		adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev);
1557}
1558
1559static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj,
1560				struct amdgpu_iv_entry *entry)
1561{
1562	bool poison_stat = false;
1563	struct amdgpu_device *adev = obj->adev;
1564	struct amdgpu_ras_block_object *block_obj =
1565		amdgpu_ras_get_ras_block(adev, obj->head.block, 0);
1566
1567	if (!block_obj || !block_obj->hw_ops)
1568		return;
1569
1570	/* both query_poison_status and handle_poison_consumption are optional,
1571	 * but at least one of them should be implemented if we need poison
1572	 * consumption handler
1573	 */
1574	if (block_obj->hw_ops->query_poison_status) {
1575		poison_stat = block_obj->hw_ops->query_poison_status(adev);
1576		if (!poison_stat) {
1577			/* Not poison consumption interrupt, no need to handle it */
1578			dev_info(adev->dev, "No RAS poison status in %s poison IH.\n",
1579					block_obj->ras_comm.name);
1580
1581			return;
1582		}
1583	}
1584
1585	if (!adev->gmc.xgmi.connected_to_cpu)
1586		amdgpu_umc_poison_handler(adev, false);
1587
1588	if (block_obj->hw_ops->handle_poison_consumption)
1589		poison_stat = block_obj->hw_ops->handle_poison_consumption(adev);
1590
1591	/* gpu reset is fallback for failed and default cases */
1592	if (poison_stat) {
1593		dev_info(adev->dev, "GPU reset for %s RAS poison consumption is issued!\n",
1594				block_obj->ras_comm.name);
1595		amdgpu_ras_reset_gpu(adev);
1596	}
1597}
1598
1599static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj,
1600				struct amdgpu_iv_entry *entry)
1601{
1602	dev_info(obj->adev->dev,
1603		"Poison is created, no user action is needed.\n");
1604}
1605
1606static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
1607				struct amdgpu_iv_entry *entry)
1608{
1609	struct ras_ih_data *data = &obj->ih_data;
1610	struct ras_err_data err_data = {0, 0, 0, NULL};
1611	int ret;
1612
1613	if (!data->cb)
1614		return;
1615
1616	/* Let IP handle its data, maybe we need get the output
1617	 * from the callback to update the error type/count, etc
1618	 */
1619	ret = data->cb(obj->adev, &err_data, entry);
1620	/* ue will trigger an interrupt, and in that case
1621	 * we need do a reset to recovery the whole system.
1622	 * But leave IP do that recovery, here we just dispatch
1623	 * the error.
1624	 */
1625	if (ret == AMDGPU_RAS_SUCCESS) {
1626		/* these counts could be left as 0 if
1627		 * some blocks do not count error number
1628		 */
1629		obj->err_data.ue_count += err_data.ue_count;
1630		obj->err_data.ce_count += err_data.ce_count;
1631	}
1632}
1633
1634static void amdgpu_ras_interrupt_handler(struct ras_manager *obj)
1635{
1636	struct ras_ih_data *data = &obj->ih_data;
1637	struct amdgpu_iv_entry entry;
1638
1639	while (data->rptr != data->wptr) {
1640		rmb();
1641		memcpy(&entry, &data->ring[data->rptr],
1642				data->element_size);
1643
1644		wmb();
1645		data->rptr = (data->aligned_element_size +
1646				data->rptr) % data->ring_size;
1647
1648		if (amdgpu_ras_is_poison_mode_supported(obj->adev)) {
1649			if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
1650				amdgpu_ras_interrupt_poison_creation_handler(obj, &entry);
1651			else
1652				amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry);
1653		} else {
1654			if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
1655				amdgpu_ras_interrupt_umc_handler(obj, &entry);
1656			else
1657				dev_warn(obj->adev->dev,
1658					"No RAS interrupt handler for non-UMC block with poison disabled.\n");
1659		}
1660	}
1661}
1662
1663static void amdgpu_ras_interrupt_process_handler(struct work_struct *work)
1664{
1665	struct ras_ih_data *data =
1666		container_of(work, struct ras_ih_data, ih_work);
1667	struct ras_manager *obj =
1668		container_of(data, struct ras_manager, ih_data);
1669
1670	amdgpu_ras_interrupt_handler(obj);
1671}
1672
1673int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
1674		struct ras_dispatch_if *info)
1675{
1676	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1677	struct ras_ih_data *data = &obj->ih_data;
1678
1679	if (!obj)
1680		return -EINVAL;
1681
1682	if (data->inuse == 0)
1683		return 0;
1684
1685	/* Might be overflow... */
1686	memcpy(&data->ring[data->wptr], info->entry,
1687			data->element_size);
1688
1689	wmb();
1690	data->wptr = (data->aligned_element_size +
1691			data->wptr) % data->ring_size;
1692
1693	schedule_work(&data->ih_work);
1694
1695	return 0;
1696}
1697
1698int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
1699		struct ras_common_if *head)
1700{
1701	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1702	struct ras_ih_data *data;
1703
1704	if (!obj)
1705		return -EINVAL;
1706
1707	data = &obj->ih_data;
1708	if (data->inuse == 0)
1709		return 0;
1710
1711	cancel_work_sync(&data->ih_work);
1712
1713	kfree(data->ring);
1714	memset(data, 0, sizeof(*data));
1715	put_obj(obj);
1716
1717	return 0;
1718}
1719
1720int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
1721		struct ras_common_if *head)
1722{
1723	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1724	struct ras_ih_data *data;
1725	struct amdgpu_ras_block_object *ras_obj;
1726
1727	if (!obj) {
1728		/* in case we registe the IH before enable ras feature */
1729		obj = amdgpu_ras_create_obj(adev, head);
1730		if (!obj)
1731			return -EINVAL;
1732	} else
1733		get_obj(obj);
1734
1735	ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm);
1736
1737	data = &obj->ih_data;
1738	/* add the callback.etc */
1739	*data = (struct ras_ih_data) {
1740		.inuse = 0,
1741		.cb = ras_obj->ras_cb,
1742		.element_size = sizeof(struct amdgpu_iv_entry),
1743		.rptr = 0,
1744		.wptr = 0,
1745	};
1746
1747	INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler);
1748
1749	data->aligned_element_size = ALIGN(data->element_size, 8);
1750	/* the ring can store 64 iv entries. */
1751	data->ring_size = 64 * data->aligned_element_size;
1752	data->ring = kmalloc(data->ring_size, GFP_KERNEL);
1753	if (!data->ring) {
1754		put_obj(obj);
1755		return -ENOMEM;
1756	}
1757
1758	/* IH is ready */
1759	data->inuse = 1;
1760
1761	return 0;
1762}
1763
1764static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
1765{
1766	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1767	struct ras_manager *obj, *tmp;
1768
1769	list_for_each_entry_safe(obj, tmp, &con->head, node) {
1770		amdgpu_ras_interrupt_remove_handler(adev, &obj->head);
1771	}
1772
1773	return 0;
1774}
1775/* ih end */
1776
1777/* traversal all IPs except NBIO to query error counter */
1778static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev)
1779{
1780	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1781	struct ras_manager *obj;
1782
1783	if (!adev->ras_enabled || !con)
1784		return;
1785
1786	list_for_each_entry(obj, &con->head, node) {
1787		struct ras_query_if info = {
1788			.head = obj->head,
1789		};
1790
1791		/*
1792		 * PCIE_BIF IP has one different isr by ras controller
1793		 * interrupt, the specific ras counter query will be
1794		 * done in that isr. So skip such block from common
1795		 * sync flood interrupt isr calling.
1796		 */
1797		if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF)
1798			continue;
1799
1800		/*
1801		 * this is a workaround for aldebaran, skip send msg to
1802		 * smu to get ecc_info table due to smu handle get ecc
1803		 * info table failed temporarily.
1804		 * should be removed until smu fix handle ecc_info table.
1805		 */
1806		if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) &&
1807			(adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 2)))
1808			continue;
1809
1810		amdgpu_ras_query_error_status(adev, &info);
1811
1812		if (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
1813		    adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4) &&
1814		    adev->ip_versions[MP0_HWIP][0] != IP_VERSION(13, 0, 0)) {
1815			if (amdgpu_ras_reset_error_status(adev, info.head.block))
1816				dev_warn(adev->dev, "Failed to reset error counter and error status");
1817		}
1818	}
1819}
1820
1821/* Parse RdRspStatus and WrRspStatus */
1822static void amdgpu_ras_error_status_query(struct amdgpu_device *adev,
1823					  struct ras_query_if *info)
1824{
1825	struct amdgpu_ras_block_object *block_obj;
1826	/*
1827	 * Only two block need to query read/write
1828	 * RspStatus at current state
1829	 */
1830	if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) &&
1831		(info->head.block != AMDGPU_RAS_BLOCK__MMHUB))
1832		return;
1833
1834	block_obj = amdgpu_ras_get_ras_block(adev,
1835					info->head.block,
1836					info->head.sub_block_index);
1837
1838	if (!block_obj || !block_obj->hw_ops) {
1839		dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1840			     get_ras_block_str(&info->head));
1841		return;
1842	}
1843
1844	if (block_obj->hw_ops->query_ras_error_status)
1845		block_obj->hw_ops->query_ras_error_status(adev);
1846
1847}
1848
1849static void amdgpu_ras_query_err_status(struct amdgpu_device *adev)
1850{
1851	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1852	struct ras_manager *obj;
1853
1854	if (!adev->ras_enabled || !con)
1855		return;
1856
1857	list_for_each_entry(obj, &con->head, node) {
1858		struct ras_query_if info = {
1859			.head = obj->head,
1860		};
1861
1862		amdgpu_ras_error_status_query(adev, &info);
1863	}
1864}
1865
1866/* recovery begin */
1867
1868/* return 0 on success.
1869 * caller need free bps.
1870 */
1871static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1872		struct ras_badpage **bps, unsigned int *count)
1873{
1874	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1875	struct ras_err_handler_data *data;
1876	int i = 0;
1877	int ret = 0, status;
1878
1879	if (!con || !con->eh_data || !bps || !count)
1880		return -EINVAL;
1881
1882	mutex_lock(&con->recovery_lock);
1883	data = con->eh_data;
1884	if (!data || data->count == 0) {
1885		*bps = NULL;
1886		ret = -EINVAL;
1887		goto out;
1888	}
1889
1890	*bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL);
1891	if (!*bps) {
1892		ret = -ENOMEM;
1893		goto out;
1894	}
1895
1896	for (; i < data->count; i++) {
1897		(*bps)[i] = (struct ras_badpage){
1898			.bp = data->bps[i].retired_page,
1899			.size = AMDGPU_GPU_PAGE_SIZE,
1900			.flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED,
1901		};
1902		status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr,
1903				data->bps[i].retired_page);
1904		if (status == -EBUSY)
1905			(*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING;
1906		else if (status == -ENOENT)
1907			(*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT;
1908	}
1909
1910	*count = data->count;
1911out:
1912	mutex_unlock(&con->recovery_lock);
1913	return ret;
1914}
1915
1916static void amdgpu_ras_do_recovery(struct work_struct *work)
1917{
1918	struct amdgpu_ras *ras =
1919		container_of(work, struct amdgpu_ras, recovery_work);
1920	struct amdgpu_device *remote_adev = NULL;
1921	struct amdgpu_device *adev = ras->adev;
1922	struct list_head device_list, *device_list_handle =  NULL;
1923
1924	if (!ras->disable_ras_err_cnt_harvest) {
1925		struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
1926
1927		/* Build list of devices to query RAS related errors */
1928		if  (hive && adev->gmc.xgmi.num_physical_nodes > 1) {
1929			device_list_handle = &hive->device_list;
1930		} else {
1931			INIT_LIST_HEAD(&device_list);
1932			list_add_tail(&adev->gmc.xgmi.head, &device_list);
1933			device_list_handle = &device_list;
1934		}
1935
1936		list_for_each_entry(remote_adev,
1937				device_list_handle, gmc.xgmi.head) {
1938			amdgpu_ras_query_err_status(remote_adev);
1939			amdgpu_ras_log_on_err_counter(remote_adev);
1940		}
1941
1942		amdgpu_put_xgmi_hive(hive);
1943	}
1944
1945	if (amdgpu_device_should_recover_gpu(ras->adev)) {
1946		struct amdgpu_reset_context reset_context;
1947		memset(&reset_context, 0, sizeof(reset_context));
1948
1949		reset_context.method = AMD_RESET_METHOD_NONE;
1950		reset_context.reset_req_dev = adev;
1951
1952		/* Perform full reset in fatal error mode */
1953		if (!amdgpu_ras_is_poison_mode_supported(ras->adev))
1954			set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
1955		else
1956			clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
1957
1958		amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context);
1959	}
1960	atomic_set(&ras->in_recovery, 0);
1961}
1962
1963/* alloc/realloc bps array */
1964static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
1965		struct ras_err_handler_data *data, int pages)
1966{
1967	unsigned int old_space = data->count + data->space_left;
1968	unsigned int new_space = old_space + pages;
1969	unsigned int align_space = ALIGN(new_space, 512);
1970	void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL);
1971
1972	if (!bps) {
1973		return -ENOMEM;
1974	}
1975
1976	if (data->bps) {
1977		memcpy(bps, data->bps,
1978				data->count * sizeof(*data->bps));
1979		kfree(data->bps);
1980	}
1981
1982	data->bps = bps;
1983	data->space_left += align_space - old_space;
1984	return 0;
1985}
1986
1987/* it deal with vram only. */
1988int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
1989		struct eeprom_table_record *bps, int pages)
1990{
1991	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1992	struct ras_err_handler_data *data;
1993	int ret = 0;
1994	uint32_t i;
1995
1996	if (!con || !con->eh_data || !bps || pages <= 0)
1997		return 0;
1998
1999	mutex_lock(&con->recovery_lock);
2000	data = con->eh_data;
2001	if (!data)
2002		goto out;
2003
2004	for (i = 0; i < pages; i++) {
2005		if (amdgpu_ras_check_bad_page_unlock(con,
2006			bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT))
2007			continue;
2008
2009		if (!data->space_left &&
2010			amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {
2011			ret = -ENOMEM;
2012			goto out;
2013		}
2014
2015		amdgpu_vram_mgr_reserve_range(&adev->mman.vram_mgr,
2016			bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT,
2017			AMDGPU_GPU_PAGE_SIZE);
2018
2019		memcpy(&data->bps[data->count], &bps[i], sizeof(*data->bps));
2020		data->count++;
2021		data->space_left--;
2022	}
2023out:
2024	mutex_unlock(&con->recovery_lock);
2025
2026	return ret;
2027}
2028
2029/*
2030 * write error record array to eeprom, the function should be
2031 * protected by recovery_lock
2032 */
2033int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev)
2034{
2035	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2036	struct ras_err_handler_data *data;
2037	struct amdgpu_ras_eeprom_control *control;
2038	int save_count;
2039
2040	if (!con || !con->eh_data)
2041		return 0;
2042
2043	mutex_lock(&con->recovery_lock);
2044	control = &con->eeprom_control;
2045	data = con->eh_data;
2046	save_count = data->count - control->ras_num_recs;
2047	mutex_unlock(&con->recovery_lock);
2048	/* only new entries are saved */
2049	if (save_count > 0) {
2050		if (amdgpu_ras_eeprom_append(control,
2051					     &data->bps[control->ras_num_recs],
2052					     save_count)) {
2053			dev_err(adev->dev, "Failed to save EEPROM table data!");
2054			return -EIO;
2055		}
2056
2057		dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count);
2058	}
2059
2060	return 0;
2061}
2062
2063/*
2064 * read error record array in eeprom and reserve enough space for
2065 * storing new bad pages
2066 */
2067static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
2068{
2069	struct amdgpu_ras_eeprom_control *control =
2070		&adev->psp.ras_context.ras->eeprom_control;
2071	struct eeprom_table_record *bps;
2072	int ret;
2073
2074	/* no bad page record, skip eeprom access */
2075	if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0)
2076		return 0;
2077
2078	bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL);
2079	if (!bps)
2080		return -ENOMEM;
2081
2082	ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs);
2083	if (ret)
2084		dev_err(adev->dev, "Failed to load EEPROM table records!");
2085	else
2086		ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs);
2087
2088	kfree(bps);
2089	return ret;
2090}
2091
2092static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
2093				uint64_t addr)
2094{
2095	struct ras_err_handler_data *data = con->eh_data;
2096	int i;
2097
2098	addr >>= AMDGPU_GPU_PAGE_SHIFT;
2099	for (i = 0; i < data->count; i++)
2100		if (addr == data->bps[i].retired_page)
2101			return true;
2102
2103	return false;
2104}
2105
2106/*
2107 * check if an address belongs to bad page
2108 *
2109 * Note: this check is only for umc block
2110 */
2111static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
2112				uint64_t addr)
2113{
2114	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2115	bool ret = false;
2116
2117	if (!con || !con->eh_data)
2118		return ret;
2119
2120	mutex_lock(&con->recovery_lock);
2121	ret = amdgpu_ras_check_bad_page_unlock(con, addr);
2122	mutex_unlock(&con->recovery_lock);
2123	return ret;
2124}
2125
2126static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev,
2127					  uint32_t max_count)
2128{
2129	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2130
2131	/*
2132	 * Justification of value bad_page_cnt_threshold in ras structure
2133	 *
2134	 * Generally, -1 <= amdgpu_bad_page_threshold <= max record length
2135	 * in eeprom, and introduce two scenarios accordingly.
2136	 *
2137	 * Bad page retirement enablement:
2138	 *    - If amdgpu_bad_page_threshold = -1,
2139	 *      bad_page_cnt_threshold = typical value by formula.
2140	 *
2141	 *    - When the value from user is 0 < amdgpu_bad_page_threshold <
2142	 *      max record length in eeprom, use it directly.
2143	 *
2144	 * Bad page retirement disablement:
2145	 *    - If amdgpu_bad_page_threshold = 0, bad page retirement
2146	 *      functionality is disabled, and bad_page_cnt_threshold will
2147	 *      take no effect.
2148	 */
2149
2150	if (amdgpu_bad_page_threshold < 0) {
2151		u64 val = adev->gmc.mc_vram_size;
2152
2153		do_div(val, RAS_BAD_PAGE_COVER);
2154		con->bad_page_cnt_threshold = min(lower_32_bits(val),
2155						  max_count);
2156	} else {
2157		con->bad_page_cnt_threshold = min_t(int, max_count,
2158						    amdgpu_bad_page_threshold);
2159	}
2160}
2161
2162int amdgpu_ras_recovery_init(struct amdgpu_device *adev)
2163{
2164	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2165	struct ras_err_handler_data **data;
2166	u32  max_eeprom_records_count = 0;
2167	bool exc_err_limit = false;
2168	int ret;
2169
2170	if (!con || amdgpu_sriov_vf(adev))
2171		return 0;
2172
2173	/* Allow access to RAS EEPROM via debugfs, when the ASIC
2174	 * supports RAS and debugfs is enabled, but when
2175	 * adev->ras_enabled is unset, i.e. when "ras_enable"
2176	 * module parameter is set to 0.
2177	 */
2178	con->adev = adev;
2179
2180	if (!adev->ras_enabled)
2181		return 0;
2182
2183	data = &con->eh_data;
2184	*data = kmalloc(sizeof(**data), GFP_KERNEL | __GFP_ZERO);
2185	if (!*data) {
2186		ret = -ENOMEM;
2187		goto out;
2188	}
2189
2190	mutex_init(&con->recovery_lock);
2191	INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
2192	atomic_set(&con->in_recovery, 0);
2193	con->eeprom_control.bad_channel_bitmap = 0;
2194
2195	max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count();
2196	amdgpu_ras_validate_threshold(adev, max_eeprom_records_count);
2197
2198	/* Todo: During test the SMU might fail to read the eeprom through I2C
2199	 * when the GPU is pending on XGMI reset during probe time
2200	 * (Mostly after second bus reset), skip it now
2201	 */
2202	if (adev->gmc.xgmi.pending_reset)
2203		return 0;
2204	ret = amdgpu_ras_eeprom_init(&con->eeprom_control, &exc_err_limit);
2205	/*
2206	 * This calling fails when exc_err_limit is true or
2207	 * ret != 0.
2208	 */
2209	if (exc_err_limit || ret)
2210		goto free;
2211
2212	if (con->eeprom_control.ras_num_recs) {
2213		ret = amdgpu_ras_load_bad_pages(adev);
2214		if (ret)
2215			goto free;
2216
2217		amdgpu_dpm_send_hbm_bad_pages_num(adev, con->eeprom_control.ras_num_recs);
2218
2219		if (con->update_channel_flag == true) {
2220			amdgpu_dpm_send_hbm_bad_channel_flag(adev, con->eeprom_control.bad_channel_bitmap);
2221			con->update_channel_flag = false;
2222		}
2223	}
2224
2225#ifdef CONFIG_X86_MCE_AMD
2226	if ((adev->asic_type == CHIP_ALDEBARAN) &&
2227	    (adev->gmc.xgmi.connected_to_cpu))
2228		amdgpu_register_bad_pages_mca_notifier(adev);
2229#endif
2230	return 0;
2231
2232free:
2233	kfree((*data)->bps);
2234	kfree(*data);
2235	con->eh_data = NULL;
2236out:
2237	dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret);
2238
2239	/*
2240	 * Except error threshold exceeding case, other failure cases in this
2241	 * function would not fail amdgpu driver init.
2242	 */
2243	if (!exc_err_limit)
2244		ret = 0;
2245	else
2246		ret = -EINVAL;
2247
2248	return ret;
2249}
2250
2251static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
2252{
2253	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2254	struct ras_err_handler_data *data = con->eh_data;
2255
2256	/* recovery_init failed to init it, fini is useless */
2257	if (!data)
2258		return 0;
2259
2260	cancel_work_sync(&con->recovery_work);
2261
2262	mutex_lock(&con->recovery_lock);
2263	con->eh_data = NULL;
2264	kfree(data->bps);
2265	kfree(data);
2266	mutex_unlock(&con->recovery_lock);
2267
2268	return 0;
2269}
2270/* recovery end */
2271
2272static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
2273{
2274	if (amdgpu_sriov_vf(adev)) {
2275		switch (adev->ip_versions[MP0_HWIP][0]) {
2276		case IP_VERSION(13, 0, 2):
2277			return true;
2278		default:
2279			return false;
2280		}
2281	}
2282
2283	if (adev->asic_type == CHIP_IP_DISCOVERY) {
2284		switch (adev->ip_versions[MP0_HWIP][0]) {
2285		case IP_VERSION(13, 0, 0):
2286		case IP_VERSION(13, 0, 10):
2287			return true;
2288		default:
2289			return false;
2290		}
2291	}
2292
2293	return adev->asic_type == CHIP_VEGA10 ||
2294		adev->asic_type == CHIP_VEGA20 ||
2295		adev->asic_type == CHIP_ARCTURUS ||
2296		adev->asic_type == CHIP_ALDEBARAN ||
2297		adev->asic_type == CHIP_SIENNA_CICHLID;
2298}
2299
2300/*
2301 * this is workaround for vega20 workstation sku,
2302 * force enable gfx ras, ignore vbios gfx ras flag
2303 * due to GC EDC can not write
2304 */
2305static void amdgpu_ras_get_quirks(struct amdgpu_device *adev)
2306{
2307	struct atom_context *ctx = adev->mode_info.atom_context;
2308
2309	if (!ctx)
2310		return;
2311
2312	if (strnstr(ctx->vbios_version, "D16406",
2313		    sizeof(ctx->vbios_version)) ||
2314		strnstr(ctx->vbios_version, "D36002",
2315			sizeof(ctx->vbios_version)))
2316		adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX);
2317}
2318
2319/*
2320 * check hardware's ras ability which will be saved in hw_supported.
2321 * if hardware does not support ras, we can skip some ras initializtion and
2322 * forbid some ras operations from IP.
2323 * if software itself, say boot parameter, limit the ras ability. We still
2324 * need allow IP do some limited operations, like disable. In such case,
2325 * we have to initialize ras as normal. but need check if operation is
2326 * allowed or not in each function.
2327 */
2328static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
2329{
2330	adev->ras_hw_enabled = adev->ras_enabled = 0;
2331
2332	if (!adev->is_atom_fw ||
2333	    !amdgpu_ras_asic_supported(adev))
2334		return;
2335
2336	if (!adev->gmc.xgmi.connected_to_cpu) {
2337		if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
2338			dev_info(adev->dev, "MEM ECC is active.\n");
2339			adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC |
2340						   1 << AMDGPU_RAS_BLOCK__DF);
2341		} else {
2342			dev_info(adev->dev, "MEM ECC is not presented.\n");
2343		}
2344
2345		if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
2346			dev_info(adev->dev, "SRAM ECC is active.\n");
2347			if (!amdgpu_sriov_vf(adev)) {
2348				adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC |
2349							    1 << AMDGPU_RAS_BLOCK__DF);
2350
2351				if (adev->ip_versions[VCN_HWIP][0] == IP_VERSION(2, 6, 0) ||
2352				    adev->ip_versions[VCN_HWIP][0] == IP_VERSION(4, 0, 0))
2353					adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN |
2354							1 << AMDGPU_RAS_BLOCK__JPEG);
2355				else
2356					adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN |
2357							1 << AMDGPU_RAS_BLOCK__JPEG);
2358			} else {
2359				adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF |
2360								1 << AMDGPU_RAS_BLOCK__SDMA |
2361								1 << AMDGPU_RAS_BLOCK__GFX);
2362			}
2363		} else {
2364			dev_info(adev->dev, "SRAM ECC is not presented.\n");
2365		}
2366	} else {
2367		/* driver only manages a few IP blocks RAS feature
2368		 * when GPU is connected cpu through XGMI */
2369		adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX |
2370					   1 << AMDGPU_RAS_BLOCK__SDMA |
2371					   1 << AMDGPU_RAS_BLOCK__MMHUB);
2372	}
2373
2374	amdgpu_ras_get_quirks(adev);
2375
2376	/* hw_supported needs to be aligned with RAS block mask. */
2377	adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
2378
2379	adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
2380		adev->ras_hw_enabled & amdgpu_ras_mask;
2381}
2382
2383static void amdgpu_ras_counte_dw(struct work_struct *work)
2384{
2385	struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
2386					      ras_counte_delay_work.work);
2387	struct amdgpu_device *adev = con->adev;
2388	struct drm_device *dev = adev_to_drm(adev);
2389	unsigned long ce_count, ue_count;
2390	int res;
2391
2392	res = pm_runtime_get_sync(dev->dev);
2393	if (res < 0)
2394		goto Out;
2395
2396	/* Cache new values.
2397	 */
2398	if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count) == 0) {
2399		atomic_set(&con->ras_ce_count, ce_count);
2400		atomic_set(&con->ras_ue_count, ue_count);
2401	}
2402
2403	pm_runtime_mark_last_busy(dev->dev);
2404Out:
2405	pm_runtime_put_autosuspend(dev->dev);
2406}
2407
2408int amdgpu_ras_init(struct amdgpu_device *adev)
2409{
2410	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2411	int r;
2412	bool df_poison, umc_poison;
2413
2414	if (con)
2415		return 0;
2416
2417	con = kmalloc(sizeof(struct amdgpu_ras) +
2418			sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT +
2419			sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT,
2420			GFP_KERNEL|__GFP_ZERO);
2421	if (!con)
2422		return -ENOMEM;
2423
2424	con->adev = adev;
2425	INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw);
2426	atomic_set(&con->ras_ce_count, 0);
2427	atomic_set(&con->ras_ue_count, 0);
2428
2429	con->objs = (struct ras_manager *)(con + 1);
2430
2431	amdgpu_ras_set_context(adev, con);
2432
2433	amdgpu_ras_check_supported(adev);
2434
2435	if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) {
2436		/* set gfx block ras context feature for VEGA20 Gaming
2437		 * send ras disable cmd to ras ta during ras late init.
2438		 */
2439		if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) {
2440			con->features |= BIT(AMDGPU_RAS_BLOCK__GFX);
2441
2442			return 0;
2443		}
2444
2445		r = 0;
2446		goto release_con;
2447	}
2448
2449	con->update_channel_flag = false;
2450	con->features = 0;
2451	INIT_LIST_HEAD(&con->head);
2452	/* Might need get this flag from vbios. */
2453	con->flags = RAS_DEFAULT_FLAGS;
2454
2455	/* initialize nbio ras function ahead of any other
2456	 * ras functions so hardware fatal error interrupt
2457	 * can be enabled as early as possible */
2458	switch (adev->asic_type) {
2459	case CHIP_VEGA20:
2460	case CHIP_ARCTURUS:
2461	case CHIP_ALDEBARAN:
2462		if (!adev->gmc.xgmi.connected_to_cpu) {
2463			adev->nbio.ras = &nbio_v7_4_ras;
2464			amdgpu_ras_register_ras_block(adev, &adev->nbio.ras->ras_block);
2465			adev->nbio.ras_if = &adev->nbio.ras->ras_block.ras_comm;
2466		}
2467		break;
2468	default:
2469		/* nbio ras is not available */
2470		break;
2471	}
2472
2473	if (adev->nbio.ras &&
2474	    adev->nbio.ras->init_ras_controller_interrupt) {
2475		r = adev->nbio.ras->init_ras_controller_interrupt(adev);
2476		if (r)
2477			goto release_con;
2478	}
2479
2480	if (adev->nbio.ras &&
2481	    adev->nbio.ras->init_ras_err_event_athub_interrupt) {
2482		r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev);
2483		if (r)
2484			goto release_con;
2485	}
2486
2487	/* Init poison supported flag, the default value is false */
2488	if (adev->gmc.xgmi.connected_to_cpu) {
2489		/* enabled by default when GPU is connected to CPU */
2490		con->poison_supported = true;
2491	}
2492	else if (adev->df.funcs &&
2493	    adev->df.funcs->query_ras_poison_mode &&
2494	    adev->umc.ras &&
2495	    adev->umc.ras->query_ras_poison_mode) {
2496		df_poison =
2497			adev->df.funcs->query_ras_poison_mode(adev);
2498		umc_poison =
2499			adev->umc.ras->query_ras_poison_mode(adev);
2500		/* Only poison is set in both DF and UMC, we can support it */
2501		if (df_poison && umc_poison)
2502			con->poison_supported = true;
2503		else if (df_poison != umc_poison)
2504			dev_warn(adev->dev, "Poison setting is inconsistent in DF/UMC(%d:%d)!\n",
2505					df_poison, umc_poison);
2506	}
2507
2508	if (amdgpu_ras_fs_init(adev)) {
2509		r = -EINVAL;
2510		goto release_con;
2511	}
2512
2513	dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
2514		 "hardware ability[%x] ras_mask[%x]\n",
2515		 adev->ras_hw_enabled, adev->ras_enabled);
2516
2517	return 0;
2518release_con:
2519	amdgpu_ras_set_context(adev, NULL);
2520	kfree(con);
2521
2522	return r;
2523}
2524
2525int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev)
2526{
2527	if (adev->gmc.xgmi.connected_to_cpu)
2528		return 1;
2529	return 0;
2530}
2531
2532static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev,
2533					struct ras_common_if *ras_block)
2534{
2535	struct ras_query_if info = {
2536		.head = *ras_block,
2537	};
2538
2539	if (!amdgpu_persistent_edc_harvesting_supported(adev))
2540		return 0;
2541
2542	if (amdgpu_ras_query_error_status(adev, &info) != 0)
2543		DRM_WARN("RAS init harvest failure");
2544
2545	if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0)
2546		DRM_WARN("RAS init harvest reset failure");
2547
2548	return 0;
2549}
2550
2551bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev)
2552{
2553       struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2554
2555       if (!con)
2556               return false;
2557
2558       return con->poison_supported;
2559}
2560
2561/* helper function to handle common stuff in ip late init phase */
2562int amdgpu_ras_block_late_init(struct amdgpu_device *adev,
2563			 struct ras_common_if *ras_block)
2564{
2565	struct amdgpu_ras_block_object *ras_obj = NULL;
2566	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2567	unsigned long ue_count, ce_count;
2568	int r;
2569
2570	/* disable RAS feature per IP block if it is not supported */
2571	if (!amdgpu_ras_is_supported(adev, ras_block->block)) {
2572		amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
2573		return 0;
2574	}
2575
2576	r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1);
2577	if (r) {
2578		if (adev->in_suspend || amdgpu_in_reset(adev)) {
2579			/* in resume phase, if fail to enable ras,
2580			 * clean up all ras fs nodes, and disable ras */
2581			goto cleanup;
2582		} else
2583			return r;
2584	}
2585
2586	/* check for errors on warm reset edc persisant supported ASIC */
2587	amdgpu_persistent_edc_harvesting(adev, ras_block);
2588
2589	/* in resume phase, no need to create ras fs node */
2590	if (adev->in_suspend || amdgpu_in_reset(adev))
2591		return 0;
2592
2593	ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
2594	if (ras_obj->ras_cb || (ras_obj->hw_ops &&
2595	    (ras_obj->hw_ops->query_poison_status ||
2596	    ras_obj->hw_ops->handle_poison_consumption))) {
2597		r = amdgpu_ras_interrupt_add_handler(adev, ras_block);
2598		if (r)
2599			goto cleanup;
2600	}
2601
2602	r = amdgpu_ras_sysfs_create(adev, ras_block);
2603	if (r)
2604		goto interrupt;
2605
2606	/* Those are the cached values at init.
2607	 */
2608	if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count) == 0) {
2609		atomic_set(&con->ras_ce_count, ce_count);
2610		atomic_set(&con->ras_ue_count, ue_count);
2611	}
2612
2613	return 0;
2614
2615interrupt:
2616	if (ras_obj->ras_cb)
2617		amdgpu_ras_interrupt_remove_handler(adev, ras_block);
2618cleanup:
2619	amdgpu_ras_feature_enable(adev, ras_block, 0);
2620	return r;
2621}
2622
2623static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev,
2624			 struct ras_common_if *ras_block)
2625{
2626	return amdgpu_ras_block_late_init(adev, ras_block);
2627}
2628
2629/* helper function to remove ras fs node and interrupt handler */
2630void amdgpu_ras_block_late_fini(struct amdgpu_device *adev,
2631			  struct ras_common_if *ras_block)
2632{
2633	struct amdgpu_ras_block_object *ras_obj;
2634	if (!ras_block)
2635		return;
2636
2637	amdgpu_ras_sysfs_remove(adev, ras_block);
2638
2639	ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
2640	if (ras_obj->ras_cb)
2641		amdgpu_ras_interrupt_remove_handler(adev, ras_block);
2642}
2643
2644static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev,
2645			  struct ras_common_if *ras_block)
2646{
2647	return amdgpu_ras_block_late_fini(adev, ras_block);
2648}
2649
2650/* do some init work after IP late init as dependence.
2651 * and it runs in resume/gpu reset/booting up cases.
2652 */
2653void amdgpu_ras_resume(struct amdgpu_device *adev)
2654{
2655	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2656	struct ras_manager *obj, *tmp;
2657
2658	if (!adev->ras_enabled || !con) {
2659		/* clean ras context for VEGA20 Gaming after send ras disable cmd */
2660		amdgpu_release_ras_context(adev);
2661
2662		return;
2663	}
2664
2665	if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
2666		/* Set up all other IPs which are not implemented. There is a
2667		 * tricky thing that IP's actual ras error type should be
2668		 * MULTI_UNCORRECTABLE, but as driver does not handle it, so
2669		 * ERROR_NONE make sense anyway.
2670		 */
2671		amdgpu_ras_enable_all_features(adev, 1);
2672
2673		/* We enable ras on all hw_supported block, but as boot
2674		 * parameter might disable some of them and one or more IP has
2675		 * not implemented yet. So we disable them on behalf.
2676		 */
2677		list_for_each_entry_safe(obj, tmp, &con->head, node) {
2678			if (!amdgpu_ras_is_supported(adev, obj->head.block)) {
2679				amdgpu_ras_feature_enable(adev, &obj->head, 0);
2680				/* there should be no any reference. */
2681				WARN_ON(alive_obj(obj));
2682			}
2683		}
2684	}
2685}
2686
2687void amdgpu_ras_suspend(struct amdgpu_device *adev)
2688{
2689	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2690
2691	if (!adev->ras_enabled || !con)
2692		return;
2693
2694	amdgpu_ras_disable_all_features(adev, 0);
2695	/* Make sure all ras objects are disabled. */
2696	if (con->features)
2697		amdgpu_ras_disable_all_features(adev, 1);
2698}
2699
2700int amdgpu_ras_late_init(struct amdgpu_device *adev)
2701{
2702	struct amdgpu_ras_block_list *node, *tmp;
2703	struct amdgpu_ras_block_object *obj;
2704	int r;
2705
2706	/* Guest side doesn't need init ras feature */
2707	if (amdgpu_sriov_vf(adev))
2708		return 0;
2709
2710	list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
2711		if (!node->ras_obj) {
2712			dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
2713			continue;
2714		}
2715
2716		obj = node->ras_obj;
2717		if (obj->ras_late_init) {
2718			r = obj->ras_late_init(adev, &obj->ras_comm);
2719			if (r) {
2720				dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n",
2721					obj->ras_comm.name, r);
2722				return r;
2723			}
2724		} else
2725			amdgpu_ras_block_late_init_default(adev, &obj->ras_comm);
2726	}
2727
2728	return 0;
2729}
2730
2731/* do some fini work before IP fini as dependence */
2732int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
2733{
2734	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2735
2736	if (!adev->ras_enabled || !con)
2737		return 0;
2738
2739
2740	/* Need disable ras on all IPs here before ip [hw/sw]fini */
2741	if (con->features)
2742		amdgpu_ras_disable_all_features(adev, 0);
2743	amdgpu_ras_recovery_fini(adev);
2744	return 0;
2745}
2746
2747int amdgpu_ras_fini(struct amdgpu_device *adev)
2748{
2749	struct amdgpu_ras_block_list *ras_node, *tmp;
2750	struct amdgpu_ras_block_object *obj = NULL;
2751	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2752
2753	if (!adev->ras_enabled || !con)
2754		return 0;
2755
2756	list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) {
2757		if (ras_node->ras_obj) {
2758			obj = ras_node->ras_obj;
2759			if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) &&
2760			    obj->ras_fini)
2761				obj->ras_fini(adev, &obj->ras_comm);
2762			else
2763				amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm);
2764		}
2765
2766		/* Clear ras blocks from ras_list and free ras block list node */
2767		list_del(&ras_node->node);
2768		kfree(ras_node);
2769	}
2770
2771	amdgpu_ras_fs_fini(adev);
2772	amdgpu_ras_interrupt_remove_all(adev);
2773
2774	WARN(con->features, "Feature mask is not cleared");
2775
2776	if (con->features)
2777		amdgpu_ras_disable_all_features(adev, 1);
2778
2779	cancel_delayed_work_sync(&con->ras_counte_delay_work);
2780
2781	amdgpu_ras_set_context(adev, NULL);
2782	kfree(con);
2783
2784	return 0;
2785}
2786
2787void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
2788{
2789	amdgpu_ras_check_supported(adev);
2790	if (!adev->ras_hw_enabled)
2791		return;
2792
2793	if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
2794		dev_info(adev->dev, "uncorrectable hardware error"
2795			"(ERREVENT_ATHUB_INTERRUPT) detected!\n");
2796
2797		amdgpu_ras_reset_gpu(adev);
2798	}
2799}
2800
2801bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev)
2802{
2803	if (adev->asic_type == CHIP_VEGA20 &&
2804	    adev->pm.fw_version <= 0x283400) {
2805		return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) &&
2806				amdgpu_ras_intr_triggered();
2807	}
2808
2809	return false;
2810}
2811
2812void amdgpu_release_ras_context(struct amdgpu_device *adev)
2813{
2814	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2815
2816	if (!con)
2817		return;
2818
2819	if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
2820		con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);
2821		amdgpu_ras_set_context(adev, NULL);
2822		kfree(con);
2823	}
2824}
2825
2826#ifdef CONFIG_X86_MCE_AMD
2827static struct amdgpu_device *find_adev(uint32_t node_id)
2828{
2829	int i;
2830	struct amdgpu_device *adev = NULL;
2831
2832	for (i = 0; i < mce_adev_list.num_gpu; i++) {
2833		adev = mce_adev_list.devs[i];
2834
2835		if (adev && adev->gmc.xgmi.connected_to_cpu &&
2836		    adev->gmc.xgmi.physical_node_id == node_id)
2837			break;
2838		adev = NULL;
2839	}
2840
2841	return adev;
2842}
2843
2844#define GET_MCA_IPID_GPUID(m)	(((m) >> 44) & 0xF)
2845#define GET_UMC_INST(m)		(((m) >> 21) & 0x7)
2846#define GET_CHAN_INDEX(m)	((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4))
2847#define GPU_ID_OFFSET		8
2848
2849static int amdgpu_bad_page_notifier(struct notifier_block *nb,
2850				    unsigned long val, void *data)
2851{
2852	struct mce *m = (struct mce *)data;
2853	struct amdgpu_device *adev = NULL;
2854	uint32_t gpu_id = 0;
2855	uint32_t umc_inst = 0, ch_inst = 0;
2856
2857	/*
2858	 * If the error was generated in UMC_V2, which belongs to GPU UMCs,
2859	 * and error occurred in DramECC (Extended error code = 0) then only
2860	 * process the error, else bail out.
2861	 */
2862	if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) &&
2863		    (XEC(m->status, 0x3f) == 0x0)))
2864		return NOTIFY_DONE;
2865
2866	/*
2867	 * If it is correctable error, return.
2868	 */
2869	if (mce_is_correctable(m))
2870		return NOTIFY_OK;
2871
2872	/*
2873	 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
2874	 */
2875	gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET;
2876
2877	adev = find_adev(gpu_id);
2878	if (!adev) {
2879		DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__,
2880								gpu_id);
2881		return NOTIFY_DONE;
2882	}
2883
2884	/*
2885	 * If it is uncorrectable error, then find out UMC instance and
2886	 * channel index.
2887	 */
2888	umc_inst = GET_UMC_INST(m->ipid);
2889	ch_inst = GET_CHAN_INDEX(m->ipid);
2890
2891	dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d",
2892			     umc_inst, ch_inst);
2893
2894	if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst))
2895		return NOTIFY_OK;
2896	else
2897		return NOTIFY_DONE;
2898}
2899
2900static struct notifier_block amdgpu_bad_page_nb = {
2901	.notifier_call  = amdgpu_bad_page_notifier,
2902	.priority       = MCE_PRIO_UC,
2903};
2904
2905static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev)
2906{
2907	/*
2908	 * Add the adev to the mce_adev_list.
2909	 * During mode2 reset, amdgpu device is temporarily
2910	 * removed from the mgpu_info list which can cause
2911	 * page retirement to fail.
2912	 * Use this list instead of mgpu_info to find the amdgpu
2913	 * device on which the UMC error was reported.
2914	 */
2915	mce_adev_list.devs[mce_adev_list.num_gpu++] = adev;
2916
2917	/*
2918	 * Register the x86 notifier only once
2919	 * with MCE subsystem.
2920	 */
2921	if (notifier_registered == false) {
2922		mce_register_decode_chain(&amdgpu_bad_page_nb);
2923		notifier_registered = true;
2924	}
2925}
2926#endif
2927
2928struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev)
2929{
2930	if (!adev)
2931		return NULL;
2932
2933	return adev->psp.ras_context.ras;
2934}
2935
2936int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con)
2937{
2938	if (!adev)
2939		return -EINVAL;
2940
2941	adev->psp.ras_context.ras = ras_con;
2942	return 0;
2943}
2944
2945/* check if ras is supported on block, say, sdma, gfx */
2946int amdgpu_ras_is_supported(struct amdgpu_device *adev,
2947		unsigned int block)
2948{
2949	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2950
2951	if (block >= AMDGPU_RAS_BLOCK_COUNT)
2952		return 0;
2953	return ras && (adev->ras_enabled & (1 << block));
2954}
2955
2956int amdgpu_ras_reset_gpu(struct amdgpu_device *adev)
2957{
2958	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2959
2960	if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0)
2961		amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
2962	return 0;
2963}
2964
2965
2966/* Register each ip ras block into amdgpu ras */
2967int amdgpu_ras_register_ras_block(struct amdgpu_device *adev,
2968		struct amdgpu_ras_block_object *ras_block_obj)
2969{
2970	struct amdgpu_ras_block_list *ras_node;
2971	if (!adev || !ras_block_obj)
2972		return -EINVAL;
2973
2974	if (!amdgpu_ras_asic_supported(adev))
2975		return 0;
2976
2977	ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL);
2978	if (!ras_node)
2979		return -ENOMEM;
2980
2981	INIT_LIST_HEAD(&ras_node->node);
2982	ras_node->ras_obj = ras_block_obj;
2983	list_add_tail(&ras_node->node, &adev->ras_list);
2984
2985	return 0;
2986}