1/*
   2 * Copyright 2018 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 *
  23 */
  24#include <linux/debugfs.h>
  25#include <linux/list.h>
  26#include <linux/module.h>
  27#include <linux/uaccess.h>
  28#include <linux/reboot.h>
  29#include <linux/syscalls.h>
 
 
  30
  31#include "amdgpu.h"
  32#include "amdgpu_ras.h"
  33#include "amdgpu_atomfirmware.h"
  34#include "amdgpu_xgmi.h"
  35#include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
 
 
 
 
 
 
 
 
 
 
 
 
  36
  37const char *ras_error_string[] = {
  38	"none",
  39	"parity",
  40	"single_correctable",
  41	"multi_uncorrectable",
  42	"poison",
  43};
  44
  45const char *ras_block_string[] = {
  46	"umc",
  47	"sdma",
  48	"gfx",
  49	"mmhub",
  50	"athub",
  51	"pcie_bif",
  52	"hdp",
  53	"xgmi_wafl",
  54	"df",
  55	"smn",
  56	"sem",
  57	"mp0",
  58	"mp1",
  59	"fuse",
 
 
 
 
 
  60};
  61
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  62#define ras_err_str(i) (ras_error_string[ffs(i)])
  63#define ras_block_str(i) (ras_block_string[i])
  64
  65#define AMDGPU_RAS_FLAG_INIT_BY_VBIOS		1
  66#define AMDGPU_RAS_FLAG_INIT_NEED_RESET		2
  67#define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
  68
  69/* inject address is 52 bits */
  70#define	RAS_UMC_INJECT_ADDR_LIMIT	(0x1ULL << 52)
  71
 
 
 
 
 
 
 
 
 
  72enum amdgpu_ras_retire_page_reservation {
  73	AMDGPU_RAS_RETIRE_PAGE_RESERVED,
  74	AMDGPU_RAS_RETIRE_PAGE_PENDING,
  75	AMDGPU_RAS_RETIRE_PAGE_FAULT,
  76};
  77
  78atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0);
  79
 
 
  80static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
  81				uint64_t addr);
 
 
 
 
 
 
 
 
  82
  83void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready)
  84{
  85	if (adev && amdgpu_ras_get_context(adev))
  86		amdgpu_ras_get_context(adev)->error_query_ready = ready;
  87}
  88
  89static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev)
  90{
  91	if (adev && amdgpu_ras_get_context(adev))
  92		return amdgpu_ras_get_context(adev)->error_query_ready;
  93
  94	return false;
  95}
  96
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  97static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
  98					size_t size, loff_t *pos)
  99{
 100	struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private;
 101	struct ras_query_if info = {
 102		.head = obj->head,
 103	};
 104	ssize_t s;
 105	char val[128];
 106
 107	if (amdgpu_ras_error_query(obj->adev, &info))
 108		return -EINVAL;
 109
 
 
 
 
 
 
 
 110	s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n",
 111			"ue", info.ue_count,
 112			"ce", info.ce_count);
 113	if (*pos >= s)
 114		return 0;
 115
 116	s -= *pos;
 117	s = min_t(u64, s, size);
 118
 119
 120	if (copy_to_user(buf, &val[*pos], s))
 121		return -EINVAL;
 122
 123	*pos += s;
 124
 125	return s;
 126}
 127
 128static const struct file_operations amdgpu_ras_debugfs_ops = {
 129	.owner = THIS_MODULE,
 130	.read = amdgpu_ras_debugfs_read,
 131	.write = NULL,
 132	.llseek = default_llseek
 133};
 134
 135static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id)
 136{
 137	int i;
 138
 139	for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) {
 140		*block_id = i;
 141		if (strcmp(name, ras_block_str(i)) == 0)
 142			return 0;
 143	}
 144	return -EINVAL;
 145}
 146
 147static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
 148		const char __user *buf, size_t size,
 149		loff_t *pos, struct ras_debug_if *data)
 150{
 151	ssize_t s = min_t(u64, 64, size);
 152	char str[65];
 153	char block_name[33];
 154	char err[9] = "ue";
 155	int op = -1;
 156	int block_id;
 157	uint32_t sub_block;
 158	u64 address, value;
 
 
 159
 160	if (*pos)
 161		return -EINVAL;
 162	*pos = size;
 163
 164	memset(str, 0, sizeof(str));
 165	memset(data, 0, sizeof(*data));
 166
 167	if (copy_from_user(str, buf, s))
 168		return -EINVAL;
 169
 170	if (sscanf(str, "disable %32s", block_name) == 1)
 171		op = 0;
 172	else if (sscanf(str, "enable %32s %8s", block_name, err) == 2)
 173		op = 1;
 174	else if (sscanf(str, "inject %32s %8s", block_name, err) == 2)
 175		op = 2;
 
 
 176	else if (str[0] && str[1] && str[2] && str[3])
 177		/* ascii string, but commands are not matched. */
 178		return -EINVAL;
 179
 180	if (op != -1) {
 
 
 
 
 
 
 
 
 
 
 
 181		if (amdgpu_ras_find_block_id_by_name(block_name, &block_id))
 182			return -EINVAL;
 183
 184		data->head.block = block_id;
 185		/* only ue and ce errors are supported */
 186		if (!memcmp("ue", err, 2))
 187			data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
 188		else if (!memcmp("ce", err, 2))
 189			data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE;
 
 
 190		else
 191			return -EINVAL;
 192
 193		data->op = op;
 194
 195		if (op == 2) {
 196			if (sscanf(str, "%*s %*s %*s %u %llu %llu",
 197						&sub_block, &address, &value) != 3)
 198				if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx",
 199							&sub_block, &address, &value) != 3)
 200					return -EINVAL;
 
 
 
 
 201			data->head.sub_block_index = sub_block;
 202			data->inject.address = address;
 203			data->inject.value = value;
 
 204		}
 205	} else {
 206		if (size < sizeof(*data))
 207			return -EINVAL;
 208
 209		if (copy_from_user(data, buf, sizeof(*data)))
 210			return -EINVAL;
 211	}
 212
 213	return 0;
 214}
 215
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 216/**
 217 * DOC: AMDGPU RAS debugfs control interface
 218 *
 219 * It accepts struct ras_debug_if who has two members.
 220 *
 221 * First member: ras_debug_if::head or ras_debug_if::inject.
 222 *
 223 * head is used to indicate which IP block will be under control.
 224 *
 225 * head has four members, they are block, type, sub_block_index, name.
 226 * block: which IP will be under control.
 227 * type: what kind of error will be enabled/disabled/injected.
 228 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA.
 229 * name: the name of IP.
 230 *
 231 * inject has two more members than head, they are address, value.
 232 * As their names indicate, inject operation will write the
 233 * value to the address.
 234 *
 235 * The second member: struct ras_debug_if::op.
 236 * It has three kinds of operations.
 237 *
 238 * - 0: disable RAS on the block. Take ::head as its data.
 239 * - 1: enable RAS on the block. Take ::head as its data.
 240 * - 2: inject errors on the block. Take ::inject as its data.
 241 *
 242 * How to use the interface?
 243 *
 244 * Programs
 245 *
 246 * Copy the struct ras_debug_if in your codes and initialize it.
 247 * Write the struct to the control node.
 248 *
 249 * Shells
 250 *
 251 * .. code-block:: bash
 252 *
 253 *	echo op block [error [sub_block address value]] > .../ras/ras_ctrl
 
 
 
 
 254 *
 255 * Parameters:
 
 
 256 *
 257 * op: disable, enable, inject
 258 *	disable: only block is needed
 259 *	enable: block and error are needed
 260 *	inject: error, address, value are needed
 261 * block: umc, sdma, gfx, .........
 262 *	see ras_block_string[] for details
 263 * error: ue, ce
 264 *	ue: multi_uncorrectable
 265 *	ce: single_correctable
 266 * sub_block:
 267 *	sub block index, pass 0 if there is no sub block
 268 *
 269 * here are some examples for bash commands:
 
 
 
 
 
 
 
 
 
 270 *
 271 * .. code-block:: bash
 272 *
 273 *	echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
 274 *	echo inject umc ce 0 0 0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
 275 *	echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl
 276 *
 277 * How to check the result?
 278 *
 279 * For disable/enable, please check ras features at
 280 * /sys/class/drm/card[0/1/2...]/device/ras/features
 281 *
 282 * For inject, please check corresponding err count at
 283 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
 284 *
 285 * .. note::
 286 *	Operations are only allowed on blocks which are supported.
 287 *	Please check ras mask at /sys/module/amdgpu/parameters/ras_mask
 288 *	to see which blocks support RAS on a particular asic.
 289 *
 290 */
 291static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, const char __user *buf,
 292		size_t size, loff_t *pos)
 
 293{
 294	struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
 295	struct ras_debug_if data;
 296	int ret = 0;
 297
 298	if (!amdgpu_ras_get_error_query_ready(adev)) {
 299		dev_warn(adev->dev, "RAS WARN: error injection "
 300				"currently inaccessible\n");
 301		return size;
 302	}
 303
 304	ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data);
 305	if (ret)
 306		return -EINVAL;
 
 
 
 
 
 
 
 
 307
 308	if (!amdgpu_ras_is_supported(adev, data.head.block))
 309		return -EINVAL;
 310
 311	switch (data.op) {
 312	case 0:
 313		ret = amdgpu_ras_feature_enable(adev, &data.head, 0);
 314		break;
 315	case 1:
 316		ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
 317		break;
 318	case 2:
 319		if ((data.inject.address >= adev->gmc.mc_vram_size) ||
 
 320		    (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
 321			dev_warn(adev->dev, "RAS WARN: input address "
 322					"0x%llx is invalid.",
 323					data.inject.address);
 324			ret = -EINVAL;
 325			break;
 326		}
 327
 328		/* umc ce/ue error injection for a bad page is not allowed */
 329		if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) &&
 330		    amdgpu_ras_check_bad_page(adev, data.inject.address)) {
 331			dev_warn(adev->dev, "RAS WARN: 0x%llx has been marked "
 332					"as bad before error injection!\n",
 333					data.inject.address);
 334			break;
 335		}
 336
 
 
 337		/* data.inject.address is offset instead of absolute gpu address */
 338		ret = amdgpu_ras_error_inject(adev, &data.inject);
 339		break;
 340	default:
 341		ret = -EINVAL;
 342		break;
 343	}
 344
 345	if (ret)
 346		return -EINVAL;
 347
 348	return size;
 349}
 350
 351/**
 352 * DOC: AMDGPU RAS debugfs EEPROM table reset interface
 353 *
 354 * Some boards contain an EEPROM which is used to persistently store a list of
 355 * bad pages which experiences ECC errors in vram.  This interface provides
 356 * a way to reset the EEPROM, e.g., after testing error injection.
 357 *
 358 * Usage:
 359 *
 360 * .. code-block:: bash
 361 *
 362 *	echo 1 > ../ras/ras_eeprom_reset
 363 *
 364 * will reset EEPROM table to 0 entries.
 365 *
 366 */
 367static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f, const char __user *buf,
 368		size_t size, loff_t *pos)
 
 369{
 370	struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
 
 371	int ret;
 372
 373	ret = amdgpu_ras_eeprom_reset_table(&adev->psp.ras.ras->eeprom_control);
 
 374
 375	return ret == 1 ? size : -EIO;
 
 
 
 
 
 
 
 376}
 377
 378static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = {
 379	.owner = THIS_MODULE,
 380	.read = NULL,
 381	.write = amdgpu_ras_debugfs_ctrl_write,
 382	.llseek = default_llseek
 383};
 384
 385static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = {
 386	.owner = THIS_MODULE,
 387	.read = NULL,
 388	.write = amdgpu_ras_debugfs_eeprom_write,
 389	.llseek = default_llseek
 390};
 391
 392/**
 393 * DOC: AMDGPU RAS sysfs Error Count Interface
 394 *
 395 * It allows the user to read the error count for each IP block on the gpu through
 396 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
 397 *
 398 * It outputs the multiple lines which report the uncorrected (ue) and corrected
 399 * (ce) error counts.
 400 *
 401 * The format of one line is below,
 402 *
 403 * [ce|ue]: count
 404 *
 405 * Example:
 406 *
 407 * .. code-block:: bash
 408 *
 409 *	ue: 0
 410 *	ce: 1
 411 *
 412 */
 413static ssize_t amdgpu_ras_sysfs_read(struct device *dev,
 414		struct device_attribute *attr, char *buf)
 415{
 416	struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr);
 417	struct ras_query_if info = {
 418		.head = obj->head,
 419	};
 420
 421	if (!amdgpu_ras_get_error_query_ready(obj->adev))
 422		return snprintf(buf, PAGE_SIZE,
 423				"Query currently inaccessible\n");
 424
 425	if (amdgpu_ras_error_query(obj->adev, &info))
 426		return -EINVAL;
 427
 428	return snprintf(buf, PAGE_SIZE, "%s: %lu\n%s: %lu\n",
 429			"ue", info.ue_count,
 430			"ce", info.ce_count);
 
 
 
 
 
 
 
 
 
 431}
 432
 433/* obj begin */
 434
 435#define get_obj(obj) do { (obj)->use++; } while (0)
 436#define alive_obj(obj) ((obj)->use)
 437
 438static inline void put_obj(struct ras_manager *obj)
 439{
 440	if (obj && --obj->use == 0)
 441		list_del(&obj->node);
 442	if (obj && obj->use < 0) {
 443		 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", obj->head.name);
 444	}
 
 
 
 445}
 446
 447/* make one obj and return it. */
 448static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev,
 449		struct ras_common_if *head)
 450{
 451	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 452	struct ras_manager *obj;
 453
 454	if (!con)
 455		return NULL;
 456
 457	if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
 458		return NULL;
 459
 460	obj = &con->objs[head->block];
 
 
 
 
 
 
 
 461	/* already exist. return obj? */
 462	if (alive_obj(obj))
 463		return NULL;
 464
 
 
 
 465	obj->head = *head;
 466	obj->adev = adev;
 467	list_add(&obj->node, &con->head);
 468	get_obj(obj);
 469
 470	return obj;
 471}
 472
 473/* return an obj equal to head, or the first when head is NULL */
 474struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev,
 475		struct ras_common_if *head)
 476{
 477	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 478	struct ras_manager *obj;
 479	int i;
 480
 481	if (!con)
 482		return NULL;
 483
 484	if (head) {
 485		if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
 486			return NULL;
 487
 488		obj = &con->objs[head->block];
 
 
 489
 490		if (alive_obj(obj)) {
 491			WARN_ON(head->block != obj->head.block);
 
 
 
 492			return obj;
 493		}
 494	} else {
 495		for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) {
 496			obj = &con->objs[i];
 497			if (alive_obj(obj)) {
 498				WARN_ON(i != obj->head.block);
 499				return obj;
 500			}
 501		}
 502	}
 503
 504	return NULL;
 505}
 506/* obj end */
 507
 508static void amdgpu_ras_parse_status_code(struct amdgpu_device *adev,
 509				  const char* 		invoke_type,
 510				  const char* 		block_name,
 511				  enum ta_ras_status 	ret)
 512{
 513	switch (ret) {
 514	case TA_RAS_STATUS__SUCCESS:
 515		return;
 516	case TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE:
 517		dev_warn(adev->dev,
 518			"RAS WARN: %s %s currently unavailable\n",
 519			invoke_type,
 520			block_name);
 521		break;
 522	default:
 523		dev_err(adev->dev,
 524			"RAS ERROR: %s %s error failed ret 0x%X\n",
 525			invoke_type,
 526			block_name,
 527			ret);
 528	}
 529}
 530
 531/* feature ctl begin */
 532static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
 533		struct ras_common_if *head)
 534{
 535	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 536
 537	return con->hw_supported & BIT(head->block);
 538}
 539
 540static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
 541		struct ras_common_if *head)
 542{
 543	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 544
 545	return con->features & BIT(head->block);
 546}
 547
 548/*
 549 * if obj is not created, then create one.
 550 * set feature enable flag.
 551 */
 552static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev,
 553		struct ras_common_if *head, int enable)
 554{
 555	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 556	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
 557
 558	/* If hardware does not support ras, then do not create obj.
 559	 * But if hardware support ras, we can create the obj.
 560	 * Ras framework checks con->hw_supported to see if it need do
 561	 * corresponding initialization.
 562	 * IP checks con->support to see if it need disable ras.
 563	 */
 564	if (!amdgpu_ras_is_feature_allowed(adev, head))
 565		return 0;
 566	if (!(!!enable ^ !!amdgpu_ras_is_feature_enabled(adev, head)))
 567		return 0;
 568
 569	if (enable) {
 570		if (!obj) {
 571			obj = amdgpu_ras_create_obj(adev, head);
 572			if (!obj)
 573				return -EINVAL;
 574		} else {
 575			/* In case we create obj somewhere else */
 576			get_obj(obj);
 577		}
 578		con->features |= BIT(head->block);
 579	} else {
 580		if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
 581			con->features &= ~BIT(head->block);
 582			put_obj(obj);
 583		}
 584	}
 585
 586	return 0;
 587}
 588
 589/* wrapper of psp_ras_enable_features */
 590int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
 591		struct ras_common_if *head, bool enable)
 592{
 593	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 594	union ta_ras_cmd_input *info;
 595	int ret;
 596
 597	if (!con)
 598		return -EINVAL;
 599
 600        info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL);
 601	if (!info)
 602		return -ENOMEM;
 603
 604	if (!enable) {
 605		info->disable_features = (struct ta_ras_disable_features_input) {
 606			.block_id =  amdgpu_ras_block_to_ta(head->block),
 607			.error_type = amdgpu_ras_error_to_ta(head->type),
 608		};
 609	} else {
 610		info->enable_features = (struct ta_ras_enable_features_input) {
 611			.block_id =  amdgpu_ras_block_to_ta(head->block),
 612			.error_type = amdgpu_ras_error_to_ta(head->type),
 613		};
 614	}
 615
 616	/* Do not enable if it is not allowed. */
 617	WARN_ON(enable && !amdgpu_ras_is_feature_allowed(adev, head));
 618	/* Are we alerady in that state we are going to set? */
 619	if (!(!!enable ^ !!amdgpu_ras_is_feature_enabled(adev, head))) {
 620		ret = 0;
 621		goto out;
 622	}
 
 
 
 
 
 
 
 
 
 
 
 
 623
 624	if (!amdgpu_ras_intr_triggered()) {
 625		ret = psp_ras_enable_features(&adev->psp, info, enable);
 626		if (ret) {
 627			amdgpu_ras_parse_status_code(adev,
 628						     enable ? "enable":"disable",
 629						     ras_block_str(head->block),
 630						    (enum ta_ras_status)ret);
 631			if (ret == TA_RAS_STATUS__RESET_NEEDED)
 632				ret = -EAGAIN;
 633			else
 634				ret = -EINVAL;
 635
 636			goto out;
 637		}
 
 
 638	}
 639
 640	/* setup the obj */
 641	__amdgpu_ras_feature_enable(adev, head, enable);
 642	ret = 0;
 643out:
 644	kfree(info);
 645	return ret;
 646}
 647
 648/* Only used in device probe stage and called only once. */
 649int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
 650		struct ras_common_if *head, bool enable)
 651{
 652	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 653	int ret;
 654
 655	if (!con)
 656		return -EINVAL;
 657
 658	if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
 659		if (enable) {
 660			/* There is no harm to issue a ras TA cmd regardless of
 661			 * the currecnt ras state.
 662			 * If current state == target state, it will do nothing
 663			 * But sometimes it requests driver to reset and repost
 664			 * with error code -EAGAIN.
 665			 */
 666			ret = amdgpu_ras_feature_enable(adev, head, 1);
 667			/* With old ras TA, we might fail to enable ras.
 668			 * Log it and just setup the object.
 669			 * TODO need remove this WA in the future.
 670			 */
 671			if (ret == -EINVAL) {
 672				ret = __amdgpu_ras_feature_enable(adev, head, 1);
 673				if (!ret)
 674					dev_info(adev->dev,
 675						"RAS INFO: %s setup object\n",
 676						ras_block_str(head->block));
 677			}
 678		} else {
 679			/* setup the object then issue a ras TA disable cmd.*/
 680			ret = __amdgpu_ras_feature_enable(adev, head, 1);
 681			if (ret)
 682				return ret;
 683
 
 
 
 
 684			ret = amdgpu_ras_feature_enable(adev, head, 0);
 
 
 
 
 685		}
 686	} else
 687		ret = amdgpu_ras_feature_enable(adev, head, enable);
 688
 689	return ret;
 690}
 691
 692static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev,
 693		bool bypass)
 694{
 695	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 696	struct ras_manager *obj, *tmp;
 697
 698	list_for_each_entry_safe(obj, tmp, &con->head, node) {
 699		/* bypass psp.
 700		 * aka just release the obj and corresponding flags
 701		 */
 702		if (bypass) {
 703			if (__amdgpu_ras_feature_enable(adev, &obj->head, 0))
 704				break;
 705		} else {
 706			if (amdgpu_ras_feature_enable(adev, &obj->head, 0))
 707				break;
 708		}
 709	}
 710
 711	return con->features;
 712}
 713
 714static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev,
 715		bool bypass)
 716{
 717	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 718	int ras_block_count = AMDGPU_RAS_BLOCK_COUNT;
 719	int i;
 720	const enum amdgpu_ras_error_type default_ras_type =
 721		AMDGPU_RAS_ERROR__NONE;
 722
 723	for (i = 0; i < ras_block_count; i++) {
 724		struct ras_common_if head = {
 725			.block = i,
 726			.type = default_ras_type,
 727			.sub_block_index = 0,
 728		};
 729		strcpy(head.name, ras_block_str(i));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 730		if (bypass) {
 731			/*
 732			 * bypass psp. vbios enable ras for us.
 733			 * so just create the obj
 734			 */
 735			if (__amdgpu_ras_feature_enable(adev, &head, 1))
 736				break;
 737		} else {
 738			if (amdgpu_ras_feature_enable(adev, &head, 1))
 739				break;
 740		}
 741	}
 742
 743	return con->features;
 744}
 745/* feature ctl end */
 746
 747/* query/inject/cure begin */
 748int amdgpu_ras_error_query(struct amdgpu_device *adev,
 749		struct ras_query_if *info)
 750{
 751	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
 752	struct ras_err_data err_data = {0, 0, 0, NULL};
 753	int i;
 754
 755	if (!obj)
 756		return -EINVAL;
 757
 758	switch (info->head.block) {
 759	case AMDGPU_RAS_BLOCK__UMC:
 760		if (adev->umc.funcs->query_ras_error_count)
 761			adev->umc.funcs->query_ras_error_count(adev, &err_data);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 762		/* umc query_ras_error_address is also responsible for clearing
 763		 * error status
 764		 */
 765		if (adev->umc.funcs->query_ras_error_address)
 766			adev->umc.funcs->query_ras_error_address(adev, &err_data);
 767		break;
 768	case AMDGPU_RAS_BLOCK__SDMA:
 769		if (adev->sdma.funcs->query_ras_error_count) {
 770			for (i = 0; i < adev->sdma.num_instances; i++)
 771				adev->sdma.funcs->query_ras_error_count(adev, i,
 772									&err_data);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 773		}
 774		break;
 775	case AMDGPU_RAS_BLOCK__GFX:
 776		if (adev->gfx.funcs->query_ras_error_count)
 777			adev->gfx.funcs->query_ras_error_count(adev, &err_data);
 778		break;
 779	case AMDGPU_RAS_BLOCK__MMHUB:
 780		if (adev->mmhub.funcs->query_ras_error_count)
 781			adev->mmhub.funcs->query_ras_error_count(adev, &err_data);
 782		break;
 783	case AMDGPU_RAS_BLOCK__PCIE_BIF:
 784		if (adev->nbio.funcs->query_ras_error_count)
 785			adev->nbio.funcs->query_ras_error_count(adev, &err_data);
 786		break;
 787	case AMDGPU_RAS_BLOCK__XGMI_WAFL:
 788		amdgpu_xgmi_query_ras_error_count(adev, &err_data);
 789		break;
 790	default:
 791		break;
 792	}
 
 793
 794	obj->err_data.ue_count += err_data.ue_count;
 795	obj->err_data.ce_count += err_data.ce_count;
 
 
 796
 797	info->ue_count = obj->err_data.ue_count;
 798	info->ce_count = obj->err_data.ce_count;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 799
 800	if (err_data.ce_count) {
 801		dev_info(adev->dev, "%ld correctable hardware errors "
 802					"detected in %s block, no user "
 803					"action is needed.\n",
 804					obj->err_data.ce_count,
 805					ras_block_str(info->head.block));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 806	}
 807	if (err_data.ue_count) {
 808		dev_info(adev->dev, "%ld uncorrectable hardware errors "
 809					"detected in %s block\n",
 810					obj->err_data.ue_count,
 811					ras_block_str(info->head.block));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 812	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 813
 814	return 0;
 815}
 816
 817/* Trigger XGMI/WAFL error */
 818static int amdgpu_ras_error_inject_xgmi(struct amdgpu_device *adev,
 819				 struct ta_ras_trigger_error_input *block_info)
 820{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 821	int ret;
 822
 823	if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
 824		dev_warn(adev->dev, "Failed to disallow df cstate");
 825
 826	if (amdgpu_dpm_allow_xgmi_power_down(adev, false))
 827		dev_warn(adev->dev, "Failed to disallow XGMI power down");
 828
 829	ret = psp_ras_trigger_error(&adev->psp, block_info);
 
 
 
 
 
 
 
 
 
 
 
 830
 831	if (amdgpu_ras_intr_triggered())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 832		return ret;
 833
 834	if (amdgpu_dpm_allow_xgmi_power_down(adev, true))
 835		dev_warn(adev->dev, "Failed to allow XGMI power down");
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 836
 837	if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
 838		dev_warn(adev->dev, "Failed to allow df cstate");
 839
 840	return ret;
 841}
 842
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 843/* wrapper of psp_ras_trigger_error */
 844int amdgpu_ras_error_inject(struct amdgpu_device *adev,
 845		struct ras_inject_if *info)
 846{
 847	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
 848	struct ta_ras_trigger_error_input block_info = {
 849		.block_id =  amdgpu_ras_block_to_ta(info->head.block),
 850		.inject_error_type = amdgpu_ras_error_to_ta(info->head.type),
 851		.sub_block_index = info->head.sub_block_index,
 852		.address = info->address,
 853		.value = info->value,
 854	};
 855	int ret = 0;
 
 
 
 
 
 
 
 856
 857	if (!obj)
 858		return -EINVAL;
 859
 
 
 
 
 
 
 860	/* Calculate XGMI relative offset */
 861	if (adev->gmc.xgmi.num_physical_nodes > 1) {
 
 862		block_info.address =
 863			amdgpu_xgmi_get_relative_phy_addr(adev,
 864							  block_info.address);
 865	}
 866
 867	switch (info->head.block) {
 868	case AMDGPU_RAS_BLOCK__GFX:
 869		if (adev->gfx.funcs->ras_error_inject)
 870			ret = adev->gfx.funcs->ras_error_inject(adev, info);
 871		else
 872			ret = -EINVAL;
 873		break;
 874	case AMDGPU_RAS_BLOCK__UMC:
 875	case AMDGPU_RAS_BLOCK__MMHUB:
 876	case AMDGPU_RAS_BLOCK__PCIE_BIF:
 877		ret = psp_ras_trigger_error(&adev->psp, &block_info);
 878		break;
 879	case AMDGPU_RAS_BLOCK__XGMI_WAFL:
 880		ret = amdgpu_ras_error_inject_xgmi(adev, &block_info);
 881		break;
 882	default:
 883		dev_info(adev->dev, "%s error injection is not supported yet\n",
 884			 ras_block_str(info->head.block));
 885		ret = -EINVAL;
 886	}
 887
 888	amdgpu_ras_parse_status_code(adev,
 889				     "inject",
 890				     ras_block_str(info->head.block),
 891				     (enum ta_ras_status)ret);
 892
 893	return ret;
 894}
 895
 896int amdgpu_ras_error_cure(struct amdgpu_device *adev,
 897		struct ras_cure_if *info)
 
 
 
 
 
 
 
 
 
 
 
 
 898{
 899	/* psp fw has no cure interface for now. */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 900	return 0;
 901}
 902
 903/* get the total error counts on all IPs */
 904unsigned long amdgpu_ras_query_error_count(struct amdgpu_device *adev,
 905		bool is_ce)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 906{
 907	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 908	struct ras_manager *obj;
 909	struct ras_err_data data = {0, 0};
 
 910
 911	if (!con)
 912		return 0;
 913
 914	list_for_each_entry(obj, &con->head, node) {
 915		struct ras_query_if info = {
 916			.head = obj->head,
 917		};
 918
 919		if (amdgpu_ras_error_query(adev, &info))
 920			return 0;
 
 
 
 
 
 
 921
 922		data.ce_count += info.ce_count;
 923		data.ue_count += info.ue_count;
 
 
 
 924	}
 925
 926	return is_ce ? data.ce_count : data.ue_count;
 
 
 
 
 
 
 
 
 
 927}
 928/* query/inject/cure end */
 929
 930
 931/* sysfs begin */
 932
 933static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
 934		struct ras_badpage **bps, unsigned int *count);
 935
 936static char *amdgpu_ras_badpage_flags_str(unsigned int flags)
 937{
 938	switch (flags) {
 939	case AMDGPU_RAS_RETIRE_PAGE_RESERVED:
 940		return "R";
 941	case AMDGPU_RAS_RETIRE_PAGE_PENDING:
 942		return "P";
 943	case AMDGPU_RAS_RETIRE_PAGE_FAULT:
 944	default:
 945		return "F";
 946	};
 947}
 948
 949/**
 950 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface
 951 *
 952 * It allows user to read the bad pages of vram on the gpu through
 953 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages
 954 *
 955 * It outputs multiple lines, and each line stands for one gpu page.
 956 *
 957 * The format of one line is below,
 958 * gpu pfn : gpu page size : flags
 959 *
 960 * gpu pfn and gpu page size are printed in hex format.
 961 * flags can be one of below character,
 962 *
 963 * R: reserved, this gpu page is reserved and not able to use.
 964 *
 965 * P: pending for reserve, this gpu page is marked as bad, will be reserved
 966 * in next window of page_reserve.
 967 *
 968 * F: unable to reserve. this gpu page can't be reserved due to some reasons.
 969 *
 970 * Examples:
 971 *
 972 * .. code-block:: bash
 973 *
 974 *	0x00000001 : 0x00001000 : R
 975 *	0x00000002 : 0x00001000 : P
 976 *
 977 */
 978
 979static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f,
 980		struct kobject *kobj, struct bin_attribute *attr,
 981		char *buf, loff_t ppos, size_t count)
 982{
 983	struct amdgpu_ras *con =
 984		container_of(attr, struct amdgpu_ras, badpages_attr);
 985	struct amdgpu_device *adev = con->adev;
 986	const unsigned int element_size =
 987		sizeof("0xabcdabcd : 0x12345678 : R\n") - 1;
 988	unsigned int start = div64_ul(ppos + element_size - 1, element_size);
 989	unsigned int end = div64_ul(ppos + count - 1, element_size);
 990	ssize_t s = 0;
 991	struct ras_badpage *bps = NULL;
 992	unsigned int bps_count = 0;
 993
 994	memset(buf, 0, count);
 995
 996	if (amdgpu_ras_badpages_read(adev, &bps, &bps_count))
 997		return 0;
 998
 999	for (; start < end && start < bps_count; start++)
1000		s += scnprintf(&buf[s], element_size + 1,
1001				"0x%08x : 0x%08x : %1s\n",
1002				bps[start].bp,
1003				bps[start].size,
1004				amdgpu_ras_badpage_flags_str(bps[start].flags));
1005
1006	kfree(bps);
1007
1008	return s;
1009}
1010
1011static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
1012		struct device_attribute *attr, char *buf)
1013{
1014	struct amdgpu_ras *con =
1015		container_of(attr, struct amdgpu_ras, features_attr);
1016
1017	return scnprintf(buf, PAGE_SIZE, "feature mask: 0x%x\n", con->features);
1018}
1019
1020static int amdgpu_ras_sysfs_create_feature_node(struct amdgpu_device *adev)
 
1021{
1022	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1023	struct attribute *attrs[] = {
1024		&con->features_attr.attr,
1025		NULL
1026	};
1027	struct bin_attribute *bin_attrs[] = {
1028		&con->badpages_attr,
1029		NULL
1030	};
1031	struct attribute_group group = {
1032		.name = "ras",
1033		.attrs = attrs,
1034		.bin_attrs = bin_attrs,
1035	};
1036
1037	con->features_attr = (struct device_attribute) {
1038		.attr = {
1039			.name = "features",
1040			.mode = S_IRUGO,
1041		},
1042			.show = amdgpu_ras_sysfs_features_read,
1043	};
1044
1045	con->badpages_attr = (struct bin_attribute) {
1046		.attr = {
1047			.name = "gpu_vram_bad_pages",
1048			.mode = S_IRUGO,
1049		},
1050		.size = 0,
1051		.private = NULL,
1052		.read = amdgpu_ras_sysfs_badpages_read,
1053	};
1054
1055	sysfs_attr_init(attrs[0]);
1056	sysfs_bin_attr_init(bin_attrs[0]);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1057
1058	return sysfs_create_group(&adev->dev->kobj, &group);
 
 
 
1059}
1060
1061static int amdgpu_ras_sysfs_remove_feature_node(struct amdgpu_device *adev)
1062{
1063	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1064	struct attribute *attrs[] = {
1065		&con->features_attr.attr,
1066		NULL
1067	};
1068	struct bin_attribute *bin_attrs[] = {
1069		&con->badpages_attr,
1070		NULL
1071	};
1072	struct attribute_group group = {
1073		.name = "ras",
1074		.attrs = attrs,
1075		.bin_attrs = bin_attrs,
1076	};
1077
1078	sysfs_remove_group(&adev->dev->kobj, &group);
 
1079
1080	return 0;
1081}
1082
1083int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
1084		struct ras_fs_if *head)
1085{
1086	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
 
 
 
1087
1088	if (!obj || obj->attr_inuse)
1089		return -EINVAL;
1090
1091	get_obj(obj);
1092
1093	memcpy(obj->fs_data.sysfs_name,
1094			head->sysfs_name,
1095			sizeof(obj->fs_data.sysfs_name));
1096
1097	obj->sysfs_attr = (struct device_attribute){
1098		.attr = {
1099			.name = obj->fs_data.sysfs_name,
1100			.mode = S_IRUGO,
1101		},
1102			.show = amdgpu_ras_sysfs_read,
1103	};
1104	sysfs_attr_init(&obj->sysfs_attr.attr);
1105
1106	if (sysfs_add_file_to_group(&adev->dev->kobj,
1107				&obj->sysfs_attr.attr,
1108				"ras")) {
1109		put_obj(obj);
1110		return -EINVAL;
1111	}
1112
1113	obj->attr_inuse = 1;
1114
1115	return 0;
1116}
1117
1118int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
1119		struct ras_common_if *head)
1120{
1121	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1122
 
 
 
1123	if (!obj || !obj->attr_inuse)
1124		return -EINVAL;
1125
1126	sysfs_remove_file_from_group(&adev->dev->kobj,
 
1127				&obj->sysfs_attr.attr,
1128				"ras");
1129	obj->attr_inuse = 0;
1130	put_obj(obj);
1131
1132	return 0;
1133}
1134
1135static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
1136{
1137	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1138	struct ras_manager *obj, *tmp;
1139
1140	list_for_each_entry_safe(obj, tmp, &con->head, node) {
1141		amdgpu_ras_sysfs_remove(adev, &obj->head);
1142	}
1143
1144	amdgpu_ras_sysfs_remove_feature_node(adev);
 
 
 
1145
1146	return 0;
1147}
1148/* sysfs end */
1149
1150/**
1151 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors
1152 *
1153 * Normally when there is an uncorrectable error, the driver will reset
1154 * the GPU to recover.  However, in the event of an unrecoverable error,
1155 * the driver provides an interface to reboot the system automatically
1156 * in that event.
1157 *
1158 * The following file in debugfs provides that interface:
1159 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot
1160 *
1161 * Usage:
1162 *
1163 * .. code-block:: bash
1164 *
1165 *	echo true > .../ras/auto_reboot
1166 *
1167 */
1168/* debugfs begin */
1169static void amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
1170{
1171	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1172	struct drm_minor *minor = adev->ddev->primary;
1173
1174	con->dir = debugfs_create_dir("ras", minor->debugfs_root);
1175	debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, con->dir,
1176				adev, &amdgpu_ras_debugfs_ctrl_ops);
1177	debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, con->dir,
1178				adev, &amdgpu_ras_debugfs_eeprom_ops);
 
 
 
 
 
 
 
 
 
 
 
 
 
1179
1180	/*
1181	 * After one uncorrectable error happens, usually GPU recovery will
1182	 * be scheduled. But due to the known problem in GPU recovery failing
1183	 * to bring GPU back, below interface provides one direct way to
1184	 * user to reboot system automatically in such case within
1185	 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine
1186	 * will never be called.
1187	 */
1188	debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, con->dir,
1189				&con->reboot);
 
 
 
 
 
 
 
1190}
1191
1192void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
1193		struct ras_fs_if *head)
 
1194{
1195	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1196	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
1197
1198	if (!obj || obj->ent)
1199		return;
1200
1201	get_obj(obj);
1202
1203	memcpy(obj->fs_data.debugfs_name,
1204			head->debugfs_name,
1205			sizeof(obj->fs_data.debugfs_name));
1206
1207	obj->ent = debugfs_create_file(obj->fs_data.debugfs_name,
1208				       S_IWUGO | S_IRUGO, con->dir, obj,
1209				       &amdgpu_ras_debugfs_ops);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1210}
1211
1212void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
1213{
1214	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 
1215	struct ras_manager *obj;
1216	struct ras_fs_if fs_info;
1217
1218	/*
1219	 * it won't be called in resume path, no need to check
1220	 * suspend and gpu reset status
1221	 */
1222	if (!con)
1223		return;
1224
1225	amdgpu_ras_debugfs_create_ctrl_node(adev);
1226
1227	list_for_each_entry(obj, &con->head, node) {
1228		if (amdgpu_ras_is_supported(adev, obj->head.block) &&
1229			(obj->attr_inuse == 1)) {
1230			sprintf(fs_info.debugfs_name, "%s_err_inject",
1231					ras_block_str(obj->head.block));
1232			fs_info.head = obj->head;
1233			amdgpu_ras_debugfs_create(adev, &fs_info);
1234		}
1235	}
 
 
 
 
 
 
 
1236}
1237
1238void amdgpu_ras_debugfs_remove(struct amdgpu_device *adev,
1239		struct ras_common_if *head)
 
 
 
 
 
 
 
 
 
 
 
 
1240{
1241	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1242
1243	if (!obj || !obj->ent)
1244		return;
1245
1246	obj->ent = NULL;
1247	put_obj(obj);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1248}
1249
1250static void amdgpu_ras_debugfs_remove_all(struct amdgpu_device *adev)
1251{
1252	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1253	struct ras_manager *obj, *tmp;
1254
1255	list_for_each_entry_safe(obj, tmp, &con->head, node) {
1256		amdgpu_ras_debugfs_remove(adev, &obj->head);
 
 
 
 
1257	}
1258
1259	con->dir = NULL;
 
1260}
1261/* debugfs end */
1262
1263/* ras fs */
1264
1265static int amdgpu_ras_fs_init(struct amdgpu_device *adev)
 
 
 
 
 
1266{
1267	amdgpu_ras_sysfs_create_feature_node(adev);
 
 
1268
1269	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1270}
1271
1272static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
 
1273{
1274	amdgpu_ras_debugfs_remove_all(adev);
1275	amdgpu_ras_sysfs_remove_all(adev);
1276	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1277}
1278/* ras fs end */
1279
1280/* ih begin */
1281static void amdgpu_ras_interrupt_handler(struct ras_manager *obj)
1282{
1283	struct ras_ih_data *data = &obj->ih_data;
1284	struct amdgpu_iv_entry entry;
1285	int ret;
1286	struct ras_err_data err_data = {0, 0, 0, NULL};
1287
1288	while (data->rptr != data->wptr) {
1289		rmb();
1290		memcpy(&entry, &data->ring[data->rptr],
1291				data->element_size);
1292
1293		wmb();
1294		data->rptr = (data->aligned_element_size +
1295				data->rptr) % data->ring_size;
1296
1297		/* Let IP handle its data, maybe we need get the output
1298		 * from the callback to udpate the error type/count, etc
1299		 */
1300		if (data->cb) {
1301			ret = data->cb(obj->adev, &err_data, &entry);
1302			/* ue will trigger an interrupt, and in that case
1303			 * we need do a reset to recovery the whole system.
1304			 * But leave IP do that recovery, here we just dispatch
1305			 * the error.
1306			 */
1307			if (ret == AMDGPU_RAS_SUCCESS) {
1308				/* these counts could be left as 0 if
1309				 * some blocks do not count error number
1310				 */
1311				obj->err_data.ue_count += err_data.ue_count;
1312				obj->err_data.ce_count += err_data.ce_count;
1313			}
1314		}
1315	}
1316}
1317
1318static void amdgpu_ras_interrupt_process_handler(struct work_struct *work)
1319{
1320	struct ras_ih_data *data =
1321		container_of(work, struct ras_ih_data, ih_work);
1322	struct ras_manager *obj =
1323		container_of(data, struct ras_manager, ih_data);
1324
1325	amdgpu_ras_interrupt_handler(obj);
1326}
1327
1328int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
1329		struct ras_dispatch_if *info)
1330{
1331	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1332	struct ras_ih_data *data = &obj->ih_data;
1333
 
1334	if (!obj)
1335		return -EINVAL;
1336
 
 
1337	if (data->inuse == 0)
1338		return 0;
1339
1340	/* Might be overflow... */
1341	memcpy(&data->ring[data->wptr], info->entry,
1342			data->element_size);
1343
1344	wmb();
1345	data->wptr = (data->aligned_element_size +
1346			data->wptr) % data->ring_size;
1347
1348	schedule_work(&data->ih_work);
1349
1350	return 0;
1351}
1352
1353int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
1354		struct ras_ih_if *info)
1355{
1356	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1357	struct ras_ih_data *data;
1358
1359	if (!obj)
1360		return -EINVAL;
1361
1362	data = &obj->ih_data;
1363	if (data->inuse == 0)
1364		return 0;
1365
1366	cancel_work_sync(&data->ih_work);
1367
1368	kfree(data->ring);
1369	memset(data, 0, sizeof(*data));
1370	put_obj(obj);
1371
1372	return 0;
1373}
1374
1375int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
1376		struct ras_ih_if *info)
1377{
1378	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1379	struct ras_ih_data *data;
 
1380
1381	if (!obj) {
1382		/* in case we registe the IH before enable ras feature */
1383		obj = amdgpu_ras_create_obj(adev, &info->head);
1384		if (!obj)
1385			return -EINVAL;
1386	} else
1387		get_obj(obj);
1388
 
 
1389	data = &obj->ih_data;
1390	/* add the callback.etc */
1391	*data = (struct ras_ih_data) {
1392		.inuse = 0,
1393		.cb = info->cb,
1394		.element_size = sizeof(struct amdgpu_iv_entry),
1395		.rptr = 0,
1396		.wptr = 0,
1397	};
1398
1399	INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler);
1400
1401	data->aligned_element_size = ALIGN(data->element_size, 8);
1402	/* the ring can store 64 iv entries. */
1403	data->ring_size = 64 * data->aligned_element_size;
1404	data->ring = kmalloc(data->ring_size, GFP_KERNEL);
1405	if (!data->ring) {
1406		put_obj(obj);
1407		return -ENOMEM;
1408	}
1409
1410	/* IH is ready */
1411	data->inuse = 1;
1412
1413	return 0;
1414}
1415
1416static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
1417{
1418	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1419	struct ras_manager *obj, *tmp;
1420
1421	list_for_each_entry_safe(obj, tmp, &con->head, node) {
1422		struct ras_ih_if info = {
1423			.head = obj->head,
1424		};
1425		amdgpu_ras_interrupt_remove_handler(adev, &info);
1426	}
1427
1428	return 0;
1429}
1430/* ih end */
1431
1432/* traversal all IPs except NBIO to query error counter */
1433static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev)
1434{
1435	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1436	struct ras_manager *obj;
1437
1438	if (!con)
1439		return;
1440
1441	list_for_each_entry(obj, &con->head, node) {
1442		struct ras_query_if info = {
1443			.head = obj->head,
1444		};
1445
1446		/*
1447		 * PCIE_BIF IP has one different isr by ras controller
1448		 * interrupt, the specific ras counter query will be
1449		 * done in that isr. So skip such block from common
1450		 * sync flood interrupt isr calling.
1451		 */
1452		if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF)
1453			continue;
1454
1455		amdgpu_ras_error_query(adev, &info);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1456	}
1457}
1458
1459/* recovery begin */
1460
1461/* return 0 on success.
1462 * caller need free bps.
1463 */
1464static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1465		struct ras_badpage **bps, unsigned int *count)
1466{
1467	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1468	struct ras_err_handler_data *data;
1469	int i = 0;
1470	int ret = 0;
1471
1472	if (!con || !con->eh_data || !bps || !count)
1473		return -EINVAL;
1474
1475	mutex_lock(&con->recovery_lock);
1476	data = con->eh_data;
1477	if (!data || data->count == 0) {
1478		*bps = NULL;
1479		ret = -EINVAL;
1480		goto out;
1481	}
1482
1483	*bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL);
1484	if (!*bps) {
1485		ret = -ENOMEM;
1486		goto out;
1487	}
1488
1489	for (; i < data->count; i++) {
1490		(*bps)[i] = (struct ras_badpage){
1491			.bp = data->bps[i].retired_page,
1492			.size = AMDGPU_GPU_PAGE_SIZE,
1493			.flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED,
1494		};
1495
1496		if (data->last_reserved <= i)
 
1497			(*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING;
1498		else if (data->bps_bo[i] == NULL)
1499			(*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT;
1500	}
1501
1502	*count = data->count;
1503out:
1504	mutex_unlock(&con->recovery_lock);
1505	return ret;
1506}
1507
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1508static void amdgpu_ras_do_recovery(struct work_struct *work)
1509{
1510	struct amdgpu_ras *ras =
1511		container_of(work, struct amdgpu_ras, recovery_work);
1512	struct amdgpu_device *remote_adev = NULL;
1513	struct amdgpu_device *adev = ras->adev;
1514	struct list_head device_list, *device_list_handle =  NULL;
1515	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev, false);
 
 
 
 
1516
1517	/* Build list of devices to query RAS related errors */
1518	if  (hive && adev->gmc.xgmi.num_physical_nodes > 1)
1519		device_list_handle = &hive->device_list;
1520	else {
1521		INIT_LIST_HEAD(&device_list);
1522		list_add_tail(&adev->gmc.xgmi.head, &device_list);
1523		device_list_handle = &device_list;
 
 
 
 
1524	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1525
1526	list_for_each_entry(remote_adev, device_list_handle, gmc.xgmi.head) {
1527		amdgpu_ras_log_on_err_counter(remote_adev);
1528	}
1529
1530	if (amdgpu_device_should_recover_gpu(ras->adev))
1531		amdgpu_device_gpu_recover(ras->adev, 0);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1532	atomic_set(&ras->in_recovery, 0);
 
 
 
 
1533}
1534
1535/* alloc/realloc bps array */
1536static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
1537		struct ras_err_handler_data *data, int pages)
1538{
1539	unsigned int old_space = data->count + data->space_left;
1540	unsigned int new_space = old_space + pages;
1541	unsigned int align_space = ALIGN(new_space, 512);
1542	void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL);
1543	struct amdgpu_bo **bps_bo =
1544			kmalloc(align_space * sizeof(*data->bps_bo), GFP_KERNEL);
1545
1546	if (!bps || !bps_bo) {
1547		kfree(bps);
1548		kfree(bps_bo);
1549		return -ENOMEM;
1550	}
1551
1552	if (data->bps) {
1553		memcpy(bps, data->bps,
1554				data->count * sizeof(*data->bps));
1555		kfree(data->bps);
1556	}
1557	if (data->bps_bo) {
1558		memcpy(bps_bo, data->bps_bo,
1559				data->count * sizeof(*data->bps_bo));
1560		kfree(data->bps_bo);
1561	}
1562
1563	data->bps = bps;
1564	data->bps_bo = bps_bo;
1565	data->space_left += align_space - old_space;
1566	return 0;
1567}
1568
1569/* it deal with vram only. */
1570int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
1571		struct eeprom_table_record *bps, int pages)
1572{
1573	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1574	struct ras_err_handler_data *data;
1575	int ret = 0;
 
1576
1577	if (!con || !con->eh_data || !bps || pages <= 0)
1578		return 0;
1579
1580	mutex_lock(&con->recovery_lock);
1581	data = con->eh_data;
1582	if (!data)
1583		goto out;
1584
1585	if (data->space_left <= pages)
1586		if (amdgpu_ras_realloc_eh_data_space(adev, data, pages)) {
 
 
 
 
 
1587			ret = -ENOMEM;
1588			goto out;
1589		}
1590
1591	memcpy(&data->bps[data->count], bps, pages * sizeof(*data->bps));
1592	data->count += pages;
1593	data->space_left -= pages;
1594
 
 
 
 
1595out:
1596	mutex_unlock(&con->recovery_lock);
1597
1598	return ret;
1599}
1600
1601/*
1602 * write error record array to eeprom, the function should be
1603 * protected by recovery_lock
 
1604 */
1605static int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev)
 
1606{
1607	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1608	struct ras_err_handler_data *data;
1609	struct amdgpu_ras_eeprom_control *control;
1610	int save_count;
1611
1612	if (!con || !con->eh_data)
 
 
 
1613		return 0;
 
1614
 
1615	control = &con->eeprom_control;
1616	data = con->eh_data;
1617	save_count = data->count - control->num_recs;
 
 
 
 
 
1618	/* only new entries are saved */
1619	if (save_count > 0) {
1620		if (amdgpu_ras_eeprom_process_recods(control,
1621							&data->bps[control->num_recs],
1622							true,
1623							save_count)) {
1624			dev_err(adev->dev, "Failed to save EEPROM table data!");
1625			return -EIO;
1626		}
1627
1628		dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count);
1629	}
1630
1631	return 0;
1632}
1633
1634/*
1635 * read error record array in eeprom and reserve enough space for
1636 * storing new bad pages
1637 */
1638static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
1639{
1640	struct amdgpu_ras_eeprom_control *control =
1641					&adev->psp.ras.ras->eeprom_control;
1642	struct eeprom_table_record *bps = NULL;
1643	int ret = 0;
1644
1645	/* no bad page record, skip eeprom access */
1646	if (!control->num_recs)
1647		return ret;
1648
1649	bps = kcalloc(control->num_recs, sizeof(*bps), GFP_KERNEL);
1650	if (!bps)
1651		return -ENOMEM;
1652
1653	if (amdgpu_ras_eeprom_process_recods(control, bps, false,
1654		control->num_recs)) {
1655		dev_err(adev->dev, "Failed to load EEPROM table records!");
1656		ret = -EIO;
1657		goto out;
1658	}
1659
1660	ret = amdgpu_ras_add_bad_pages(adev, bps, control->num_recs);
1661
1662out:
1663	kfree(bps);
1664	return ret;
1665}
1666
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1667/*
1668 * check if an address belongs to bad page
1669 *
1670 * Note: this check is only for umc block
1671 */
1672static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
1673				uint64_t addr)
1674{
1675	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1676	struct ras_err_handler_data *data;
1677	int i;
1678	bool ret = false;
1679
1680	if (!con || !con->eh_data)
1681		return ret;
1682
1683	mutex_lock(&con->recovery_lock);
1684	data = con->eh_data;
1685	if (!data)
1686		goto out;
 
1687
1688	addr >>= AMDGPU_GPU_PAGE_SHIFT;
1689	for (i = 0; i < data->count; i++)
1690		if (addr == data->bps[i].retired_page) {
1691			ret = true;
1692			goto out;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1693		}
1694
1695out:
1696	mutex_unlock(&con->recovery_lock);
1697	return ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1698}
1699
1700/* called in gpu recovery/init */
1701int amdgpu_ras_reserve_bad_pages(struct amdgpu_device *adev)
1702{
1703	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1704	struct ras_err_handler_data *data;
1705	uint64_t bp;
1706	struct amdgpu_bo *bo = NULL;
1707	int i, ret = 0;
1708
1709	if (!con || !con->eh_data)
1710		return 0;
 
 
1711
1712	mutex_lock(&con->recovery_lock);
1713	data = con->eh_data;
1714	if (!data)
1715		goto out;
1716	/* reserve vram at driver post stage. */
1717	for (i = data->last_reserved; i < data->count; i++) {
1718		bp = data->bps[i].retired_page;
1719
1720		/* There are two cases of reserve error should be ignored:
1721		 * 1) a ras bad page has been allocated (used by someone);
1722		 * 2) a ras bad page has been reserved (duplicate error injection
1723		 *    for one page);
1724		 */
1725		if (amdgpu_bo_create_kernel_at(adev, bp << AMDGPU_GPU_PAGE_SHIFT,
1726					       AMDGPU_GPU_PAGE_SIZE,
1727					       AMDGPU_GEM_DOMAIN_VRAM,
1728					       &bo, NULL))
1729			dev_warn(adev->dev, "RAS WARN: reserve vram for "
1730					"retired page %llx fail\n", bp);
1731
1732		data->bps_bo[i] = bo;
1733		data->last_reserved = i + 1;
1734		bo = NULL;
1735	}
1736
1737	/* continue to save bad pages to eeprom even reesrve_vram fails */
1738	ret = amdgpu_ras_save_bad_pages(adev);
1739out:
1740	mutex_unlock(&con->recovery_lock);
1741	return ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1742}
1743
1744/* called when driver unload */
1745static int amdgpu_ras_release_bad_pages(struct amdgpu_device *adev)
1746{
 
1747	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1748	struct ras_err_handler_data *data;
1749	struct amdgpu_bo *bo;
1750	int i;
1751
1752	if (!con || !con->eh_data)
1753		return 0;
1754
1755	mutex_lock(&con->recovery_lock);
1756	data = con->eh_data;
1757	if (!data)
1758		goto out;
 
 
 
 
1759
1760	for (i = data->last_reserved - 1; i >= 0; i--) {
1761		bo = data->bps_bo[i];
 
 
 
 
 
 
 
 
 
1762
1763		amdgpu_bo_free_kernel(&bo, NULL, NULL);
 
 
 
 
 
 
 
 
 
1764
1765		data->bps_bo[i] = bo;
1766		data->last_reserved = i;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1767	}
1768out:
1769	mutex_unlock(&con->recovery_lock);
1770	return 0;
1771}
1772
1773int amdgpu_ras_recovery_init(struct amdgpu_device *adev)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1774{
1775	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1776	struct ras_err_handler_data **data;
 
1777	int ret;
1778
1779	if (con)
1780		data = &con->eh_data;
1781	else
1782		return 0;
1783
1784	*data = kmalloc(sizeof(**data), GFP_KERNEL | __GFP_ZERO);
 
 
 
 
 
 
 
 
 
 
 
1785	if (!*data) {
1786		ret = -ENOMEM;
1787		goto out;
1788	}
1789
1790	mutex_init(&con->recovery_lock);
1791	INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
1792	atomic_set(&con->in_recovery, 0);
1793	con->adev = adev;
1794
1795	ret = amdgpu_ras_eeprom_init(&con->eeprom_control);
1796	if (ret)
1797		goto free;
1798
1799	if (con->eeprom_control.num_recs) {
1800		ret = amdgpu_ras_load_bad_pages(adev);
1801		if (ret)
1802			goto free;
1803		ret = amdgpu_ras_reserve_bad_pages(adev);
1804		if (ret)
1805			goto release;
1806	}
1807
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1808	return 0;
1809
1810release:
1811	amdgpu_ras_release_bad_pages(adev);
1812free:
1813	kfree((*data)->bps);
1814	kfree((*data)->bps_bo);
1815	kfree(*data);
1816	con->eh_data = NULL;
1817out:
1818	dev_warn(adev->dev, "Failed to initialize ras recovery!\n");
 
 
 
 
 
 
 
 
 
1819
1820	return ret;
1821}
1822
1823static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
1824{
1825	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1826	struct ras_err_handler_data *data = con->eh_data;
 
 
1827
1828	/* recovery_init failed to init it, fini is useless */
1829	if (!data)
1830		return 0;
1831
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1832	cancel_work_sync(&con->recovery_work);
1833	amdgpu_ras_release_bad_pages(adev);
 
 
 
1834
1835	mutex_lock(&con->recovery_lock);
1836	con->eh_data = NULL;
1837	kfree(data->bps);
1838	kfree(data->bps_bo);
1839	kfree(data);
1840	mutex_unlock(&con->recovery_lock);
1841
1842	return 0;
1843}
1844/* recovery end */
1845
1846/* return 0 if ras will reset gpu and repost.*/
1847int amdgpu_ras_request_reset_on_boot(struct amdgpu_device *adev,
1848		unsigned int block)
1849{
1850	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 
 
 
 
 
 
 
 
 
1851
1852	if (!ras)
1853		return -EINVAL;
 
 
 
 
 
 
 
 
 
1854
1855	ras->flags |= AMDGPU_RAS_FLAG_INIT_NEED_RESET;
1856	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1857}
1858
1859/*
1860 * check hardware's ras ability which will be saved in hw_supported.
1861 * if hardware does not support ras, we can skip some ras initializtion and
1862 * forbid some ras operations from IP.
1863 * if software itself, say boot parameter, limit the ras ability. We still
1864 * need allow IP do some limited operations, like disable. In such case,
1865 * we have to initialize ras as normal. but need check if operation is
1866 * allowed or not in each function.
1867 */
1868static void amdgpu_ras_check_supported(struct amdgpu_device *adev,
1869		uint32_t *hw_supported, uint32_t *supported)
1870{
1871	*hw_supported = 0;
1872	*supported = 0;
1873
1874	if (amdgpu_sriov_vf(adev) || !adev->is_atom_fw ||
1875	    (adev->asic_type != CHIP_VEGA20 &&
1876	     adev->asic_type != CHIP_ARCTURUS))
1877		return;
1878
1879	if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
1880		dev_info(adev->dev, "HBM ECC is active.\n");
1881		*hw_supported |= (1 << AMDGPU_RAS_BLOCK__UMC |
1882				1 << AMDGPU_RAS_BLOCK__DF);
1883	} else
1884		dev_info(adev->dev, "HBM ECC is not presented.\n");
1885
1886	if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
1887		dev_info(adev->dev, "SRAM ECC is active.\n");
1888		*hw_supported |= ~(1 << AMDGPU_RAS_BLOCK__UMC |
1889				1 << AMDGPU_RAS_BLOCK__DF);
1890	} else
1891		dev_info(adev->dev, "SRAM ECC is not presented.\n");
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1892
 
1893	/* hw_supported needs to be aligned with RAS block mask. */
1894	*hw_supported &= AMDGPU_RAS_BLOCK_MASK;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1895
1896	*supported = amdgpu_ras_enable == 0 ?
1897			0 : *hw_supported & amdgpu_ras_mask;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1898}
1899
1900int amdgpu_ras_init(struct amdgpu_device *adev)
1901{
1902	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1903	int r;
1904
1905	if (con)
1906		return 0;
1907
1908	con = kmalloc(sizeof(struct amdgpu_ras) +
1909			sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT,
1910			GFP_KERNEL|__GFP_ZERO);
 
1911	if (!con)
1912		return -ENOMEM;
1913
 
 
 
 
 
1914	con->objs = (struct ras_manager *)(con + 1);
1915
1916	amdgpu_ras_set_context(adev, con);
1917
1918	amdgpu_ras_check_supported(adev, &con->hw_supported,
1919			&con->supported);
1920	if (!con->hw_supported) {
 
 
 
 
 
 
 
 
 
1921		r = 0;
1922		goto err_out;
1923	}
1924
 
1925	con->features = 0;
 
1926	INIT_LIST_HEAD(&con->head);
1927	/* Might need get this flag from vbios. */
1928	con->flags = RAS_DEFAULT_FLAGS;
1929
1930	if (adev->nbio.funcs->init_ras_controller_interrupt) {
1931		r = adev->nbio.funcs->init_ras_controller_interrupt(adev);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1932		if (r)
1933			goto err_out;
1934	}
1935
1936	if (adev->nbio.funcs->init_ras_err_event_athub_interrupt) {
1937		r = adev->nbio.funcs->init_ras_err_event_athub_interrupt(adev);
 
1938		if (r)
1939			goto err_out;
1940	}
1941
 
 
 
 
 
 
 
 
 
 
 
1942	if (amdgpu_ras_fs_init(adev)) {
1943		r = -EINVAL;
1944		goto err_out;
 
 
 
 
 
 
 
 
 
1945	}
1946
1947	dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
1948			"hardware ability[%x] ras_mask[%x]\n",
1949			con->hw_supported, con->supported);
 
1950	return 0;
1951err_out:
1952	amdgpu_ras_set_context(adev, NULL);
1953	kfree(con);
1954
1955	return r;
1956}
1957
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1958/* helper function to handle common stuff in ip late init phase */
1959int amdgpu_ras_late_init(struct amdgpu_device *adev,
1960			 struct ras_common_if *ras_block,
1961			 struct ras_fs_if *fs_info,
1962			 struct ras_ih_if *ih_info)
1963{
 
 
 
 
1964	int r;
1965
1966	/* disable RAS feature per IP block if it is not supported */
1967	if (!amdgpu_ras_is_supported(adev, ras_block->block)) {
1968		amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
1969		return 0;
1970	}
1971
1972	r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1);
1973	if (r) {
1974		if (r == -EAGAIN) {
1975			/* request gpu reset. will run again */
1976			amdgpu_ras_request_reset_on_boot(adev,
1977					ras_block->block);
1978			return 0;
1979		} else if (adev->in_suspend || adev->in_gpu_reset) {
1980			/* in resume phase, if fail to enable ras,
1981			 * clean up all ras fs nodes, and disable ras */
1982			goto cleanup;
1983		} else
1984			return r;
1985	}
1986
 
 
 
1987	/* in resume phase, no need to create ras fs node */
1988	if (adev->in_suspend || adev->in_gpu_reset)
1989		return 0;
1990
1991	if (ih_info->cb) {
1992		r = amdgpu_ras_interrupt_add_handler(adev, ih_info);
 
 
 
1993		if (r)
1994			goto interrupt;
1995	}
1996
1997	r = amdgpu_ras_sysfs_create(adev, fs_info);
1998	if (r)
1999		goto sysfs;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2000
2001	return 0;
2002cleanup:
2003	amdgpu_ras_sysfs_remove(adev, ras_block);
2004sysfs:
2005	if (ih_info->cb)
2006		amdgpu_ras_interrupt_remove_handler(adev, ih_info);
2007interrupt:
 
 
 
2008	amdgpu_ras_feature_enable(adev, ras_block, 0);
2009	return r;
2010}
2011
 
 
 
 
 
 
2012/* helper function to remove ras fs node and interrupt handler */
2013void amdgpu_ras_late_fini(struct amdgpu_device *adev,
2014			  struct ras_common_if *ras_block,
2015			  struct ras_ih_if *ih_info)
2016{
2017	if (!ras_block || !ih_info)
 
2018		return;
2019
2020	amdgpu_ras_sysfs_remove(adev, ras_block);
2021	if (ih_info->cb)
2022                amdgpu_ras_interrupt_remove_handler(adev, ih_info);
2023	amdgpu_ras_feature_enable(adev, ras_block, 0);
 
 
 
 
 
 
 
2024}
2025
2026/* do some init work after IP late init as dependence.
2027 * and it runs in resume/gpu reset/booting up cases.
2028 */
2029void amdgpu_ras_resume(struct amdgpu_device *adev)
2030{
2031	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2032	struct ras_manager *obj, *tmp;
2033
2034	if (!con)
 
 
 
2035		return;
 
2036
2037	if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
2038		/* Set up all other IPs which are not implemented. There is a
2039		 * tricky thing that IP's actual ras error type should be
2040		 * MULTI_UNCORRECTABLE, but as driver does not handle it, so
2041		 * ERROR_NONE make sense anyway.
2042		 */
2043		amdgpu_ras_enable_all_features(adev, 1);
2044
2045		/* We enable ras on all hw_supported block, but as boot
2046		 * parameter might disable some of them and one or more IP has
2047		 * not implemented yet. So we disable them on behalf.
2048		 */
2049		list_for_each_entry_safe(obj, tmp, &con->head, node) {
2050			if (!amdgpu_ras_is_supported(adev, obj->head.block)) {
2051				amdgpu_ras_feature_enable(adev, &obj->head, 0);
2052				/* there should be no any reference. */
2053				WARN_ON(alive_obj(obj));
2054			}
2055		}
2056	}
2057
2058	if (con->flags & AMDGPU_RAS_FLAG_INIT_NEED_RESET) {
2059		con->flags &= ~AMDGPU_RAS_FLAG_INIT_NEED_RESET;
2060		/* setup ras obj state as disabled.
2061		 * for init_by_vbios case.
2062		 * if we want to enable ras, just enable it in a normal way.
2063		 * If we want do disable it, need setup ras obj as enabled,
2064		 * then issue another TA disable cmd.
2065		 * See feature_enable_on_boot
2066		 */
2067		amdgpu_ras_disable_all_features(adev, 1);
2068		amdgpu_ras_reset_gpu(adev);
2069	}
2070}
2071
2072void amdgpu_ras_suspend(struct amdgpu_device *adev)
2073{
2074	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2075
2076	if (!con)
2077		return;
2078
2079	amdgpu_ras_disable_all_features(adev, 0);
2080	/* Make sure all ras objects are disabled. */
2081	if (con->features)
2082		amdgpu_ras_disable_all_features(adev, 1);
2083}
2084
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2085/* do some fini work before IP fini as dependence */
2086int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
2087{
2088	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2089
2090	if (!con)
2091		return 0;
2092
 
2093	/* Need disable ras on all IPs here before ip [hw/sw]fini */
2094	amdgpu_ras_disable_all_features(adev, 0);
 
2095	amdgpu_ras_recovery_fini(adev);
2096	return 0;
2097}
2098
2099int amdgpu_ras_fini(struct amdgpu_device *adev)
2100{
 
 
2101	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2102
2103	if (!con)
2104		return 0;
2105
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2106	amdgpu_ras_fs_fini(adev);
2107	amdgpu_ras_interrupt_remove_all(adev);
2108
2109	WARN(con->features, "Feature mask is not cleared");
 
 
 
 
 
2110
2111	if (con->features)
2112		amdgpu_ras_disable_all_features(adev, 1);
 
 
 
 
2113
2114	amdgpu_ras_set_context(adev, NULL);
2115	kfree(con);
2116
2117	return 0;
2118}
2119
2120void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
2121{
2122	uint32_t hw_supported, supported;
2123
2124	amdgpu_ras_check_supported(adev, &hw_supported, &supported);
2125	if (!hw_supported)
2126		return;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2127
 
 
 
 
 
 
 
 
 
 
 
 
 
2128	if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
2129		dev_info(adev->dev, "uncorrectable hardware error"
2130			"(ERREVENT_ATHUB_INTERRUPT) detected!\n");
 
 
 
 
 
 
 
 
 
2131
 
 
2132		amdgpu_ras_reset_gpu(adev);
2133	}
2134}
2135
2136bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev)
2137{
2138	if (adev->asic_type == CHIP_VEGA20 &&
2139	    adev->pm.fw_version <= 0x283400) {
2140		return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) &&
2141				amdgpu_ras_intr_triggered();
2142	}
2143
2144	return false;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2145}