1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   4 */
   5#include <linux/list_sort.h>
   6#include <linux/libnvdimm.h>
   7#include <linux/module.h>
   8#include <linux/mutex.h>
   9#include <linux/ndctl.h>
  10#include <linux/sysfs.h>
  11#include <linux/delay.h>
  12#include <linux/list.h>
  13#include <linux/acpi.h>
  14#include <linux/sort.h>
  15#include <linux/io.h>
  16#include <linux/nd.h>
  17#include <asm/cacheflush.h>
  18#include <acpi/nfit.h>
  19#include "intel.h"
  20#include "nfit.h"
  21
  22/*
  23 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
  24 * irrelevant.
  25 */
  26#include <linux/io-64-nonatomic-hi-lo.h>
  27
  28static bool force_enable_dimms;
  29module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR);
  30MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status");
  31
  32static bool disable_vendor_specific;
  33module_param(disable_vendor_specific, bool, S_IRUGO);
  34MODULE_PARM_DESC(disable_vendor_specific,
  35		"Limit commands to the publicly specified set");
  36
  37static unsigned long override_dsm_mask;
  38module_param(override_dsm_mask, ulong, S_IRUGO);
  39MODULE_PARM_DESC(override_dsm_mask, "Bitmask of allowed NVDIMM DSM functions");
  40
  41static int default_dsm_family = -1;
  42module_param(default_dsm_family, int, S_IRUGO);
  43MODULE_PARM_DESC(default_dsm_family,
  44		"Try this DSM type first when identifying NVDIMM family");
  45
  46static bool no_init_ars;
  47module_param(no_init_ars, bool, 0644);
  48MODULE_PARM_DESC(no_init_ars, "Skip ARS run at nfit init time");
  49
  50static bool force_labels;
  51module_param(force_labels, bool, 0444);
  52MODULE_PARM_DESC(force_labels, "Opt-in to labels despite missing methods");
  53
  54LIST_HEAD(acpi_descs);
  55DEFINE_MUTEX(acpi_desc_lock);
  56
  57static struct workqueue_struct *nfit_wq;
  58
  59struct nfit_table_prev {
  60	struct list_head spas;
  61	struct list_head memdevs;
  62	struct list_head dcrs;
  63	struct list_head bdws;
  64	struct list_head idts;
  65	struct list_head flushes;
  66};
  67
  68static guid_t nfit_uuid[NFIT_UUID_MAX];
  69
  70const guid_t *to_nfit_uuid(enum nfit_uuids id)
  71{
  72	return &nfit_uuid[id];
  73}
  74EXPORT_SYMBOL(to_nfit_uuid);
  75
  76static const guid_t *to_nfit_bus_uuid(int family)
  77{
  78	if (WARN_ONCE(family == NVDIMM_BUS_FAMILY_NFIT,
  79			"only secondary bus families can be translated\n"))
  80		return NULL;
  81	/*
  82	 * The index of bus UUIDs starts immediately following the last
  83	 * NVDIMM/leaf family.
  84	 */
  85	return to_nfit_uuid(family + NVDIMM_FAMILY_MAX);
  86}
  87
  88static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc)
  89{
  90	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
  91
  92	/*
  93	 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct
  94	 * acpi_device.
  95	 */
  96	if (!nd_desc->provider_name
  97			|| strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0)
  98		return NULL;
  99
 100	return to_acpi_device(acpi_desc->dev);
 101}
 102
 103static int xlat_bus_status(void *buf, unsigned int cmd, u32 status)
 104{
 105	struct nd_cmd_clear_error *clear_err;
 106	struct nd_cmd_ars_status *ars_status;
 107	u16 flags;
 108
 109	switch (cmd) {
 110	case ND_CMD_ARS_CAP:
 111		if ((status & 0xffff) == NFIT_ARS_CAP_NONE)
 112			return -ENOTTY;
 113
 114		/* Command failed */
 115		if (status & 0xffff)
 116			return -EIO;
 117
 118		/* No supported scan types for this range */
 119		flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE;
 120		if ((status >> 16 & flags) == 0)
 121			return -ENOTTY;
 122		return 0;
 123	case ND_CMD_ARS_START:
 124		/* ARS is in progress */
 125		if ((status & 0xffff) == NFIT_ARS_START_BUSY)
 126			return -EBUSY;
 127
 128		/* Command failed */
 129		if (status & 0xffff)
 130			return -EIO;
 131		return 0;
 132	case ND_CMD_ARS_STATUS:
 133		ars_status = buf;
 134		/* Command failed */
 135		if (status & 0xffff)
 136			return -EIO;
 137		/* Check extended status (Upper two bytes) */
 138		if (status == NFIT_ARS_STATUS_DONE)
 139			return 0;
 140
 141		/* ARS is in progress */
 142		if (status == NFIT_ARS_STATUS_BUSY)
 143			return -EBUSY;
 144
 145		/* No ARS performed for the current boot */
 146		if (status == NFIT_ARS_STATUS_NONE)
 147			return -EAGAIN;
 148
 149		/*
 150		 * ARS interrupted, either we overflowed or some other
 151		 * agent wants the scan to stop.  If we didn't overflow
 152		 * then just continue with the returned results.
 153		 */
 154		if (status == NFIT_ARS_STATUS_INTR) {
 155			if (ars_status->out_length >= 40 && (ars_status->flags
 156						& NFIT_ARS_F_OVERFLOW))
 157				return -ENOSPC;
 158			return 0;
 159		}
 160
 161		/* Unknown status */
 162		if (status >> 16)
 163			return -EIO;
 164		return 0;
 165	case ND_CMD_CLEAR_ERROR:
 166		clear_err = buf;
 167		if (status & 0xffff)
 168			return -EIO;
 169		if (!clear_err->cleared)
 170			return -EIO;
 171		if (clear_err->length > clear_err->cleared)
 172			return clear_err->cleared;
 173		return 0;
 174	default:
 175		break;
 176	}
 177
 178	/* all other non-zero status results in an error */
 179	if (status)
 180		return -EIO;
 181	return 0;
 182}
 183
 184#define ACPI_LABELS_LOCKED 3
 185
 186static int xlat_nvdimm_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd,
 187		u32 status)
 188{
 189	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 190
 191	switch (cmd) {
 192	case ND_CMD_GET_CONFIG_SIZE:
 193		/*
 194		 * In the _LSI, _LSR, _LSW case the locked status is
 195		 * communicated via the read/write commands
 196		 */
 197		if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags))
 198			break;
 199
 200		if (status >> 16 & ND_CONFIG_LOCKED)
 201			return -EACCES;
 202		break;
 203	case ND_CMD_GET_CONFIG_DATA:
 204		if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags)
 205				&& status == ACPI_LABELS_LOCKED)
 206			return -EACCES;
 207		break;
 208	case ND_CMD_SET_CONFIG_DATA:
 209		if (test_bit(NFIT_MEM_LSW, &nfit_mem->flags)
 210				&& status == ACPI_LABELS_LOCKED)
 211			return -EACCES;
 212		break;
 213	default:
 214		break;
 215	}
 216
 217	/* all other non-zero status results in an error */
 218	if (status)
 219		return -EIO;
 220	return 0;
 221}
 222
 223static int xlat_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd,
 224		u32 status)
 225{
 226	if (!nvdimm)
 227		return xlat_bus_status(buf, cmd, status);
 228	return xlat_nvdimm_status(nvdimm, buf, cmd, status);
 229}
 230
 231/* convert _LS{I,R} packages to the buffer object acpi_nfit_ctl expects */
 232static union acpi_object *pkg_to_buf(union acpi_object *pkg)
 233{
 234	int i;
 235	void *dst;
 236	size_t size = 0;
 237	union acpi_object *buf = NULL;
 238
 239	if (pkg->type != ACPI_TYPE_PACKAGE) {
 240		WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n",
 241				pkg->type);
 242		goto err;
 243	}
 244
 245	for (i = 0; i < pkg->package.count; i++) {
 246		union acpi_object *obj = &pkg->package.elements[i];
 247
 248		if (obj->type == ACPI_TYPE_INTEGER)
 249			size += 4;
 250		else if (obj->type == ACPI_TYPE_BUFFER)
 251			size += obj->buffer.length;
 252		else {
 253			WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n",
 254					obj->type);
 255			goto err;
 256		}
 257	}
 258
 259	buf = ACPI_ALLOCATE(sizeof(*buf) + size);
 260	if (!buf)
 261		goto err;
 262
 263	dst = buf + 1;
 264	buf->type = ACPI_TYPE_BUFFER;
 265	buf->buffer.length = size;
 266	buf->buffer.pointer = dst;
 267	for (i = 0; i < pkg->package.count; i++) {
 268		union acpi_object *obj = &pkg->package.elements[i];
 269
 270		if (obj->type == ACPI_TYPE_INTEGER) {
 271			memcpy(dst, &obj->integer.value, 4);
 272			dst += 4;
 273		} else if (obj->type == ACPI_TYPE_BUFFER) {
 274			memcpy(dst, obj->buffer.pointer, obj->buffer.length);
 275			dst += obj->buffer.length;
 276		}
 277	}
 278err:
 279	ACPI_FREE(pkg);
 280	return buf;
 281}
 282
 283static union acpi_object *int_to_buf(union acpi_object *integer)
 284{
 285	union acpi_object *buf = ACPI_ALLOCATE(sizeof(*buf) + 4);
 286	void *dst = NULL;
 287
 288	if (!buf)
 289		goto err;
 290
 291	if (integer->type != ACPI_TYPE_INTEGER) {
 292		WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n",
 293				integer->type);
 294		goto err;
 295	}
 296
 297	dst = buf + 1;
 298	buf->type = ACPI_TYPE_BUFFER;
 299	buf->buffer.length = 4;
 300	buf->buffer.pointer = dst;
 301	memcpy(dst, &integer->integer.value, 4);
 302err:
 303	ACPI_FREE(integer);
 304	return buf;
 305}
 306
 307static union acpi_object *acpi_label_write(acpi_handle handle, u32 offset,
 308		u32 len, void *data)
 309{
 310	acpi_status rc;
 311	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
 312	struct acpi_object_list input = {
 313		.count = 3,
 314		.pointer = (union acpi_object []) {
 315			[0] = {
 316				.integer.type = ACPI_TYPE_INTEGER,
 317				.integer.value = offset,
 318			},
 319			[1] = {
 320				.integer.type = ACPI_TYPE_INTEGER,
 321				.integer.value = len,
 322			},
 323			[2] = {
 324				.buffer.type = ACPI_TYPE_BUFFER,
 325				.buffer.pointer = data,
 326				.buffer.length = len,
 327			},
 328		},
 329	};
 330
 331	rc = acpi_evaluate_object(handle, "_LSW", &input, &buf);
 332	if (ACPI_FAILURE(rc))
 333		return NULL;
 334	return int_to_buf(buf.pointer);
 335}
 336
 337static union acpi_object *acpi_label_read(acpi_handle handle, u32 offset,
 338		u32 len)
 339{
 340	acpi_status rc;
 341	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
 342	struct acpi_object_list input = {
 343		.count = 2,
 344		.pointer = (union acpi_object []) {
 345			[0] = {
 346				.integer.type = ACPI_TYPE_INTEGER,
 347				.integer.value = offset,
 348			},
 349			[1] = {
 350				.integer.type = ACPI_TYPE_INTEGER,
 351				.integer.value = len,
 352			},
 353		},
 354	};
 355
 356	rc = acpi_evaluate_object(handle, "_LSR", &input, &buf);
 357	if (ACPI_FAILURE(rc))
 358		return NULL;
 359	return pkg_to_buf(buf.pointer);
 360}
 361
 362static union acpi_object *acpi_label_info(acpi_handle handle)
 363{
 364	acpi_status rc;
 365	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
 366
 367	rc = acpi_evaluate_object(handle, "_LSI", NULL, &buf);
 368	if (ACPI_FAILURE(rc))
 369		return NULL;
 370	return pkg_to_buf(buf.pointer);
 371}
 372
 373static u8 nfit_dsm_revid(unsigned family, unsigned func)
 374{
 375	static const u8 revid_table[NVDIMM_FAMILY_MAX+1][NVDIMM_CMD_MAX+1] = {
 376		[NVDIMM_FAMILY_INTEL] = {
 377			[NVDIMM_INTEL_GET_MODES ...
 378				NVDIMM_INTEL_FW_ACTIVATE_ARM] = 2,
 379		},
 380	};
 381	u8 id;
 382
 383	if (family > NVDIMM_FAMILY_MAX)
 384		return 0;
 385	if (func > NVDIMM_CMD_MAX)
 386		return 0;
 387	id = revid_table[family][func];
 388	if (id == 0)
 389		return 1; /* default */
 390	return id;
 391}
 392
 393static bool payload_dumpable(struct nvdimm *nvdimm, unsigned int func)
 394{
 395	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 396
 397	if (nfit_mem && nfit_mem->family == NVDIMM_FAMILY_INTEL
 398			&& func >= NVDIMM_INTEL_GET_SECURITY_STATE
 399			&& func <= NVDIMM_INTEL_MASTER_SECURE_ERASE)
 400		return IS_ENABLED(CONFIG_NFIT_SECURITY_DEBUG);
 401	return true;
 402}
 403
 404static int cmd_to_func(struct nfit_mem *nfit_mem, unsigned int cmd,
 405		struct nd_cmd_pkg *call_pkg, int *family)
 406{
 407	if (call_pkg) {
 408		int i;
 409
 410		if (nfit_mem && nfit_mem->family != call_pkg->nd_family)
 411			return -ENOTTY;
 412
 413		for (i = 0; i < ARRAY_SIZE(call_pkg->nd_reserved2); i++)
 414			if (call_pkg->nd_reserved2[i])
 415				return -EINVAL;
 416		*family = call_pkg->nd_family;
 417		return call_pkg->nd_command;
 418	}
 419
 420	/* In the !call_pkg case, bus commands == bus functions */
 421	if (!nfit_mem)
 422		return cmd;
 423
 424	/* Linux ND commands == NVDIMM_FAMILY_INTEL function numbers */
 425	if (nfit_mem->family == NVDIMM_FAMILY_INTEL)
 426		return cmd;
 427
 428	/*
 429	 * Force function number validation to fail since 0 is never
 430	 * published as a valid function in dsm_mask.
 431	 */
 432	return 0;
 433}
 434
 435int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm,
 436		unsigned int cmd, void *buf, unsigned int buf_len, int *cmd_rc)
 437{
 438	struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
 439	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 440	union acpi_object in_obj, in_buf, *out_obj;
 441	const struct nd_cmd_desc *desc = NULL;
 442	struct device *dev = acpi_desc->dev;
 443	struct nd_cmd_pkg *call_pkg = NULL;
 444	const char *cmd_name, *dimm_name;
 445	unsigned long cmd_mask, dsm_mask;
 446	u32 offset, fw_status = 0;
 447	acpi_handle handle;
 448	const guid_t *guid;
 449	int func, rc, i;
 450	int family = 0;
 451
 452	if (cmd_rc)
 453		*cmd_rc = -EINVAL;
 454
 455	if (cmd == ND_CMD_CALL)
 456		call_pkg = buf;
 457	func = cmd_to_func(nfit_mem, cmd, call_pkg, &family);
 458	if (func < 0)
 459		return func;
 460
 461	if (nvdimm) {
 462		struct acpi_device *adev = nfit_mem->adev;
 463
 464		if (!adev)
 465			return -ENOTTY;
 466
 467		dimm_name = nvdimm_name(nvdimm);
 468		cmd_name = nvdimm_cmd_name(cmd);
 469		cmd_mask = nvdimm_cmd_mask(nvdimm);
 470		dsm_mask = nfit_mem->dsm_mask;
 471		desc = nd_cmd_dimm_desc(cmd);
 472		guid = to_nfit_uuid(nfit_mem->family);
 473		handle = adev->handle;
 474	} else {
 475		struct acpi_device *adev = to_acpi_dev(acpi_desc);
 476
 477		cmd_name = nvdimm_bus_cmd_name(cmd);
 478		cmd_mask = nd_desc->cmd_mask;
 479		if (cmd == ND_CMD_CALL && call_pkg->nd_family) {
 480			family = call_pkg->nd_family;
 481			if (!test_bit(family, &nd_desc->bus_family_mask))
 482				return -EINVAL;
 483			dsm_mask = acpi_desc->family_dsm_mask[family];
 484			guid = to_nfit_bus_uuid(family);
 485		} else {
 486			dsm_mask = acpi_desc->bus_dsm_mask;
 487			guid = to_nfit_uuid(NFIT_DEV_BUS);
 488		}
 489		desc = nd_cmd_bus_desc(cmd);
 490		handle = adev->handle;
 491		dimm_name = "bus";
 492	}
 493
 494	if (!desc || (cmd && (desc->out_num + desc->in_num == 0)))
 495		return -ENOTTY;
 496
 497	/*
 498	 * Check for a valid command.  For ND_CMD_CALL, we also have to
 499	 * make sure that the DSM function is supported.
 500	 */
 501	if (cmd == ND_CMD_CALL &&
 502	    (func > NVDIMM_CMD_MAX || !test_bit(func, &dsm_mask)))
 503		return -ENOTTY;
 504	else if (!test_bit(cmd, &cmd_mask))
 505		return -ENOTTY;
 506
 507	in_obj.type = ACPI_TYPE_PACKAGE;
 508	in_obj.package.count = 1;
 509	in_obj.package.elements = &in_buf;
 510	in_buf.type = ACPI_TYPE_BUFFER;
 511	in_buf.buffer.pointer = buf;
 512	in_buf.buffer.length = 0;
 513
 514	/* libnvdimm has already validated the input envelope */
 515	for (i = 0; i < desc->in_num; i++)
 516		in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc,
 517				i, buf);
 518
 519	if (call_pkg) {
 520		/* skip over package wrapper */
 521		in_buf.buffer.pointer = (void *) &call_pkg->nd_payload;
 522		in_buf.buffer.length = call_pkg->nd_size_in;
 523	}
 524
 525	dev_dbg(dev, "%s cmd: %d: family: %d func: %d input length: %d\n",
 526		dimm_name, cmd, family, func, in_buf.buffer.length);
 527	if (payload_dumpable(nvdimm, func))
 528		print_hex_dump_debug("nvdimm in  ", DUMP_PREFIX_OFFSET, 4, 4,
 529				in_buf.buffer.pointer,
 530				min_t(u32, 256, in_buf.buffer.length), true);
 531
 532	/* call the BIOS, prefer the named methods over _DSM if available */
 533	if (nvdimm && cmd == ND_CMD_GET_CONFIG_SIZE
 534			&& test_bit(NFIT_MEM_LSR, &nfit_mem->flags))
 535		out_obj = acpi_label_info(handle);
 536	else if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA
 537			&& test_bit(NFIT_MEM_LSR, &nfit_mem->flags)) {
 538		struct nd_cmd_get_config_data_hdr *p = buf;
 539
 540		out_obj = acpi_label_read(handle, p->in_offset, p->in_length);
 541	} else if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA
 542			&& test_bit(NFIT_MEM_LSW, &nfit_mem->flags)) {
 543		struct nd_cmd_set_config_hdr *p = buf;
 544
 545		out_obj = acpi_label_write(handle, p->in_offset, p->in_length,
 546				p->in_buf);
 547	} else {
 548		u8 revid;
 549
 550		if (nvdimm)
 551			revid = nfit_dsm_revid(nfit_mem->family, func);
 552		else
 553			revid = 1;
 554		out_obj = acpi_evaluate_dsm(handle, guid, revid, func, &in_obj);
 555	}
 556
 557	if (!out_obj) {
 558		dev_dbg(dev, "%s _DSM failed cmd: %s\n", dimm_name, cmd_name);
 559		return -EINVAL;
 560	}
 561
 562	if (out_obj->type != ACPI_TYPE_BUFFER) {
 563		dev_dbg(dev, "%s unexpected output object type cmd: %s type: %d\n",
 564				dimm_name, cmd_name, out_obj->type);
 565		rc = -EINVAL;
 566		goto out;
 567	}
 568
 569	dev_dbg(dev, "%s cmd: %s output length: %d\n", dimm_name,
 570			cmd_name, out_obj->buffer.length);
 571	print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4, 4,
 572			out_obj->buffer.pointer,
 573			min_t(u32, 128, out_obj->buffer.length), true);
 574
 575	if (call_pkg) {
 576		call_pkg->nd_fw_size = out_obj->buffer.length;
 577		memcpy(call_pkg->nd_payload + call_pkg->nd_size_in,
 578			out_obj->buffer.pointer,
 579			min(call_pkg->nd_fw_size, call_pkg->nd_size_out));
 580
 581		ACPI_FREE(out_obj);
 582		/*
 583		 * Need to support FW function w/o known size in advance.
 584		 * Caller can determine required size based upon nd_fw_size.
 585		 * If we return an error (like elsewhere) then caller wouldn't
 586		 * be able to rely upon data returned to make calculation.
 587		 */
 588		if (cmd_rc)
 589			*cmd_rc = 0;
 590		return 0;
 591	}
 592
 593	for (i = 0, offset = 0; i < desc->out_num; i++) {
 594		u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf,
 595				(u32 *) out_obj->buffer.pointer,
 596				out_obj->buffer.length - offset);
 597
 598		if (offset + out_size > out_obj->buffer.length) {
 599			dev_dbg(dev, "%s output object underflow cmd: %s field: %d\n",
 600					dimm_name, cmd_name, i);
 601			break;
 602		}
 603
 604		if (in_buf.buffer.length + offset + out_size > buf_len) {
 605			dev_dbg(dev, "%s output overrun cmd: %s field: %d\n",
 606					dimm_name, cmd_name, i);
 607			rc = -ENXIO;
 608			goto out;
 609		}
 610		memcpy(buf + in_buf.buffer.length + offset,
 611				out_obj->buffer.pointer + offset, out_size);
 612		offset += out_size;
 613	}
 614
 615	/*
 616	 * Set fw_status for all the commands with a known format to be
 617	 * later interpreted by xlat_status().
 618	 */
 619	if (i >= 1 && ((!nvdimm && cmd >= ND_CMD_ARS_CAP
 620					&& cmd <= ND_CMD_CLEAR_ERROR)
 621				|| (nvdimm && cmd >= ND_CMD_SMART
 622					&& cmd <= ND_CMD_VENDOR)))
 623		fw_status = *(u32 *) out_obj->buffer.pointer;
 624
 625	if (offset + in_buf.buffer.length < buf_len) {
 626		if (i >= 1) {
 627			/*
 628			 * status valid, return the number of bytes left
 629			 * unfilled in the output buffer
 630			 */
 631			rc = buf_len - offset - in_buf.buffer.length;
 632			if (cmd_rc)
 633				*cmd_rc = xlat_status(nvdimm, buf, cmd,
 634						fw_status);
 635		} else {
 636			dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n",
 637					__func__, dimm_name, cmd_name, buf_len,
 638					offset);
 639			rc = -ENXIO;
 640		}
 641	} else {
 642		rc = 0;
 643		if (cmd_rc)
 644			*cmd_rc = xlat_status(nvdimm, buf, cmd, fw_status);
 645	}
 646
 647 out:
 648	ACPI_FREE(out_obj);
 649
 650	return rc;
 651}
 652EXPORT_SYMBOL_GPL(acpi_nfit_ctl);
 653
 654static const char *spa_type_name(u16 type)
 655{
 656	static const char *to_name[] = {
 657		[NFIT_SPA_VOLATILE] = "volatile",
 658		[NFIT_SPA_PM] = "pmem",
 659		[NFIT_SPA_DCR] = "dimm-control-region",
 660		[NFIT_SPA_BDW] = "block-data-window",
 661		[NFIT_SPA_VDISK] = "volatile-disk",
 662		[NFIT_SPA_VCD] = "volatile-cd",
 663		[NFIT_SPA_PDISK] = "persistent-disk",
 664		[NFIT_SPA_PCD] = "persistent-cd",
 665
 666	};
 667
 668	if (type > NFIT_SPA_PCD)
 669		return "unknown";
 670
 671	return to_name[type];
 672}
 673
 674int nfit_spa_type(struct acpi_nfit_system_address *spa)
 675{
 676	int i;
 677
 678	for (i = 0; i < NFIT_UUID_MAX; i++)
 679		if (guid_equal(to_nfit_uuid(i), (guid_t *)&spa->range_guid))
 680			return i;
 681	return -1;
 682}
 683
 684static bool add_spa(struct acpi_nfit_desc *acpi_desc,
 685		struct nfit_table_prev *prev,
 686		struct acpi_nfit_system_address *spa)
 687{
 688	struct device *dev = acpi_desc->dev;
 689	struct nfit_spa *nfit_spa;
 690
 691	if (spa->header.length != sizeof(*spa))
 692		return false;
 693
 694	list_for_each_entry(nfit_spa, &prev->spas, list) {
 695		if (memcmp(nfit_spa->spa, spa, sizeof(*spa)) == 0) {
 696			list_move_tail(&nfit_spa->list, &acpi_desc->spas);
 697			return true;
 698		}
 699	}
 700
 701	nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa) + sizeof(*spa),
 702			GFP_KERNEL);
 703	if (!nfit_spa)
 704		return false;
 705	INIT_LIST_HEAD(&nfit_spa->list);
 706	memcpy(nfit_spa->spa, spa, sizeof(*spa));
 707	list_add_tail(&nfit_spa->list, &acpi_desc->spas);
 708	dev_dbg(dev, "spa index: %d type: %s\n",
 709			spa->range_index,
 710			spa_type_name(nfit_spa_type(spa)));
 711	return true;
 712}
 713
 714static bool add_memdev(struct acpi_nfit_desc *acpi_desc,
 715		struct nfit_table_prev *prev,
 716		struct acpi_nfit_memory_map *memdev)
 717{
 718	struct device *dev = acpi_desc->dev;
 719	struct nfit_memdev *nfit_memdev;
 720
 721	if (memdev->header.length != sizeof(*memdev))
 722		return false;
 723
 724	list_for_each_entry(nfit_memdev, &prev->memdevs, list)
 725		if (memcmp(nfit_memdev->memdev, memdev, sizeof(*memdev)) == 0) {
 726			list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs);
 727			return true;
 728		}
 729
 730	nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev) + sizeof(*memdev),
 731			GFP_KERNEL);
 732	if (!nfit_memdev)
 733		return false;
 734	INIT_LIST_HEAD(&nfit_memdev->list);
 735	memcpy(nfit_memdev->memdev, memdev, sizeof(*memdev));
 736	list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs);
 737	dev_dbg(dev, "memdev handle: %#x spa: %d dcr: %d flags: %#x\n",
 738			memdev->device_handle, memdev->range_index,
 739			memdev->region_index, memdev->flags);
 740	return true;
 741}
 742
 743int nfit_get_smbios_id(u32 device_handle, u16 *flags)
 744{
 745	struct acpi_nfit_memory_map *memdev;
 746	struct acpi_nfit_desc *acpi_desc;
 747	struct nfit_mem *nfit_mem;
 748	u16 physical_id;
 749
 750	mutex_lock(&acpi_desc_lock);
 751	list_for_each_entry(acpi_desc, &acpi_descs, list) {
 752		mutex_lock(&acpi_desc->init_mutex);
 753		list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
 754			memdev = __to_nfit_memdev(nfit_mem);
 755			if (memdev->device_handle == device_handle) {
 756				*flags = memdev->flags;
 757				physical_id = memdev->physical_id;
 758				mutex_unlock(&acpi_desc->init_mutex);
 759				mutex_unlock(&acpi_desc_lock);
 760				return physical_id;
 761			}
 762		}
 763		mutex_unlock(&acpi_desc->init_mutex);
 764	}
 765	mutex_unlock(&acpi_desc_lock);
 766
 767	return -ENODEV;
 768}
 769EXPORT_SYMBOL_GPL(nfit_get_smbios_id);
 770
 771/*
 772 * An implementation may provide a truncated control region if no block windows
 773 * are defined.
 774 */
 775static size_t sizeof_dcr(struct acpi_nfit_control_region *dcr)
 776{
 777	if (dcr->header.length < offsetof(struct acpi_nfit_control_region,
 778				window_size))
 779		return 0;
 780	if (dcr->windows)
 781		return sizeof(*dcr);
 782	return offsetof(struct acpi_nfit_control_region, window_size);
 783}
 784
 785static bool add_dcr(struct acpi_nfit_desc *acpi_desc,
 786		struct nfit_table_prev *prev,
 787		struct acpi_nfit_control_region *dcr)
 788{
 789	struct device *dev = acpi_desc->dev;
 790	struct nfit_dcr *nfit_dcr;
 791
 792	if (!sizeof_dcr(dcr))
 793		return false;
 794
 795	list_for_each_entry(nfit_dcr, &prev->dcrs, list)
 796		if (memcmp(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)) == 0) {
 797			list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs);
 798			return true;
 799		}
 800
 801	nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr) + sizeof(*dcr),
 802			GFP_KERNEL);
 803	if (!nfit_dcr)
 804		return false;
 805	INIT_LIST_HEAD(&nfit_dcr->list);
 806	memcpy(nfit_dcr->dcr, dcr, sizeof_dcr(dcr));
 807	list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs);
 808	dev_dbg(dev, "dcr index: %d windows: %d\n",
 809			dcr->region_index, dcr->windows);
 810	return true;
 811}
 812
 813static bool add_bdw(struct acpi_nfit_desc *acpi_desc,
 814		struct nfit_table_prev *prev,
 815		struct acpi_nfit_data_region *bdw)
 816{
 817	struct device *dev = acpi_desc->dev;
 818	struct nfit_bdw *nfit_bdw;
 819
 820	if (bdw->header.length != sizeof(*bdw))
 821		return false;
 822	list_for_each_entry(nfit_bdw, &prev->bdws, list)
 823		if (memcmp(nfit_bdw->bdw, bdw, sizeof(*bdw)) == 0) {
 824			list_move_tail(&nfit_bdw->list, &acpi_desc->bdws);
 825			return true;
 826		}
 827
 828	nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw) + sizeof(*bdw),
 829			GFP_KERNEL);
 830	if (!nfit_bdw)
 831		return false;
 832	INIT_LIST_HEAD(&nfit_bdw->list);
 833	memcpy(nfit_bdw->bdw, bdw, sizeof(*bdw));
 834	list_add_tail(&nfit_bdw->list, &acpi_desc->bdws);
 835	dev_dbg(dev, "bdw dcr: %d windows: %d\n",
 836			bdw->region_index, bdw->windows);
 837	return true;
 838}
 839
 840static size_t sizeof_idt(struct acpi_nfit_interleave *idt)
 841{
 842	if (idt->header.length < sizeof(*idt))
 843		return 0;
 844	return sizeof(*idt) + sizeof(u32) * (idt->line_count - 1);
 845}
 846
 847static bool add_idt(struct acpi_nfit_desc *acpi_desc,
 848		struct nfit_table_prev *prev,
 849		struct acpi_nfit_interleave *idt)
 850{
 851	struct device *dev = acpi_desc->dev;
 852	struct nfit_idt *nfit_idt;
 853
 854	if (!sizeof_idt(idt))
 855		return false;
 856
 857	list_for_each_entry(nfit_idt, &prev->idts, list) {
 858		if (sizeof_idt(nfit_idt->idt) != sizeof_idt(idt))
 859			continue;
 860
 861		if (memcmp(nfit_idt->idt, idt, sizeof_idt(idt)) == 0) {
 862			list_move_tail(&nfit_idt->list, &acpi_desc->idts);
 863			return true;
 864		}
 865	}
 866
 867	nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt) + sizeof_idt(idt),
 868			GFP_KERNEL);
 869	if (!nfit_idt)
 870		return false;
 871	INIT_LIST_HEAD(&nfit_idt->list);
 872	memcpy(nfit_idt->idt, idt, sizeof_idt(idt));
 873	list_add_tail(&nfit_idt->list, &acpi_desc->idts);
 874	dev_dbg(dev, "idt index: %d num_lines: %d\n",
 875			idt->interleave_index, idt->line_count);
 876	return true;
 877}
 878
 879static size_t sizeof_flush(struct acpi_nfit_flush_address *flush)
 880{
 881	if (flush->header.length < sizeof(*flush))
 882		return 0;
 883	return sizeof(*flush) + sizeof(u64) * (flush->hint_count - 1);
 884}
 885
 886static bool add_flush(struct acpi_nfit_desc *acpi_desc,
 887		struct nfit_table_prev *prev,
 888		struct acpi_nfit_flush_address *flush)
 889{
 890	struct device *dev = acpi_desc->dev;
 891	struct nfit_flush *nfit_flush;
 892
 893	if (!sizeof_flush(flush))
 894		return false;
 895
 896	list_for_each_entry(nfit_flush, &prev->flushes, list) {
 897		if (sizeof_flush(nfit_flush->flush) != sizeof_flush(flush))
 898			continue;
 899
 900		if (memcmp(nfit_flush->flush, flush,
 901					sizeof_flush(flush)) == 0) {
 902			list_move_tail(&nfit_flush->list, &acpi_desc->flushes);
 903			return true;
 904		}
 905	}
 906
 907	nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush)
 908			+ sizeof_flush(flush), GFP_KERNEL);
 909	if (!nfit_flush)
 910		return false;
 911	INIT_LIST_HEAD(&nfit_flush->list);
 912	memcpy(nfit_flush->flush, flush, sizeof_flush(flush));
 913	list_add_tail(&nfit_flush->list, &acpi_desc->flushes);
 914	dev_dbg(dev, "nfit_flush handle: %d hint_count: %d\n",
 915			flush->device_handle, flush->hint_count);
 916	return true;
 917}
 918
 919static bool add_platform_cap(struct acpi_nfit_desc *acpi_desc,
 920		struct acpi_nfit_capabilities *pcap)
 921{
 922	struct device *dev = acpi_desc->dev;
 923	u32 mask;
 924
 925	mask = (1 << (pcap->highest_capability + 1)) - 1;
 926	acpi_desc->platform_cap = pcap->capabilities & mask;
 927	dev_dbg(dev, "cap: %#x\n", acpi_desc->platform_cap);
 928	return true;
 929}
 930
 931static void *add_table(struct acpi_nfit_desc *acpi_desc,
 932		struct nfit_table_prev *prev, void *table, const void *end)
 933{
 934	struct device *dev = acpi_desc->dev;
 935	struct acpi_nfit_header *hdr;
 936	void *err = ERR_PTR(-ENOMEM);
 937
 938	if (table >= end)
 939		return NULL;
 940
 941	hdr = table;
 942	if (!hdr->length) {
 943		dev_warn(dev, "found a zero length table '%d' parsing nfit\n",
 944			hdr->type);
 945		return NULL;
 946	}
 947
 948	switch (hdr->type) {
 949	case ACPI_NFIT_TYPE_SYSTEM_ADDRESS:
 950		if (!add_spa(acpi_desc, prev, table))
 951			return err;
 952		break;
 953	case ACPI_NFIT_TYPE_MEMORY_MAP:
 954		if (!add_memdev(acpi_desc, prev, table))
 955			return err;
 956		break;
 957	case ACPI_NFIT_TYPE_CONTROL_REGION:
 958		if (!add_dcr(acpi_desc, prev, table))
 959			return err;
 960		break;
 961	case ACPI_NFIT_TYPE_DATA_REGION:
 962		if (!add_bdw(acpi_desc, prev, table))
 963			return err;
 964		break;
 965	case ACPI_NFIT_TYPE_INTERLEAVE:
 966		if (!add_idt(acpi_desc, prev, table))
 967			return err;
 968		break;
 969	case ACPI_NFIT_TYPE_FLUSH_ADDRESS:
 970		if (!add_flush(acpi_desc, prev, table))
 971			return err;
 972		break;
 973	case ACPI_NFIT_TYPE_SMBIOS:
 974		dev_dbg(dev, "smbios\n");
 975		break;
 976	case ACPI_NFIT_TYPE_CAPABILITIES:
 977		if (!add_platform_cap(acpi_desc, table))
 978			return err;
 979		break;
 980	default:
 981		dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type);
 982		break;
 983	}
 984
 985	return table + hdr->length;
 986}
 987
 988static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc,
 989		struct nfit_mem *nfit_mem)
 990{
 991	u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
 992	u16 dcr = nfit_mem->dcr->region_index;
 993	struct nfit_spa *nfit_spa;
 994
 995	list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
 996		u16 range_index = nfit_spa->spa->range_index;
 997		int type = nfit_spa_type(nfit_spa->spa);
 998		struct nfit_memdev *nfit_memdev;
 999
1000		if (type != NFIT_SPA_BDW)
1001			continue;
1002
1003		list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1004			if (nfit_memdev->memdev->range_index != range_index)
1005				continue;
1006			if (nfit_memdev->memdev->device_handle != device_handle)
1007				continue;
1008			if (nfit_memdev->memdev->region_index != dcr)
1009				continue;
1010
1011			nfit_mem->spa_bdw = nfit_spa->spa;
1012			return;
1013		}
1014	}
1015
1016	dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n",
1017			nfit_mem->spa_dcr->range_index);
1018	nfit_mem->bdw = NULL;
1019}
1020
1021static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc,
1022		struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa)
1023{
1024	u16 dcr = __to_nfit_memdev(nfit_mem)->region_index;
1025	struct nfit_memdev *nfit_memdev;
1026	struct nfit_bdw *nfit_bdw;
1027	struct nfit_idt *nfit_idt;
1028	u16 idt_idx, range_index;
1029
1030	list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) {
1031		if (nfit_bdw->bdw->region_index != dcr)
1032			continue;
1033		nfit_mem->bdw = nfit_bdw->bdw;
1034		break;
1035	}
1036
1037	if (!nfit_mem->bdw)
1038		return;
1039
1040	nfit_mem_find_spa_bdw(acpi_desc, nfit_mem);
1041
1042	if (!nfit_mem->spa_bdw)
1043		return;
1044
1045	range_index = nfit_mem->spa_bdw->range_index;
1046	list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1047		if (nfit_memdev->memdev->range_index != range_index ||
1048				nfit_memdev->memdev->region_index != dcr)
1049			continue;
1050		nfit_mem->memdev_bdw = nfit_memdev->memdev;
1051		idt_idx = nfit_memdev->memdev->interleave_index;
1052		list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
1053			if (nfit_idt->idt->interleave_index != idt_idx)
1054				continue;
1055			nfit_mem->idt_bdw = nfit_idt->idt;
1056			break;
1057		}
1058		break;
1059	}
1060}
1061
1062static int __nfit_mem_init(struct acpi_nfit_desc *acpi_desc,
1063		struct acpi_nfit_system_address *spa)
1064{
1065	struct nfit_mem *nfit_mem, *found;
1066	struct nfit_memdev *nfit_memdev;
1067	int type = spa ? nfit_spa_type(spa) : 0;
1068
1069	switch (type) {
1070	case NFIT_SPA_DCR:
1071	case NFIT_SPA_PM:
1072		break;
1073	default:
1074		if (spa)
1075			return 0;
1076	}
1077
1078	/*
1079	 * This loop runs in two modes, when a dimm is mapped the loop
1080	 * adds memdev associations to an existing dimm, or creates a
1081	 * dimm. In the unmapped dimm case this loop sweeps for memdev
1082	 * instances with an invalid / zero range_index and adds those
1083	 * dimms without spa associations.
1084	 */
1085	list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1086		struct nfit_flush *nfit_flush;
1087		struct nfit_dcr *nfit_dcr;
1088		u32 device_handle;
1089		u16 dcr;
1090
1091		if (spa && nfit_memdev->memdev->range_index != spa->range_index)
1092			continue;
1093		if (!spa && nfit_memdev->memdev->range_index)
1094			continue;
1095		found = NULL;
1096		dcr = nfit_memdev->memdev->region_index;
1097		device_handle = nfit_memdev->memdev->device_handle;
1098		list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
1099			if (__to_nfit_memdev(nfit_mem)->device_handle
1100					== device_handle) {
1101				found = nfit_mem;
1102				break;
1103			}
1104
1105		if (found)
1106			nfit_mem = found;
1107		else {
1108			nfit_mem = devm_kzalloc(acpi_desc->dev,
1109					sizeof(*nfit_mem), GFP_KERNEL);
1110			if (!nfit_mem)
1111				return -ENOMEM;
1112			INIT_LIST_HEAD(&nfit_mem->list);
1113			nfit_mem->acpi_desc = acpi_desc;
1114			list_add(&nfit_mem->list, &acpi_desc->dimms);
1115		}
1116
1117		list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
1118			if (nfit_dcr->dcr->region_index != dcr)
1119				continue;
1120			/*
1121			 * Record the control region for the dimm.  For
1122			 * the ACPI 6.1 case, where there are separate
1123			 * control regions for the pmem vs blk
1124			 * interfaces, be sure to record the extended
1125			 * blk details.
1126			 */
1127			if (!nfit_mem->dcr)
1128				nfit_mem->dcr = nfit_dcr->dcr;
1129			else if (nfit_mem->dcr->windows == 0
1130					&& nfit_dcr->dcr->windows)
1131				nfit_mem->dcr = nfit_dcr->dcr;
1132			break;
1133		}
1134
1135		list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) {
1136			struct acpi_nfit_flush_address *flush;
1137			u16 i;
1138
1139			if (nfit_flush->flush->device_handle != device_handle)
1140				continue;
1141			nfit_mem->nfit_flush = nfit_flush;
1142			flush = nfit_flush->flush;
1143			nfit_mem->flush_wpq = devm_kcalloc(acpi_desc->dev,
1144					flush->hint_count,
1145					sizeof(struct resource),
1146					GFP_KERNEL);
1147			if (!nfit_mem->flush_wpq)
1148				return -ENOMEM;
1149			for (i = 0; i < flush->hint_count; i++) {
1150				struct resource *res = &nfit_mem->flush_wpq[i];
1151
1152				res->start = flush->hint_address[i];
1153				res->end = res->start + 8 - 1;
1154			}
1155			break;
1156		}
1157
1158		if (dcr && !nfit_mem->dcr) {
1159			dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n",
1160					spa->range_index, dcr);
1161			return -ENODEV;
1162		}
1163
1164		if (type == NFIT_SPA_DCR) {
1165			struct nfit_idt *nfit_idt;
1166			u16 idt_idx;
1167
1168			/* multiple dimms may share a SPA when interleaved */
1169			nfit_mem->spa_dcr = spa;
1170			nfit_mem->memdev_dcr = nfit_memdev->memdev;
1171			idt_idx = nfit_memdev->memdev->interleave_index;
1172			list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
1173				if (nfit_idt->idt->interleave_index != idt_idx)
1174					continue;
1175				nfit_mem->idt_dcr = nfit_idt->idt;
1176				break;
1177			}
1178			nfit_mem_init_bdw(acpi_desc, nfit_mem, spa);
1179		} else if (type == NFIT_SPA_PM) {
1180			/*
1181			 * A single dimm may belong to multiple SPA-PM
1182			 * ranges, record at least one in addition to
1183			 * any SPA-DCR range.
1184			 */
1185			nfit_mem->memdev_pmem = nfit_memdev->memdev;
1186		} else
1187			nfit_mem->memdev_dcr = nfit_memdev->memdev;
1188	}
1189
1190	return 0;
1191}
1192
1193static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b)
1194{
1195	struct nfit_mem *a = container_of(_a, typeof(*a), list);
1196	struct nfit_mem *b = container_of(_b, typeof(*b), list);
1197	u32 handleA, handleB;
1198
1199	handleA = __to_nfit_memdev(a)->device_handle;
1200	handleB = __to_nfit_memdev(b)->device_handle;
1201	if (handleA < handleB)
1202		return -1;
1203	else if (handleA > handleB)
1204		return 1;
1205	return 0;
1206}
1207
1208static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc)
1209{
1210	struct nfit_spa *nfit_spa;
1211	int rc;
1212
1213
1214	/*
1215	 * For each SPA-DCR or SPA-PMEM address range find its
1216	 * corresponding MEMDEV(s).  From each MEMDEV find the
1217	 * corresponding DCR.  Then, if we're operating on a SPA-DCR,
1218	 * try to find a SPA-BDW and a corresponding BDW that references
1219	 * the DCR.  Throw it all into an nfit_mem object.  Note, that
1220	 * BDWs are optional.
1221	 */
1222	list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
1223		rc = __nfit_mem_init(acpi_desc, nfit_spa->spa);
1224		if (rc)
1225			return rc;
1226	}
1227
1228	/*
1229	 * If a DIMM has failed to be mapped into SPA there will be no
1230	 * SPA entries above. Find and register all the unmapped DIMMs
1231	 * for reporting and recovery purposes.
1232	 */
1233	rc = __nfit_mem_init(acpi_desc, NULL);
1234	if (rc)
1235		return rc;
1236
1237	list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp);
1238
1239	return 0;
1240}
1241
1242static ssize_t bus_dsm_mask_show(struct device *dev,
1243		struct device_attribute *attr, char *buf)
1244{
1245	struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1246	struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1247	struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1248
1249	return sprintf(buf, "%#lx\n", acpi_desc->bus_dsm_mask);
1250}
1251static struct device_attribute dev_attr_bus_dsm_mask =
1252		__ATTR(dsm_mask, 0444, bus_dsm_mask_show, NULL);
1253
1254static ssize_t revision_show(struct device *dev,
1255		struct device_attribute *attr, char *buf)
1256{
1257	struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1258	struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1259	struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1260
1261	return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision);
1262}
1263static DEVICE_ATTR_RO(revision);
1264
1265static ssize_t hw_error_scrub_show(struct device *dev,
1266		struct device_attribute *attr, char *buf)
1267{
1268	struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1269	struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1270	struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1271
1272	return sprintf(buf, "%d\n", acpi_desc->scrub_mode);
1273}
1274
1275/*
1276 * The 'hw_error_scrub' attribute can have the following values written to it:
1277 * '0': Switch to the default mode where an exception will only insert
1278 *      the address of the memory error into the poison and badblocks lists.
1279 * '1': Enable a full scrub to happen if an exception for a memory error is
1280 *      received.
1281 */
1282static ssize_t hw_error_scrub_store(struct device *dev,
1283		struct device_attribute *attr, const char *buf, size_t size)
1284{
1285	struct nvdimm_bus_descriptor *nd_desc;
1286	ssize_t rc;
1287	long val;
1288
1289	rc = kstrtol(buf, 0, &val);
1290	if (rc)
1291		return rc;
1292
1293	nfit_device_lock(dev);
1294	nd_desc = dev_get_drvdata(dev);
1295	if (nd_desc) {
1296		struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1297
1298		switch (val) {
1299		case HW_ERROR_SCRUB_ON:
1300			acpi_desc->scrub_mode = HW_ERROR_SCRUB_ON;
1301			break;
1302		case HW_ERROR_SCRUB_OFF:
1303			acpi_desc->scrub_mode = HW_ERROR_SCRUB_OFF;
1304			break;
1305		default:
1306			rc = -EINVAL;
1307			break;
1308		}
1309	}
1310	nfit_device_unlock(dev);
1311	if (rc)
1312		return rc;
1313	return size;
1314}
1315static DEVICE_ATTR_RW(hw_error_scrub);
1316
1317/*
1318 * This shows the number of full Address Range Scrubs that have been
1319 * completed since driver load time. Userspace can wait on this using
1320 * select/poll etc. A '+' at the end indicates an ARS is in progress
1321 */
1322static ssize_t scrub_show(struct device *dev,
1323		struct device_attribute *attr, char *buf)
1324{
1325	struct nvdimm_bus_descriptor *nd_desc;
1326	struct acpi_nfit_desc *acpi_desc;
1327	ssize_t rc = -ENXIO;
1328	bool busy;
1329
1330	nfit_device_lock(dev);
1331	nd_desc = dev_get_drvdata(dev);
1332	if (!nd_desc) {
1333		nfit_device_unlock(dev);
1334		return rc;
1335	}
1336	acpi_desc = to_acpi_desc(nd_desc);
1337
1338	mutex_lock(&acpi_desc->init_mutex);
1339	busy = test_bit(ARS_BUSY, &acpi_desc->scrub_flags)
1340		&& !test_bit(ARS_CANCEL, &acpi_desc->scrub_flags);
1341	rc = sprintf(buf, "%d%s", acpi_desc->scrub_count, busy ? "+\n" : "\n");
1342	/* Allow an admin to poll the busy state at a higher rate */
1343	if (busy && capable(CAP_SYS_RAWIO) && !test_and_set_bit(ARS_POLL,
1344				&acpi_desc->scrub_flags)) {
1345		acpi_desc->scrub_tmo = 1;
1346		mod_delayed_work(nfit_wq, &acpi_desc->dwork, HZ);
1347	}
1348
1349	mutex_unlock(&acpi_desc->init_mutex);
1350	nfit_device_unlock(dev);
1351	return rc;
1352}
1353
1354static ssize_t scrub_store(struct device *dev,
1355		struct device_attribute *attr, const char *buf, size_t size)
1356{
1357	struct nvdimm_bus_descriptor *nd_desc;
1358	ssize_t rc;
1359	long val;
1360
1361	rc = kstrtol(buf, 0, &val);
1362	if (rc)
1363		return rc;
1364	if (val != 1)
1365		return -EINVAL;
1366
1367	nfit_device_lock(dev);
1368	nd_desc = dev_get_drvdata(dev);
1369	if (nd_desc) {
1370		struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1371
1372		rc = acpi_nfit_ars_rescan(acpi_desc, ARS_REQ_LONG);
1373	}
1374	nfit_device_unlock(dev);
1375	if (rc)
1376		return rc;
1377	return size;
1378}
1379static DEVICE_ATTR_RW(scrub);
1380
1381static bool ars_supported(struct nvdimm_bus *nvdimm_bus)
1382{
1383	struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1384	const unsigned long mask = 1 << ND_CMD_ARS_CAP | 1 << ND_CMD_ARS_START
1385		| 1 << ND_CMD_ARS_STATUS;
1386
1387	return (nd_desc->cmd_mask & mask) == mask;
1388}
1389
1390static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n)
1391{
1392	struct device *dev = container_of(kobj, struct device, kobj);
1393	struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1394
1395	if (a == &dev_attr_scrub.attr)
1396		return ars_supported(nvdimm_bus) ? a->mode : 0;
1397
1398	if (a == &dev_attr_firmware_activate_noidle.attr)
1399		return intel_fwa_supported(nvdimm_bus) ? a->mode : 0;
1400
1401	return a->mode;
1402}
1403
1404static struct attribute *acpi_nfit_attributes[] = {
1405	&dev_attr_revision.attr,
1406	&dev_attr_scrub.attr,
1407	&dev_attr_hw_error_scrub.attr,
1408	&dev_attr_bus_dsm_mask.attr,
1409	&dev_attr_firmware_activate_noidle.attr,
1410	NULL,
1411};
1412
1413static const struct attribute_group acpi_nfit_attribute_group = {
1414	.name = "nfit",
1415	.attrs = acpi_nfit_attributes,
1416	.is_visible = nfit_visible,
1417};
1418
1419static const struct attribute_group *acpi_nfit_attribute_groups[] = {
1420	&acpi_nfit_attribute_group,
1421	NULL,
1422};
1423
1424static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev)
1425{
1426	struct nvdimm *nvdimm = to_nvdimm(dev);
1427	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1428
1429	return __to_nfit_memdev(nfit_mem);
1430}
1431
1432static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev)
1433{
1434	struct nvdimm *nvdimm = to_nvdimm(dev);
1435	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1436
1437	return nfit_mem->dcr;
1438}
1439
1440static ssize_t handle_show(struct device *dev,
1441		struct device_attribute *attr, char *buf)
1442{
1443	struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
1444
1445	return sprintf(buf, "%#x\n", memdev->device_handle);
1446}
1447static DEVICE_ATTR_RO(handle);
1448
1449static ssize_t phys_id_show(struct device *dev,
1450		struct device_attribute *attr, char *buf)
1451{
1452	struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
1453
1454	return sprintf(buf, "%#x\n", memdev->physical_id);
1455}
1456static DEVICE_ATTR_RO(phys_id);
1457
1458static ssize_t vendor_show(struct device *dev,
1459		struct device_attribute *attr, char *buf)
1460{
1461	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1462
1463	return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->vendor_id));
1464}
1465static DEVICE_ATTR_RO(vendor);
1466
1467static ssize_t rev_id_show(struct device *dev,
1468		struct device_attribute *attr, char *buf)
1469{
1470	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1471
1472	return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->revision_id));
1473}
1474static DEVICE_ATTR_RO(rev_id);
1475
1476static ssize_t device_show(struct device *dev,
1477		struct device_attribute *attr, char *buf)
1478{
1479	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1480
1481	return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->device_id));
1482}
1483static DEVICE_ATTR_RO(device);
1484
1485static ssize_t subsystem_vendor_show(struct device *dev,
1486		struct device_attribute *attr, char *buf)
1487{
1488	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1489
1490	return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_vendor_id));
1491}
1492static DEVICE_ATTR_RO(subsystem_vendor);
1493
1494static ssize_t subsystem_rev_id_show(struct device *dev,
1495		struct device_attribute *attr, char *buf)
1496{
1497	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1498
1499	return sprintf(buf, "0x%04x\n",
1500			be16_to_cpu(dcr->subsystem_revision_id));
1501}
1502static DEVICE_ATTR_RO(subsystem_rev_id);
1503
1504static ssize_t subsystem_device_show(struct device *dev,
1505		struct device_attribute *attr, char *buf)
1506{
1507	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1508
1509	return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_device_id));
1510}
1511static DEVICE_ATTR_RO(subsystem_device);
1512
1513static int num_nvdimm_formats(struct nvdimm *nvdimm)
1514{
1515	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1516	int formats = 0;
1517
1518	if (nfit_mem->memdev_pmem)
1519		formats++;
1520	if (nfit_mem->memdev_bdw)
1521		formats++;
1522	return formats;
1523}
1524
1525static ssize_t format_show(struct device *dev,
1526		struct device_attribute *attr, char *buf)
1527{
1528	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1529
1530	return sprintf(buf, "0x%04x\n", le16_to_cpu(dcr->code));
1531}
1532static DEVICE_ATTR_RO(format);
1533
1534static ssize_t format1_show(struct device *dev,
1535		struct device_attribute *attr, char *buf)
1536{
1537	u32 handle;
1538	ssize_t rc = -ENXIO;
1539	struct nfit_mem *nfit_mem;
1540	struct nfit_memdev *nfit_memdev;
1541	struct acpi_nfit_desc *acpi_desc;
1542	struct nvdimm *nvdimm = to_nvdimm(dev);
1543	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1544
1545	nfit_mem = nvdimm_provider_data(nvdimm);
1546	acpi_desc = nfit_mem->acpi_desc;
1547	handle = to_nfit_memdev(dev)->device_handle;
1548
1549	/* assumes DIMMs have at most 2 published interface codes */
1550	mutex_lock(&acpi_desc->init_mutex);
1551	list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1552		struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
1553		struct nfit_dcr *nfit_dcr;
1554
1555		if (memdev->device_handle != handle)
1556			continue;
1557
1558		list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
1559			if (nfit_dcr->dcr->region_index != memdev->region_index)
1560				continue;
1561			if (nfit_dcr->dcr->code == dcr->code)
1562				continue;
1563			rc = sprintf(buf, "0x%04x\n",
1564					le16_to_cpu(nfit_dcr->dcr->code));
1565			break;
1566		}
1567		if (rc != ENXIO)
1568			break;
1569	}
1570	mutex_unlock(&acpi_desc->init_mutex);
1571	return rc;
1572}
1573static DEVICE_ATTR_RO(format1);
1574
1575static ssize_t formats_show(struct device *dev,
1576		struct device_attribute *attr, char *buf)
1577{
1578	struct nvdimm *nvdimm = to_nvdimm(dev);
1579
1580	return sprintf(buf, "%d\n", num_nvdimm_formats(nvdimm));
1581}
1582static DEVICE_ATTR_RO(formats);
1583
1584static ssize_t serial_show(struct device *dev,
1585		struct device_attribute *attr, char *buf)
1586{
1587	struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1588
1589	return sprintf(buf, "0x%08x\n", be32_to_cpu(dcr->serial_number));
1590}
1591static DEVICE_ATTR_RO(serial);
1592
1593static ssize_t family_show(struct device *dev,
1594		struct device_attribute *attr, char *buf)
1595{
1596	struct nvdimm *nvdimm = to_nvdimm(dev);
1597	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1598
1599	if (nfit_mem->family < 0)
1600		return -ENXIO;
1601	return sprintf(buf, "%d\n", nfit_mem->family);
1602}
1603static DEVICE_ATTR_RO(family);
1604
1605static ssize_t dsm_mask_show(struct device *dev,
1606		struct device_attribute *attr, char *buf)
1607{
1608	struct nvdimm *nvdimm = to_nvdimm(dev);
1609	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1610
1611	if (nfit_mem->family < 0)
1612		return -ENXIO;
1613	return sprintf(buf, "%#lx\n", nfit_mem->dsm_mask);
1614}
1615static DEVICE_ATTR_RO(dsm_mask);
1616
1617static ssize_t flags_show(struct device *dev,
1618		struct device_attribute *attr, char *buf)
1619{
1620	struct nvdimm *nvdimm = to_nvdimm(dev);
1621	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1622	u16 flags = __to_nfit_memdev(nfit_mem)->flags;
1623
1624	if (test_bit(NFIT_MEM_DIRTY, &nfit_mem->flags))
1625		flags |= ACPI_NFIT_MEM_FLUSH_FAILED;
1626
1627	return sprintf(buf, "%s%s%s%s%s%s%s\n",
1628		flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "",
1629		flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "",
1630		flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "",
1631		flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "",
1632		flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "",
1633		flags & ACPI_NFIT_MEM_MAP_FAILED ? "map_fail " : "",
1634		flags & ACPI_NFIT_MEM_HEALTH_ENABLED ? "smart_notify " : "");
1635}
1636static DEVICE_ATTR_RO(flags);
1637
1638static ssize_t id_show(struct device *dev,
1639		struct device_attribute *attr, char *buf)
1640{
1641	struct nvdimm *nvdimm = to_nvdimm(dev);
1642	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1643
1644	return sprintf(buf, "%s\n", nfit_mem->id);
1645}
1646static DEVICE_ATTR_RO(id);
1647
1648static ssize_t dirty_shutdown_show(struct device *dev,
1649		struct device_attribute *attr, char *buf)
1650{
1651	struct nvdimm *nvdimm = to_nvdimm(dev);
1652	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1653
1654	return sprintf(buf, "%d\n", nfit_mem->dirty_shutdown);
1655}
1656static DEVICE_ATTR_RO(dirty_shutdown);
1657
1658static struct attribute *acpi_nfit_dimm_attributes[] = {
1659	&dev_attr_handle.attr,
1660	&dev_attr_phys_id.attr,
1661	&dev_attr_vendor.attr,
1662	&dev_attr_device.attr,
1663	&dev_attr_rev_id.attr,
1664	&dev_attr_subsystem_vendor.attr,
1665	&dev_attr_subsystem_device.attr,
1666	&dev_attr_subsystem_rev_id.attr,
1667	&dev_attr_format.attr,
1668	&dev_attr_formats.attr,
1669	&dev_attr_format1.attr,
1670	&dev_attr_serial.attr,
1671	&dev_attr_flags.attr,
1672	&dev_attr_id.attr,
1673	&dev_attr_family.attr,
1674	&dev_attr_dsm_mask.attr,
1675	&dev_attr_dirty_shutdown.attr,
1676	NULL,
1677};
1678
1679static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj,
1680		struct attribute *a, int n)
1681{
1682	struct device *dev = container_of(kobj, struct device, kobj);
1683	struct nvdimm *nvdimm = to_nvdimm(dev);
1684	struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1685
1686	if (!to_nfit_dcr(dev)) {
1687		/* Without a dcr only the memdev attributes can be surfaced */
1688		if (a == &dev_attr_handle.attr || a == &dev_attr_phys_id.attr
1689				|| a == &dev_attr_flags.attr
1690				|| a == &dev_attr_family.attr
1691				|| a == &dev_attr_dsm_mask.attr)
1692			return a->mode;
1693		return 0;
1694	}
1695
1696	if (a == &dev_attr_format1.attr && num_nvdimm_formats(nvdimm) <= 1)
1697		return 0;
1698
1699	if (!test_bit(NFIT_MEM_DIRTY_COUNT, &nfit_mem->flags)
1700			&& a == &dev_attr_dirty_shutdown.attr)
1701		return 0;
1702
1703	return a->mode;
1704}
1705
1706static const struct attribute_group acpi_nfit_dimm_attribute_group = {
1707	.name = "nfit",
1708	.attrs = acpi_nfit_dimm_attributes,
1709	.is_visible = acpi_nfit_dimm_attr_visible,
1710};
1711
1712static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = {
1713	&acpi_nfit_dimm_attribute_group,
1714	NULL,
1715};
1716
1717static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc,
1718		u32 device_handle)
1719{
1720	struct nfit_mem *nfit_mem;
1721
1722	list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
1723		if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle)
1724			return nfit_mem->nvdimm;
1725
1726	return NULL;
1727}
1728
1729void __acpi_nvdimm_notify(struct device *dev, u32 event)
1730{
1731	struct nfit_mem *nfit_mem;
1732	struct acpi_nfit_desc *acpi_desc;
1733
1734	dev_dbg(dev->parent, "%s: event: %d\n", dev_name(dev),
1735			event);
1736
1737	if (event != NFIT_NOTIFY_DIMM_HEALTH) {
1738		dev_dbg(dev->parent, "%s: unknown event: %d\n", dev_name(dev),
1739				event);
1740		return;
1741	}
1742
1743	acpi_desc = dev_get_drvdata(dev->parent);
1744	if (!acpi_desc)
1745		return;
1746
1747	/*
1748	 * If we successfully retrieved acpi_desc, then we know nfit_mem data
1749	 * is still valid.
1750	 */
1751	nfit_mem = dev_get_drvdata(dev);
1752	if (nfit_mem && nfit_mem->flags_attr)
1753		sysfs_notify_dirent(nfit_mem->flags_attr);
1754}
1755EXPORT_SYMBOL_GPL(__acpi_nvdimm_notify);
1756
1757static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data)
1758{
1759	struct acpi_device *adev = data;
1760	struct device *dev = &adev->dev;
1761
1762	nfit_device_lock(dev->parent);
1763	__acpi_nvdimm_notify(dev, event);
1764	nfit_device_unlock(dev->parent);
1765}
1766
1767static bool acpi_nvdimm_has_method(struct acpi_device *adev, char *method)
1768{
1769	acpi_handle handle;
1770	acpi_status status;
1771
1772	status = acpi_get_handle(adev->handle, method, &handle);
1773
1774	if (ACPI_SUCCESS(status))
1775		return true;
1776	return false;
1777}
1778
1779__weak void nfit_intel_shutdown_status(struct nfit_mem *nfit_mem)
1780{
1781	struct device *dev = &nfit_mem->adev->dev;
1782	struct nd_intel_smart smart = { 0 };
1783	union acpi_object in_buf = {
1784		.buffer.type = ACPI_TYPE_BUFFER,
1785		.buffer.length = 0,
1786	};
1787	union acpi_object in_obj = {
1788		.package.type = ACPI_TYPE_PACKAGE,
1789		.package.count = 1,
1790		.package.elements = &in_buf,
1791	};
1792	const u8 func = ND_INTEL_SMART;
1793	const guid_t *guid = to_nfit_uuid(nfit_mem->family);
1794	u8 revid = nfit_dsm_revid(nfit_mem->family, func);
1795	struct acpi_device *adev = nfit_mem->adev;
1796	acpi_handle handle = adev->handle;
1797	union acpi_object *out_obj;
1798
1799	if ((nfit_mem->dsm_mask & (1 << func)) == 0)
1800		return;
1801
1802	out_obj = acpi_evaluate_dsm(handle, guid, revid, func, &in_obj);
1803	if (!out_obj || out_obj->type != ACPI_TYPE_BUFFER
1804			|| out_obj->buffer.length < sizeof(smart)) {
1805		dev_dbg(dev->parent, "%s: failed to retrieve initial health\n",
1806				dev_name(dev));
1807		ACPI_FREE(out_obj);
1808		return;
1809	}
1810	memcpy(&smart, out_obj->buffer.pointer, sizeof(smart));
1811	ACPI_FREE(out_obj);
1812
1813	if (smart.flags & ND_INTEL_SMART_SHUTDOWN_VALID) {
1814		if (smart.shutdown_state)
1815			set_bit(NFIT_MEM_DIRTY, &nfit_mem->flags);
1816	}
1817
1818	if (smart.flags & ND_INTEL_SMART_SHUTDOWN_COUNT_VALID) {
1819		set_bit(NFIT_MEM_DIRTY_COUNT, &nfit_mem->flags);
1820		nfit_mem->dirty_shutdown = smart.shutdown_count;
1821	}
1822}
1823
1824static void populate_shutdown_status(struct nfit_mem *nfit_mem)
1825{
1826	/*
1827	 * For DIMMs that provide a dynamic facility to retrieve a
1828	 * dirty-shutdown status and/or a dirty-shutdown count, cache
1829	 * these values in nfit_mem.
1830	 */
1831	if (nfit_mem->family == NVDIMM_FAMILY_INTEL)
1832		nfit_intel_shutdown_status(nfit_mem);
1833}
1834
1835static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
1836		struct nfit_mem *nfit_mem, u32 device_handle)
1837{
1838	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1839	struct acpi_device *adev, *adev_dimm;
1840	struct device *dev = acpi_desc->dev;
1841	unsigned long dsm_mask, label_mask;
1842	const guid_t *guid;
1843	int i;
1844	int family = -1;
1845	struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
1846
1847	/* nfit test assumes 1:1 relationship between commands and dsms */
1848	nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en;
1849	nfit_mem->family = NVDIMM_FAMILY_INTEL;
1850	set_bit(NVDIMM_FAMILY_INTEL, &nd_desc->dimm_family_mask);
1851
1852	if (dcr->valid_fields & ACPI_NFIT_CONTROL_MFG_INFO_VALID)
1853		sprintf(nfit_mem->id, "%04x-%02x-%04x-%08x",
1854				be16_to_cpu(dcr->vendor_id),
1855				dcr->manufacturing_location,
1856				be16_to_cpu(dcr->manufacturing_date),
1857				be32_to_cpu(dcr->serial_number));
1858	else
1859		sprintf(nfit_mem->id, "%04x-%08x",
1860				be16_to_cpu(dcr->vendor_id),
1861				be32_to_cpu(dcr->serial_number));
1862
1863	adev = to_acpi_dev(acpi_desc);
1864	if (!adev) {
1865		/* unit test case */
1866		populate_shutdown_status(nfit_mem);
1867		return 0;
1868	}
1869
1870	adev_dimm = acpi_find_child_device(adev, device_handle, false);
1871	nfit_mem->adev = adev_dimm;
1872	if (!adev_dimm) {
1873		dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n",
1874				device_handle);
1875		return force_enable_dimms ? 0 : -ENODEV;
1876	}
1877
1878	if (ACPI_FAILURE(acpi_install_notify_handler(adev_dimm->handle,
1879		ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify, adev_dimm))) {
1880		dev_err(dev, "%s: notification registration failed\n",
1881				dev_name(&adev_dimm->dev));
1882		return -ENXIO;
1883	}
1884	/*
1885	 * Record nfit_mem for the notification path to track back to
1886	 * the nfit sysfs attributes for this dimm device object.
1887	 */
1888	dev_set_drvdata(&adev_dimm->dev, nfit_mem);
1889
1890	/*
1891	 * There are 4 "legacy" NVDIMM command sets
1892	 * (NVDIMM_FAMILY_{INTEL,MSFT,HPE1,HPE2}) that were created before
1893	 * an EFI working group was established to constrain this
1894	 * proliferation. The nfit driver probes for the supported command
1895	 * set by GUID. Note, if you're a platform developer looking to add
1896	 * a new command set to this probe, consider using an existing set,
1897	 * or otherwise seek approval to publish the command set at
1898	 * http://www.uefi.org/RFIC_LIST.
1899	 *
1900	 * Note, that checking for function0 (bit0) tells us if any commands
1901	 * are reachable through this GUID.
1902	 */
1903	clear_bit(NVDIMM_FAMILY_INTEL, &nd_desc->dimm_family_mask);
1904	for (i = 0; i <= NVDIMM_FAMILY_MAX; i++)
1905		if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1)) {
1906			set_bit(i, &nd_desc->dimm_family_mask);
1907			if (family < 0 || i == default_dsm_family)
1908				family = i;
1909		}
1910
1911	/* limit the supported commands to those that are publicly documented */
1912	nfit_mem->family = family;
1913	if (override_dsm_mask && !disable_vendor_specific)
1914		dsm_mask = override_dsm_mask;
1915	else if (nfit_mem->family == NVDIMM_FAMILY_INTEL) {
1916		dsm_mask = NVDIMM_INTEL_CMDMASK;
1917		if (disable_vendor_specific)
1918			dsm_mask &= ~(1 << ND_CMD_VENDOR);
1919	} else if (nfit_mem->family == NVDIMM_FAMILY_HPE1) {
1920		dsm_mask = 0x1c3c76;
1921	} else if (nfit_mem->family == NVDIMM_FAMILY_HPE2) {
1922		dsm_mask = 0x1fe;
1923		if (disable_vendor_specific)
1924			dsm_mask &= ~(1 << 8);
1925	} else if (nfit_mem->family == NVDIMM_FAMILY_MSFT) {
1926		dsm_mask = 0xffffffff;
1927	} else if (nfit_mem->family == NVDIMM_FAMILY_HYPERV) {
1928		dsm_mask = 0x1f;
1929	} else {
1930		dev_dbg(dev, "unknown dimm command family\n");
1931		nfit_mem->family = -1;
1932		/* DSMs are optional, continue loading the driver... */
1933		return 0;
1934	}
1935
1936	/*
1937	 * Function 0 is the command interrogation function, don't
1938	 * export it to potential userspace use, and enable it to be
1939	 * used as an error value in acpi_nfit_ctl().
1940	 */
1941	dsm_mask &= ~1UL;
1942
1943	guid = to_nfit_uuid(nfit_mem->family);
1944	for_each_set_bit(i, &dsm_mask, BITS_PER_LONG)
1945		if (acpi_check_dsm(adev_dimm->handle, guid,
1946					nfit_dsm_revid(nfit_mem->family, i),
1947					1ULL << i))
1948			set_bit(i, &nfit_mem->dsm_mask);
1949
1950	/*
1951	 * Prefer the NVDIMM_FAMILY_INTEL label read commands if present
1952	 * due to their better semantics handling locked capacity.
1953	 */
1954	label_mask = 1 << ND_CMD_GET_CONFIG_SIZE | 1 << ND_CMD_GET_CONFIG_DATA
1955		| 1 << ND_CMD_SET_CONFIG_DATA;
1956	if (family == NVDIMM_FAMILY_INTEL
1957			&& (dsm_mask & label_mask) == label_mask)
1958		/* skip _LS{I,R,W} enabling */;
1959	else {
1960		if (acpi_nvdimm_has_method(adev_dimm, "_LSI")
1961				&& acpi_nvdimm_has_method(adev_dimm, "_LSR")) {
1962			dev_dbg(dev, "%s: has _LSR\n", dev_name(&adev_dimm->dev));
1963			set_bit(NFIT_MEM_LSR, &nfit_mem->flags);
1964		}
1965
1966		if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags)
1967				&& acpi_nvdimm_has_method(adev_dimm, "_LSW")) {
1968			dev_dbg(dev, "%s: has _LSW\n", dev_name(&adev_dimm->dev));
1969			set_bit(NFIT_MEM_LSW, &nfit_mem->flags);
1970		}
1971
1972		/*
1973		 * Quirk read-only label configurations to preserve
1974		 * access to label-less namespaces by default.
1975		 */
1976		if (!test_bit(NFIT_MEM_LSW, &nfit_mem->flags)
1977				&& !force_labels) {
1978			dev_dbg(dev, "%s: No _LSW, disable labels\n",
1979					dev_name(&adev_dimm->dev));
1980			clear_bit(NFIT_MEM_LSR, &nfit_mem->flags);
1981		} else
1982			dev_dbg(dev, "%s: Force enable labels\n",
1983					dev_name(&adev_dimm->dev));
1984	}
1985
1986	populate_shutdown_status(nfit_mem);
1987
1988	return 0;
1989}
1990
1991static void shutdown_dimm_notify(void *data)
1992{
1993	struct acpi_nfit_desc *acpi_desc = data;
1994	struct nfit_mem *nfit_mem;
1995
1996	mutex_lock(&acpi_desc->init_mutex);
1997	/*
1998	 * Clear out the nfit_mem->flags_attr and shut down dimm event
1999	 * notifications.
2000	 */
2001	list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
2002		struct acpi_device *adev_dimm = nfit_mem->adev;
2003
2004		if (nfit_mem->flags_attr) {
2005			sysfs_put(nfit_mem->flags_attr);
2006			nfit_mem->flags_attr = NULL;
2007		}
2008		if (adev_dimm) {
2009			acpi_remove_notify_handler(adev_dimm->handle,
2010					ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify);
2011			dev_set_drvdata(&adev_dimm->dev, NULL);
2012		}
2013	}
2014	mutex_unlock(&acpi_desc->init_mutex);
2015}
2016
2017static const struct nvdimm_security_ops *acpi_nfit_get_security_ops(int family)
2018{
2019	switch (family) {
2020	case NVDIMM_FAMILY_INTEL:
2021		return intel_security_ops;
2022	default:
2023		return NULL;
2024	}
2025}
2026
2027static const struct nvdimm_fw_ops *acpi_nfit_get_fw_ops(
2028		struct nfit_mem *nfit_mem)
2029{
2030	unsigned long mask;
2031	struct acpi_nfit_desc *acpi_desc = nfit_mem->acpi_desc;
2032	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2033
2034	if (!nd_desc->fw_ops)
2035		return NULL;
2036
2037	if (nfit_mem->family != NVDIMM_FAMILY_INTEL)
2038		return NULL;
2039
2040	mask = nfit_mem->dsm_mask & NVDIMM_INTEL_FW_ACTIVATE_CMDMASK;
2041	if (mask != NVDIMM_INTEL_FW_ACTIVATE_CMDMASK)
2042		return NULL;
2043
2044	return intel_fw_ops;
2045}
2046
2047static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
2048{
2049	struct nfit_mem *nfit_mem;
2050	int dimm_count = 0, rc;
2051	struct nvdimm *nvdimm;
2052
2053	list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
2054		struct acpi_nfit_flush_address *flush;
2055		unsigned long flags = 0, cmd_mask;
2056		struct nfit_memdev *nfit_memdev;
2057		u32 device_handle;
2058		u16 mem_flags;
2059
2060		device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
2061		nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle);
2062		if (nvdimm) {
2063			dimm_count++;
2064			continue;
2065		}
2066
2067		if (nfit_mem->bdw && nfit_mem->memdev_pmem) {
2068			set_bit(NDD_ALIASING, &flags);
2069			set_bit(NDD_LABELING, &flags);
2070		}
2071
2072		/* collate flags across all memdevs for this dimm */
2073		list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
2074			struct acpi_nfit_memory_map *dimm_memdev;
2075
2076			dimm_memdev = __to_nfit_memdev(nfit_mem);
2077			if (dimm_memdev->device_handle
2078					!= nfit_memdev->memdev->device_handle)
2079				continue;
2080			dimm_memdev->flags |= nfit_memdev->memdev->flags;
2081		}
2082
2083		mem_flags = __to_nfit_memdev(nfit_mem)->flags;
2084		if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED)
2085			set_bit(NDD_UNARMED, &flags);
2086
2087		rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle);
2088		if (rc)
2089			continue;
2090
2091		/*
2092		 * TODO: provide translation for non-NVDIMM_FAMILY_INTEL
2093		 * devices (i.e. from nd_cmd to acpi_dsm) to standardize the
2094		 * userspace interface.
2095		 */
2096		cmd_mask = 1UL << ND_CMD_CALL;
2097		if (nfit_mem->family == NVDIMM_FAMILY_INTEL) {
2098			/*
2099			 * These commands have a 1:1 correspondence
2100			 * between DSM payload and libnvdimm ioctl
2101			 * payload format.
2102			 */
2103			cmd_mask |= nfit_mem->dsm_mask & NVDIMM_STANDARD_CMDMASK;
2104		}
2105
2106		/* Quirk to ignore LOCAL for labels on HYPERV DIMMs */
2107		if (nfit_mem->family == NVDIMM_FAMILY_HYPERV)
2108			set_bit(NDD_NOBLK, &flags);
2109
2110		if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags)) {
2111			set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
2112			set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
2113		}
2114		if (test_bit(NFIT_MEM_LSW, &nfit_mem->flags))
2115			set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
2116
2117		flush = nfit_mem->nfit_flush ? nfit_mem->nfit_flush->flush
2118			: NULL;
2119		nvdimm = __nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem,
2120				acpi_nfit_dimm_attribute_groups,
2121				flags, cmd_mask, flush ? flush->hint_count : 0,
2122				nfit_mem->flush_wpq, &nfit_mem->id[0],
2123				acpi_nfit_get_security_ops(nfit_mem->family),
2124				acpi_nfit_get_fw_ops(nfit_mem));
2125		if (!nvdimm)
2126			return -ENOMEM;
2127
2128		nfit_mem->nvdimm = nvdimm;
2129		dimm_count++;
2130
2131		if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0)
2132			continue;
2133
2134		dev_err(acpi_desc->dev, "Error found in NVDIMM %s flags:%s%s%s%s%s\n",
2135				nvdimm_name(nvdimm),
2136		  mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "",
2137		  mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"",
2138		  mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "",
2139		  mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "",
2140		  mem_flags & ACPI_NFIT_MEM_MAP_FAILED ? " map_fail" : "");
2141
2142	}
2143
2144	rc = nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count);
2145	if (rc)
2146		return rc;
2147
2148	/*
2149	 * Now that dimms are successfully registered, and async registration
2150	 * is flushed, attempt to enable event notification.
2151	 */
2152	list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
2153		struct kernfs_node *nfit_kernfs;
2154
2155		nvdimm = nfit_mem->nvdimm;
2156		if (!nvdimm)
2157			continue;
2158
2159		nfit_kernfs = sysfs_get_dirent(nvdimm_kobj(nvdimm)->sd, "nfit");
2160		if (nfit_kernfs)
2161			nfit_mem->flags_attr = sysfs_get_dirent(nfit_kernfs,
2162					"flags");
2163		sysfs_put(nfit_kernfs);
2164		if (!nfit_mem->flags_attr)
2165			dev_warn(acpi_desc->dev, "%s: notifications disabled\n",
2166					nvdimm_name(nvdimm));
2167	}
2168
2169	return devm_add_action_or_reset(acpi_desc->dev, shutdown_dimm_notify,
2170			acpi_desc);
2171}
2172
2173/*
2174 * These constants are private because there are no kernel consumers of
2175 * these commands.
2176 */
2177enum nfit_aux_cmds {
2178        NFIT_CMD_TRANSLATE_SPA = 5,
2179        NFIT_CMD_ARS_INJECT_SET = 7,
2180        NFIT_CMD_ARS_INJECT_CLEAR = 8,
2181        NFIT_CMD_ARS_INJECT_GET = 9,
2182};
2183
2184static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc)
2185{
2186	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2187	const guid_t *guid = to_nfit_uuid(NFIT_DEV_BUS);
2188	unsigned long dsm_mask, *mask;
2189	struct acpi_device *adev;
2190	int i;
2191
2192	set_bit(ND_CMD_CALL, &nd_desc->cmd_mask);
2193	set_bit(NVDIMM_BUS_FAMILY_NFIT, &nd_desc->bus_family_mask);
2194
2195	/* enable nfit_test to inject bus command emulation */
2196	if (acpi_desc->bus_cmd_force_en) {
2197		nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en;
2198		mask = &nd_desc->bus_family_mask;
2199		if (acpi_desc->family_dsm_mask[NVDIMM_BUS_FAMILY_INTEL]) {
2200			set_bit(NVDIMM_BUS_FAMILY_INTEL, mask);
2201			nd_desc->fw_ops = intel_bus_fw_ops;
2202		}
2203	}
2204
2205	adev = to_acpi_dev(acpi_desc);
2206	if (!adev)
2207		return;
2208
2209	for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++)
2210		if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i))
2211			set_bit(i, &nd_desc->cmd_mask);
2212
2213	dsm_mask =
2214		(1 << ND_CMD_ARS_CAP) |
2215		(1 << ND_CMD_ARS_START) |
2216		(1 << ND_CMD_ARS_STATUS) |
2217		(1 << ND_CMD_CLEAR_ERROR) |
2218		(1 << NFIT_CMD_TRANSLATE_SPA) |
2219		(1 << NFIT_CMD_ARS_INJECT_SET) |
2220		(1 << NFIT_CMD_ARS_INJECT_CLEAR) |
2221		(1 << NFIT_CMD_ARS_INJECT_GET);
2222	for_each_set_bit(i, &dsm_mask, BITS_PER_LONG)
2223		if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i))
2224			set_bit(i, &acpi_desc->bus_dsm_mask);
2225
2226	/* Enumerate allowed NVDIMM_BUS_FAMILY_INTEL commands */
2227	dsm_mask = NVDIMM_BUS_INTEL_FW_ACTIVATE_CMDMASK;
2228	guid = to_nfit_bus_uuid(NVDIMM_BUS_FAMILY_INTEL);
2229	mask = &acpi_desc->family_dsm_mask[NVDIMM_BUS_FAMILY_INTEL];
2230	for_each_set_bit(i, &dsm_mask, BITS_PER_LONG)
2231		if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i))
2232			set_bit(i, mask);
2233
2234	if (*mask == dsm_mask) {
2235		set_bit(NVDIMM_BUS_FAMILY_INTEL, &nd_desc->bus_family_mask);
2236		nd_desc->fw_ops = intel_bus_fw_ops;
2237	}
2238}
2239
2240static ssize_t range_index_show(struct device *dev,
2241		struct device_attribute *attr, char *buf)
2242{
2243	struct nd_region *nd_region = to_nd_region(dev);
2244	struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region);
2245
2246	return sprintf(buf, "%d\n", nfit_spa->spa->range_index);
2247}
2248static DEVICE_ATTR_RO(range_index);
2249
2250static struct attribute *acpi_nfit_region_attributes[] = {
2251	&dev_attr_range_index.attr,
2252	NULL,
2253};
2254
2255static const struct attribute_group acpi_nfit_region_attribute_group = {
2256	.name = "nfit",
2257	.attrs = acpi_nfit_region_attributes,
2258};
2259
2260static const struct attribute_group *acpi_nfit_region_attribute_groups[] = {
2261	&acpi_nfit_region_attribute_group,
2262	NULL,
2263};
2264
2265/* enough info to uniquely specify an interleave set */
2266struct nfit_set_info {
2267	struct nfit_set_info_map {
2268		u64 region_offset;
2269		u32 serial_number;
2270		u32 pad;
2271	} mapping[0];
2272};
2273
2274struct nfit_set_info2 {
2275	struct nfit_set_info_map2 {
2276		u64 region_offset;
2277		u32 serial_number;
2278		u16 vendor_id;
2279		u16 manufacturing_date;
2280		u8  manufacturing_location;
2281		u8  reserved[31];
2282	} mapping[0];
2283};
2284
2285static size_t sizeof_nfit_set_info(int num_mappings)
2286{
2287	return sizeof(struct nfit_set_info)
2288		+ num_mappings * sizeof(struct nfit_set_info_map);
2289}
2290
2291static size_t sizeof_nfit_set_info2(int num_mappings)
2292{
2293	return sizeof(struct nfit_set_info2)
2294		+ num_mappings * sizeof(struct nfit_set_info_map2);
2295}
2296
2297static int cmp_map_compat(const void *m0, const void *m1)
2298{
2299	const struct nfit_set_info_map *map0 = m0;
2300	const struct nfit_set_info_map *map1 = m1;
2301
2302	return memcmp(&map0->region_offset, &map1->region_offset,
2303			sizeof(u64));
2304}
2305
2306static int cmp_map(const void *m0, const void *m1)
2307{
2308	const struct nfit_set_info_map *map0 = m0;
2309	const struct nfit_set_info_map *map1 = m1;
2310
2311	if (map0->region_offset < map1->region_offset)
2312		return -1;
2313	else if (map0->region_offset > map1->region_offset)
2314		return 1;
2315	return 0;
2316}
2317
2318static int cmp_map2(const void *m0, const void *m1)
2319{
2320	const struct nfit_set_info_map2 *map0 = m0;
2321	const struct nfit_set_info_map2 *map1 = m1;
2322
2323	if (map0->region_offset < map1->region_offset)
2324		return -1;
2325	else if (map0->region_offset > map1->region_offset)
2326		return 1;
2327	return 0;
2328}
2329
2330/* Retrieve the nth entry referencing this spa */
2331static struct acpi_nfit_memory_map *memdev_from_spa(
2332		struct acpi_nfit_desc *acpi_desc, u16 range_index, int n)
2333{
2334	struct nfit_memdev *nfit_memdev;
2335
2336	list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list)
2337		if (nfit_memdev->memdev->range_index == range_index)
2338			if (n-- == 0)
2339				return nfit_memdev->memdev;
2340	return NULL;
2341}
2342
2343static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
2344		struct nd_region_desc *ndr_desc,
2345		struct acpi_nfit_system_address *spa)
2346{
2347	struct device *dev = acpi_desc->dev;
2348	struct nd_interleave_set *nd_set;
2349	u16 nr = ndr_desc->num_mappings;
2350	struct nfit_set_info2 *info2;
2351	struct nfit_set_info *info;
2352	int i;
2353
2354	nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
2355	if (!nd_set)
2356		return -ENOMEM;
2357	import_guid(&nd_set->type_guid, spa->range_guid);
2358
2359	info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL);
2360	if (!info)
2361		return -ENOMEM;
2362
2363	info2 = devm_kzalloc(dev, sizeof_nfit_set_info2(nr), GFP_KERNEL);
2364	if (!info2)
2365		return -ENOMEM;
2366
2367	for (i = 0; i < nr; i++) {
2368		struct nd_mapping_desc *mapping = &ndr_desc->mapping[i];
2369		struct nfit_set_info_map *map = &info->mapping[i];
2370		struct nfit_set_info_map2 *map2 = &info2->mapping[i];
2371		struct nvdimm *nvdimm = mapping->nvdimm;
2372		struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
2373		struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc,
2374				spa->range_index, i);
2375		struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
2376
2377		if (!memdev || !nfit_mem->dcr) {
2378			dev_err(dev, "%s: failed to find DCR\n", __func__);
2379			return -ENODEV;
2380		}
2381
2382		map->region_offset = memdev->region_offset;
2383		map->serial_number = dcr->serial_number;
2384
2385		map2->region_offset = memdev->region_offset;
2386		map2->serial_number = dcr->serial_number;
2387		map2->vendor_id = dcr->vendor_id;
2388		map2->manufacturing_date = dcr->manufacturing_date;
2389		map2->manufacturing_location = dcr->manufacturing_location;
2390	}
2391
2392	/* v1.1 namespaces */
2393	sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
2394			cmp_map, NULL);
2395	nd_set->cookie1 = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
2396
2397	/* v1.2 namespaces */
2398	sort(&info2->mapping[0], nr, sizeof(struct nfit_set_info_map2),
2399			cmp_map2, NULL);
2400	nd_set->cookie2 = nd_fletcher64(info2, sizeof_nfit_set_info2(nr), 0);
2401
2402	/* support v1.1 namespaces created with the wrong sort order */
2403	sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
2404			cmp_map_compat, NULL);
2405	nd_set->altcookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
2406
2407	/* record the result of the sort for the mapping position */
2408	for (i = 0; i < nr; i++) {
2409		struct nfit_set_info_map2 *map2 = &info2->mapping[i];
2410		int j;
2411
2412		for (j = 0; j < nr; j++) {
2413			struct nd_mapping_desc *mapping = &ndr_desc->mapping[j];
2414			struct nvdimm *nvdimm = mapping->nvdimm;
2415			struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
2416			struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
2417
2418			if (map2->serial_number == dcr->serial_number &&
2419			    map2->vendor_id == dcr->vendor_id &&
2420			    map2->manufacturing_date == dcr->manufacturing_date &&
2421			    map2->manufacturing_location
2422				    == dcr->manufacturing_location) {
2423				mapping->position = i;
2424				break;
2425			}
2426		}
2427	}
2428
2429	ndr_desc->nd_set = nd_set;
2430	devm_kfree(dev, info);
2431	devm_kfree(dev, info2);
2432
2433	return 0;
2434}
2435
2436static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio)
2437{
2438	struct acpi_nfit_interleave *idt = mmio->idt;
2439	u32 sub_line_offset, line_index, line_offset;
2440	u64 line_no, table_skip_count, table_offset;
2441
2442	line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset);
2443	table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index);
2444	line_offset = idt->line_offset[line_index]
2445		* mmio->line_size;
2446	table_offset = table_skip_count * mmio->table_size;
2447
2448	return mmio->base_offset + line_offset + table_offset + sub_line_offset;
2449}
2450
2451static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw)
2452{
2453	struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
2454	u64 offset = nfit_blk->stat_offset + mmio->size * bw;
2455	const u32 STATUS_MASK = 0x80000037;
2456
2457	if (mmio->num_lines)
2458		offset = to_interleave_offset(offset, mmio);
2459
2460	return readl(mmio->addr.base + offset) & STATUS_MASK;
2461}
2462
2463static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw,
2464		resource_size_t dpa, unsigned int len, unsigned int write)
2465{
2466	u64 cmd, offset;
2467	struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
2468
2469	enum {
2470		BCW_OFFSET_MASK = (1ULL << 48)-1,
2471		BCW_LEN_SHIFT = 48,
2472		BCW_LEN_MASK = (1ULL << 8) - 1,
2473		BCW_CMD_SHIFT = 56,
2474	};
2475
2476	cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK;
2477	len = len >> L1_CACHE_SHIFT;
2478	cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT;
2479	cmd |= ((u64) write) << BCW_CMD_SHIFT;
2480
2481	offset = nfit_blk->cmd_offset + mmio->size * bw;
2482	if (mmio->num_lines)
2483		offset = to_interleave_offset(offset, mmio);
2484
2485	writeq(cmd, mmio->addr.base + offset);
2486	nvdimm_flush(nfit_blk->nd_region, NULL);
2487
2488	if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH)
2489		readq(mmio->addr.base + offset);
2490}
2491
2492static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,
2493		resource_size_t dpa, void *iobuf, size_t len, int rw,
2494		unsigned int lane)
2495{
2496	struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
2497	unsigned int copied = 0;
2498	u64 base_offset;
2499	int rc;
2500
2501	base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES
2502		+ lane * mmio->size;
2503	write_blk_ctl(nfit_blk, lane, dpa, len, rw);
2504	while (len) {
2505		unsigned int c;
2506		u64 offset;
2507
2508		if (mmio->num_lines) {
2509			u32 line_offset;
2510
2511			offset = to_interleave_offset(base_offset + copied,
2512					mmio);
2513			div_u64_rem(offset, mmio->line_size, &line_offset);
2514			c = min_t(size_t, len, mmio->line_size - line_offset);
2515		} else {
2516			offset = base_offset + nfit_blk->bdw_offset;
2517			c = len;
2518		}
2519
2520		if (rw)
2521			memcpy_flushcache(mmio->addr.aperture + offset, iobuf + copied, c);
2522		else {
2523			if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH)
2524				arch_invalidate_pmem((void __force *)
2525					mmio->addr.aperture + offset, c);
2526
2527			memcpy(iobuf + copied, mmio->addr.aperture + offset, c);
2528		}
2529
2530		copied += c;
2531		len -= c;
2532	}
2533
2534	if (rw)
2535		nvdimm_flush(nfit_blk->nd_region, NULL);
2536
2537	rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0;
2538	return rc;
2539}
2540
2541static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr,
2542		resource_size_t dpa, void *iobuf, u64 len, int rw)
2543{
2544	struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
2545	struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
2546	struct nd_region *nd_region = nfit_blk->nd_region;
2547	unsigned int lane, copied = 0;
2548	int rc = 0;
2549
2550	lane = nd_region_acquire_lane(nd_region);
2551	while (len) {
2552		u64 c = min(len, mmio->size);
2553
2554		rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied,
2555				iobuf + copied, c, rw, lane);
2556		if (rc)
2557			break;
2558
2559		copied += c;
2560		len -= c;
2561	}
2562	nd_region_release_lane(nd_region, lane);
2563
2564	return rc;
2565}
2566
2567static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio,
2568		struct acpi_nfit_interleave *idt, u16 interleave_ways)
2569{
2570	if (idt) {
2571		mmio->num_lines = idt->line_count;
2572		mmio->line_size = idt->line_size;
2573		if (interleave_ways == 0)
2574			return -ENXIO;
2575		mmio->table_size = mmio->num_lines * interleave_ways
2576			* mmio->line_size;
2577	}
2578
2579	return 0;
2580}
2581
2582static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc,
2583		struct nvdimm *nvdimm, struct nfit_blk *nfit_blk)
2584{
2585	struct nd_cmd_dimm_flags flags;
2586	int rc;
2587
2588	memset(&flags, 0, sizeof(flags));
2589	rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags,
2590			sizeof(flags), NULL);
2591
2592	if (rc >= 0 && flags.status == 0)
2593		nfit_blk->dimm_flags = flags.flags;
2594	else if (rc == -ENOTTY) {
2595		/* fall back to a conservative default */
2596		nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH;
2597		rc = 0;
2598	} else
2599		rc = -ENXIO;
2600
2601	return rc;
2602}
2603
2604static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus,
2605		struct device *dev)
2606{
2607	struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
2608	struct nd_blk_region *ndbr = to_nd_blk_region(dev);
2609	struct nfit_blk_mmio *mmio;
2610	struct nfit_blk *nfit_blk;
2611	struct nfit_mem *nfit_mem;
2612	struct nvdimm *nvdimm;
2613	int rc;
2614
2615	nvdimm = nd_blk_region_to_dimm(ndbr);
2616	nfit_mem = nvdimm_provider_data(nvdimm);
2617	if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) {
2618		dev_dbg(dev, "missing%s%s%s\n",
2619				nfit_mem ? "" : " nfit_mem",
2620				(nfit_mem && nfit_mem->dcr) ? "" : " dcr",
2621				(nfit_mem && nfit_mem->bdw) ? "" : " bdw");
2622		return -ENXIO;
2623	}
2624
2625	nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL);
2626	if (!nfit_blk)
2627		return -ENOMEM;
2628	nd_blk_region_set_provider_data(ndbr, nfit_blk);
2629	nfit_blk->nd_region = to_nd_region(dev);
2630
2631	/* map block aperture memory */
2632	nfit_blk->bdw_offset = nfit_mem->bdw->offset;
2633	mmio = &nfit_blk->mmio[BDW];
2634	mmio->addr.base = devm_nvdimm_memremap(dev, nfit_mem->spa_bdw->address,
2635                        nfit_mem->spa_bdw->length, nd_blk_memremap_flags(ndbr));
2636	if (!mmio->addr.base) {
2637		dev_dbg(dev, "%s failed to map bdw\n",
2638				nvdimm_name(nvdimm));
2639		return -ENOMEM;
2640	}
2641	mmio->size = nfit_mem->bdw->size;
2642	mmio->base_offset = nfit_mem->memdev_bdw->region_offset;
2643	mmio->idt = nfit_mem->idt_bdw;
2644	mmio->spa = nfit_mem->spa_bdw;
2645	rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw,
2646			nfit_mem->memdev_bdw->interleave_ways);
2647	if (rc) {
2648		dev_dbg(dev, "%s failed to init bdw interleave\n",
2649				nvdimm_name(nvdimm));
2650		return rc;
2651	}
2652
2653	/* map block control memory */
2654	nfit_blk->cmd_offset = nfit_mem->dcr->command_offset;
2655	nfit_blk->stat_offset = nfit_mem->dcr->status_offset;
2656	mmio = &nfit_blk->mmio[DCR];
2657	mmio->addr.base = devm_nvdimm_ioremap(dev, nfit_mem->spa_dcr->address,
2658			nfit_mem->spa_dcr->length);
2659	if (!mmio->addr.base) {
2660		dev_dbg(dev, "%s failed to map dcr\n",
2661				nvdimm_name(nvdimm));
2662		return -ENOMEM;
2663	}
2664	mmio->size = nfit_mem->dcr->window_size;
2665	mmio->base_offset = nfit_mem->memdev_dcr->region_offset;
2666	mmio->idt = nfit_mem->idt_dcr;
2667	mmio->spa = nfit_mem->spa_dcr;
2668	rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr,
2669			nfit_mem->memdev_dcr->interleave_ways);
2670	if (rc) {
2671		dev_dbg(dev, "%s failed to init dcr interleave\n",
2672				nvdimm_name(nvdimm));
2673		return rc;
2674	}
2675
2676	rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk);
2677	if (rc < 0) {
2678		dev_dbg(dev, "%s failed get DIMM flags\n",
2679				nvdimm_name(nvdimm));
2680		return rc;
2681	}
2682
2683	if (nvdimm_has_flush(nfit_blk->nd_region) < 0)
2684		dev_warn(dev, "unable to guarantee persistence of writes\n");
2685
2686	if (mmio->line_size == 0)
2687		return 0;
2688
2689	if ((u32) nfit_blk->cmd_offset % mmio->line_size
2690			+ 8 > mmio->line_size) {
2691		dev_dbg(dev, "cmd_offset crosses interleave boundary\n");
2692		return -ENXIO;
2693	} else if ((u32) nfit_blk->stat_offset % mmio->line_size
2694			+ 8 > mmio->line_size) {
2695		dev_dbg(dev, "stat_offset crosses interleave boundary\n");
2696		return -ENXIO;
2697	}
2698
2699	return 0;
2700}
2701
2702static int ars_get_cap(struct acpi_nfit_desc *acpi_desc,
2703		struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa)
2704{
2705	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2706	struct acpi_nfit_system_address *spa = nfit_spa->spa;
2707	int cmd_rc, rc;
2708
2709	cmd->address = spa->address;
2710	cmd->length = spa->length;
2711	rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd,
2712			sizeof(*cmd), &cmd_rc);
2713	if (rc < 0)
2714		return rc;
2715	return cmd_rc;
2716}
2717
2718static int ars_start(struct acpi_nfit_desc *acpi_desc,
2719		struct nfit_spa *nfit_spa, enum nfit_ars_state req_type)
2720{
2721	int rc;
2722	int cmd_rc;
2723	struct nd_cmd_ars_start ars_start;
2724	struct acpi_nfit_system_address *spa = nfit_spa->spa;
2725	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2726
2727	memset(&ars_start, 0, sizeof(ars_start));
2728	ars_start.address = spa->address;
2729	ars_start.length = spa->length;
2730	if (req_type == ARS_REQ_SHORT)
2731		ars_start.flags = ND_ARS_RETURN_PREV_DATA;
2732	if (nfit_spa_type(spa) == NFIT_SPA_PM)
2733		ars_start.type = ND_ARS_PERSISTENT;
2734	else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE)
2735		ars_start.type = ND_ARS_VOLATILE;
2736	else
2737		return -ENOTTY;
2738
2739	rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
2740			sizeof(ars_start), &cmd_rc);
2741
2742	if (rc < 0)
2743		return rc;
2744	if (cmd_rc < 0)
2745		return cmd_rc;
2746	set_bit(ARS_VALID, &acpi_desc->scrub_flags);
2747	return 0;
2748}
2749
2750static int ars_continue(struct acpi_nfit_desc *acpi_desc)
2751{
2752	int rc, cmd_rc;
2753	struct nd_cmd_ars_start ars_start;
2754	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2755	struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2756
2757	ars_start = (struct nd_cmd_ars_start) {
2758		.address = ars_status->restart_address,
2759		.length = ars_status->restart_length,
2760		.type = ars_status->type,
2761	};
2762	rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
2763			sizeof(ars_start), &cmd_rc);
2764	if (rc < 0)
2765		return rc;
2766	return cmd_rc;
2767}
2768
2769static int ars_get_status(struct acpi_nfit_desc *acpi_desc)
2770{
2771	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2772	struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2773	int rc, cmd_rc;
2774
2775	rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status,
2776			acpi_desc->max_ars, &cmd_rc);
2777	if (rc < 0)
2778		return rc;
2779	return cmd_rc;
2780}
2781
2782static void ars_complete(struct acpi_nfit_desc *acpi_desc,
2783		struct nfit_spa *nfit_spa)
2784{
2785	struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2786	struct acpi_nfit_system_address *spa = nfit_spa->spa;
2787	struct nd_region *nd_region = nfit_spa->nd_region;
2788	struct device *dev;
2789
2790	lockdep_assert_held(&acpi_desc->init_mutex);
2791	/*
2792	 * Only advance the ARS state for ARS runs initiated by the
2793	 * kernel, ignore ARS results from BIOS initiated runs for scrub
2794	 * completion tracking.
2795	 */
2796	if (acpi_desc->scrub_spa != nfit_spa)
2797		return;
2798
2799	if ((ars_status->address >= spa->address && ars_status->address
2800				< spa->address + spa->length)
2801			|| (ars_status->address < spa->address)) {
2802		/*
2803		 * Assume that if a scrub starts at an offset from the
2804		 * start of nfit_spa that we are in the continuation
2805		 * case.
2806		 *
2807		 * Otherwise, if the scrub covers the spa range, mark
2808		 * any pending request complete.
2809		 */
2810		if (ars_status->address + ars_status->length
2811				>= spa->address + spa->length)
2812				/* complete */;
2813		else
2814			return;
2815	} else
2816		return;
2817
2818	acpi_desc->scrub_spa = NULL;
2819	if (nd_region) {
2820		dev = nd_region_dev(nd_region);
2821		nvdimm_region_notify(nd_region, NVDIMM_REVALIDATE_POISON);
2822	} else
2823		dev = acpi_desc->dev;
2824	dev_dbg(dev, "ARS: range %d complete\n", spa->range_index);
2825}
2826
2827static int ars_status_process_records(struct acpi_nfit_desc *acpi_desc)
2828{
2829	struct nvdimm_bus *nvdimm_bus = acpi_desc->nvdimm_bus;
2830	struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2831	int rc;
2832	u32 i;
2833
2834	/*
2835	 * First record starts at 44 byte offset from the start of the
2836	 * payload.
2837	 */
2838	if (ars_status->out_length < 44)
2839		return 0;
2840
2841	/*
2842	 * Ignore potentially stale results that are only refreshed
2843	 * after a start-ARS event.
2844	 */
2845	if (!test_and_clear_bit(ARS_VALID, &acpi_desc->scrub_flags)) {
2846		dev_dbg(acpi_desc->dev, "skip %d stale records\n",
2847				ars_status->num_records);
2848		return 0;
2849	}
2850
2851	for (i = 0; i < ars_status->num_records; i++) {
2852		/* only process full records */
2853		if (ars_status->out_length
2854				< 44 + sizeof(struct nd_ars_record) * (i + 1))
2855			break;
2856		rc = nvdimm_bus_add_badrange(nvdimm_bus,
2857				ars_status->records[i].err_address,
2858				ars_status->records[i].length);
2859		if (rc)
2860			return rc;
2861	}
2862	if (i < ars_status->num_records)
2863		dev_warn(acpi_desc->dev, "detected truncated ars results\n");
2864
2865	return 0;
2866}
2867
2868static void acpi_nfit_remove_resource(void *data)
2869{
2870	struct resource *res = data;
2871
2872	remove_resource(res);
2873}
2874
2875static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc,
2876		struct nd_region_desc *ndr_desc)
2877{
2878	struct resource *res, *nd_res = ndr_desc->res;
2879	int is_pmem, ret;
2880
2881	/* No operation if the region is already registered as PMEM */
2882	is_pmem = region_intersects(nd_res->start, resource_size(nd_res),
2883				IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY);
2884	if (is_pmem == REGION_INTERSECTS)
2885		return 0;
2886
2887	res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL);
2888	if (!res)
2889		return -ENOMEM;
2890
2891	res->name = "Persistent Memory";
2892	res->start = nd_res->start;
2893	res->end = nd_res->end;
2894	res->flags = IORESOURCE_MEM;
2895	res->desc = IORES_DESC_PERSISTENT_MEMORY;
2896
2897	ret = insert_resource(&iomem_resource, res);
2898	if (ret)
2899		return ret;
2900
2901	ret = devm_add_action_or_reset(acpi_desc->dev,
2902					acpi_nfit_remove_resource,
2903					res);
2904	if (ret)
2905		return ret;
2906
2907	return 0;
2908}
2909
2910static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc,
2911		struct nd_mapping_desc *mapping, struct nd_region_desc *ndr_desc,
2912		struct acpi_nfit_memory_map *memdev,
2913		struct nfit_spa *nfit_spa)
2914{
2915	struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc,
2916			memdev->device_handle);
2917	struct acpi_nfit_system_address *spa = nfit_spa->spa;
2918	struct nd_blk_region_desc *ndbr_desc;
2919	struct nfit_mem *nfit_mem;
2920	int rc;
2921
2922	if (!nvdimm) {
2923		dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n",
2924				spa->range_index, memdev->device_handle);
2925		return -ENODEV;
2926	}
2927
2928	mapping->nvdimm = nvdimm;
2929	switch (nfit_spa_type(spa)) {
2930	case NFIT_SPA_PM:
2931	case NFIT_SPA_VOLATILE:
2932		mapping->start = memdev->address;
2933		mapping->size = memdev->region_size;
2934		break;
2935	case NFIT_SPA_DCR:
2936		nfit_mem = nvdimm_provider_data(nvdimm);
2937		if (!nfit_mem || !nfit_mem->bdw) {
2938			dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n",
2939					spa->range_index, nvdimm_name(nvdimm));
2940			break;
2941		}
2942
2943		mapping->size = nfit_mem->bdw->capacity;
2944		mapping->start = nfit_mem->bdw->start_address;
2945		ndr_desc->num_lanes = nfit_mem->bdw->windows;
2946		ndr_desc->mapping = mapping;
2947		ndr_desc->num_mappings = 1;
2948		ndbr_desc = to_blk_region_desc(ndr_desc);
2949		ndbr_desc->enable = acpi_nfit_blk_region_enable;
2950		ndbr_desc->do_io = acpi_desc->blk_do_io;
2951		rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
2952		if (rc)
2953			return rc;
2954		nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus,
2955				ndr_desc);
2956		if (!nfit_spa->nd_region)
2957			return -ENOMEM;
2958		break;
2959	}
2960
2961	return 0;
2962}
2963
2964static bool nfit_spa_is_virtual(struct acpi_nfit_system_address *spa)
2965{
2966	return (nfit_spa_type(spa) == NFIT_SPA_VDISK ||
2967		nfit_spa_type(spa) == NFIT_SPA_VCD   ||
2968		nfit_spa_type(spa) == NFIT_SPA_PDISK ||
2969		nfit_spa_type(spa) == NFIT_SPA_PCD);
2970}
2971
2972static bool nfit_spa_is_volatile(struct acpi_nfit_system_address *spa)
2973{
2974	return (nfit_spa_type(spa) == NFIT_SPA_VDISK ||
2975		nfit_spa_type(spa) == NFIT_SPA_VCD   ||
2976		nfit_spa_type(spa) == NFIT_SPA_VOLATILE);
2977}
2978
2979static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
2980		struct nfit_spa *nfit_spa)
2981{
2982	static struct nd_mapping_desc mappings[ND_MAX_MAPPINGS];
2983	struct acpi_nfit_system_address *spa = nfit_spa->spa;
2984	struct nd_blk_region_desc ndbr_desc;
2985	struct nd_region_desc *ndr_desc;
2986	struct nfit_memdev *nfit_memdev;
2987	struct nvdimm_bus *nvdimm_bus;
2988	struct resource res;
2989	int count = 0, rc;
2990
2991	if (nfit_spa->nd_region)
2992		return 0;
2993
2994	if (spa->range_index == 0 && !nfit_spa_is_virtual(spa)) {
2995		dev_dbg(acpi_desc->dev, "detected invalid spa index\n");
2996		return 0;
2997	}
2998
2999	memset(&res, 0, sizeof(res));
3000	memset(&mappings, 0, sizeof(mappings));
3001	memset(&ndbr_desc, 0, sizeof(ndbr_desc));
3002	res.start = spa->address;
3003	res.end = res.start + spa->length - 1;
3004	ndr_desc = &ndbr_desc.ndr_desc;
3005	ndr_desc->res = &res;
3006	ndr_desc->provider_data = nfit_spa;
3007	ndr_desc->attr_groups = acpi_nfit_region_attribute_groups;
3008	if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) {
3009		ndr_desc->numa_node = acpi_map_pxm_to_online_node(
3010						spa->proximity_domain);
3011		ndr_desc->target_node = acpi_map_pxm_to_node(
3012				spa->proximity_domain);
3013	} else {
3014		ndr_desc->numa_node = NUMA_NO_NODE;
3015		ndr_desc->target_node = NUMA_NO_NODE;
3016	}
3017
3018	/*
3019	 * Persistence domain bits are hierarchical, if
3020	 * ACPI_NFIT_CAPABILITY_CACHE_FLUSH is set then
3021	 * ACPI_NFIT_CAPABILITY_MEM_FLUSH is implied.
3022	 */
3023	if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_CACHE_FLUSH)
3024		set_bit(ND_REGION_PERSIST_CACHE, &ndr_desc->flags);
3025	else if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_MEM_FLUSH)
3026		set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc->flags);
3027
3028	list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
3029		struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
3030		struct nd_mapping_desc *mapping;
3031
3032		if (memdev->range_index != spa->range_index)
3033			continue;
3034		if (count >= ND_MAX_MAPPINGS) {
3035			dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n",
3036					spa->range_index, ND_MAX_MAPPINGS);
3037			return -ENXIO;
3038		}
3039		mapping = &mappings[count++];
3040		rc = acpi_nfit_init_mapping(acpi_desc, mapping, ndr_desc,
3041				memdev, nfit_spa);
3042		if (rc)
3043			goto out;
3044	}
3045
3046	ndr_desc->mapping = mappings;
3047	ndr_desc->num_mappings = count;
3048	rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
3049	if (rc)
3050		goto out;
3051
3052	nvdimm_bus = acpi_desc->nvdimm_bus;
3053	if (nfit_spa_type(spa) == NFIT_SPA_PM) {
3054		rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc);
3055		if (rc) {
3056			dev_warn(acpi_desc->dev,
3057				"failed to insert pmem resource to iomem: %d\n",
3058				rc);
3059			goto out;
3060		}
3061
3062		nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus,
3063				ndr_desc);
3064		if (!nfit_spa->nd_region)
3065			rc = -ENOMEM;
3066	} else if (nfit_spa_is_volatile(spa)) {
3067		nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus,
3068				ndr_desc);
3069		if (!nfit_spa->nd_region)
3070			rc = -ENOMEM;
3071	} else if (nfit_spa_is_virtual(spa)) {
3072		nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus,
3073				ndr_desc);
3074		if (!nfit_spa->nd_region)
3075			rc = -ENOMEM;
3076	}
3077
3078 out:
3079	if (rc)
3080		dev_err(acpi_desc->dev, "failed to register spa range %d\n",
3081				nfit_spa->spa->range_index);
3082	return rc;
3083}
3084
3085static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc)
3086{
3087	struct device *dev = acpi_desc->dev;
3088	struct nd_cmd_ars_status *ars_status;
3089
3090	if (acpi_desc->ars_status) {
3091		memset(acpi_desc->ars_status, 0, acpi_desc->max_ars);
3092		return 0;
3093	}
3094
3095	ars_status = devm_kzalloc(dev, acpi_desc->max_ars, GFP_KERNEL);
3096	if (!ars_status)
3097		return -ENOMEM;
3098	acpi_desc->ars_status = ars_status;
3099	return 0;
3100}
3101
3102static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc)
3103{
3104	int rc;
3105
3106	if (ars_status_alloc(acpi_desc))
3107		return -ENOMEM;
3108
3109	rc = ars_get_status(acpi_desc);
3110
3111	if (rc < 0 && rc != -ENOSPC)
3112		return rc;
3113
3114	if (ars_status_process_records(acpi_desc))
3115		dev_err(acpi_desc->dev, "Failed to process ARS records\n");
3116
3117	return rc;
3118}
3119
3120static int ars_register(struct acpi_nfit_desc *acpi_desc,
3121		struct nfit_spa *nfit_spa)
3122{
3123	int rc;
3124
3125	if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3126		return acpi_nfit_register_region(acpi_desc, nfit_spa);
3127
3128	set_bit(ARS_REQ_SHORT, &nfit_spa->ars_state);
3129	if (!no_init_ars)
3130		set_bit(ARS_REQ_LONG, &nfit_spa->ars_state);
3131
3132	switch (acpi_nfit_query_poison(acpi_desc)) {
3133	case 0:
3134	case -ENOSPC:
3135	case -EAGAIN:
3136		rc = ars_start(acpi_desc, nfit_spa, ARS_REQ_SHORT);
3137		/* shouldn't happen, try again later */
3138		if (rc == -EBUSY)
3139			break;
3140		if (rc) {
3141			set_bit(ARS_FAILED, &nfit_spa->ars_state);
3142			break;
3143		}
3144		clear_bit(ARS_REQ_SHORT, &nfit_spa->ars_state);
3145		rc = acpi_nfit_query_poison(acpi_desc);
3146		if (rc)
3147			break;
3148		acpi_desc->scrub_spa = nfit_spa;
3149		ars_complete(acpi_desc, nfit_spa);
3150		/*
3151		 * If ars_complete() says we didn't complete the
3152		 * short scrub, we'll try again with a long
3153		 * request.
3154		 */
3155		acpi_desc->scrub_spa = NULL;
3156		break;
3157	case -EBUSY:
3158	case -ENOMEM:
3159		/*
3160		 * BIOS was using ARS, wait for it to complete (or
3161		 * resources to become available) and then perform our
3162		 * own scrubs.
3163		 */
3164		break;
3165	default:
3166		set_bit(ARS_FAILED, &nfit_spa->ars_state);
3167		break;
3168	}
3169
3170	return acpi_nfit_register_region(acpi_desc, nfit_spa);
3171}
3172
3173static void ars_complete_all(struct acpi_nfit_desc *acpi_desc)
3174{
3175	struct nfit_spa *nfit_spa;
3176
3177	list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3178		if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3179			continue;
3180		ars_complete(acpi_desc, nfit_spa);
3181	}
3182}
3183
3184static unsigned int __acpi_nfit_scrub(struct acpi_nfit_desc *acpi_desc,
3185		int query_rc)
3186{
3187	unsigned int tmo = acpi_desc->scrub_tmo;
3188	struct device *dev = acpi_desc->dev;
3189	struct nfit_spa *nfit_spa;
3190
3191	lockdep_assert_held(&acpi_desc->init_mutex);
3192
3193	if (test_bit(ARS_CANCEL, &acpi_desc->scrub_flags))
3194		return 0;
3195
3196	if (query_rc == -EBUSY) {
3197		dev_dbg(dev, "ARS: ARS busy\n");
3198		return min(30U * 60U, tmo * 2);
3199	}
3200	if (query_rc == -ENOSPC) {
3201		dev_dbg(dev, "ARS: ARS continue\n");
3202		ars_continue(acpi_desc);
3203		return 1;
3204	}
3205	if (query_rc && query_rc != -EAGAIN) {
3206		unsigned long long addr, end;
3207
3208		addr = acpi_desc->ars_status->address;
3209		end = addr + acpi_desc->ars_status->length;
3210		dev_dbg(dev, "ARS: %llx-%llx failed (%d)\n", addr, end,
3211				query_rc);
3212	}
3213
3214	ars_complete_all(acpi_desc);
3215	list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3216		enum nfit_ars_state req_type;
3217		int rc;
3218
3219		if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3220			continue;
3221
3222		/* prefer short ARS requests first */
3223		if (test_bit(ARS_REQ_SHORT, &nfit_spa->ars_state))
3224			req_type = ARS_REQ_SHORT;
3225		else if (test_bit(ARS_REQ_LONG, &nfit_spa->ars_state))
3226			req_type = ARS_REQ_LONG;
3227		else
3228			continue;
3229		rc = ars_start(acpi_desc, nfit_spa, req_type);
3230
3231		dev = nd_region_dev(nfit_spa->nd_region);
3232		dev_dbg(dev, "ARS: range %d ARS start %s (%d)\n",
3233				nfit_spa->spa->range_index,
3234				req_type == ARS_REQ_SHORT ? "short" : "long",
3235				rc);
3236		/*
3237		 * Hmm, we raced someone else starting ARS? Try again in
3238		 * a bit.
3239		 */
3240		if (rc == -EBUSY)
3241			return 1;
3242		if (rc == 0) {
3243			dev_WARN_ONCE(dev, acpi_desc->scrub_spa,
3244					"scrub start while range %d active\n",
3245					acpi_desc->scrub_spa->spa->range_index);
3246			clear_bit(req_type, &nfit_spa->ars_state);
3247			acpi_desc->scrub_spa = nfit_spa;
3248			/*
3249			 * Consider this spa last for future scrub
3250			 * requests
3251			 */
3252			list_move_tail(&nfit_spa->list, &acpi_desc->spas);
3253			return 1;
3254		}
3255
3256		dev_err(dev, "ARS: range %d ARS failed (%d)\n",
3257				nfit_spa->spa->range_index, rc);
3258		set_bit(ARS_FAILED, &nfit_spa->ars_state);
3259	}
3260	return 0;
3261}
3262
3263static void __sched_ars(struct acpi_nfit_desc *acpi_desc, unsigned int tmo)
3264{
3265	lockdep_assert_held(&acpi_desc->init_mutex);
3266
3267	set_bit(ARS_BUSY, &acpi_desc->scrub_flags);
3268	/* note this should only be set from within the workqueue */
3269	if (tmo)
3270		acpi_desc->scrub_tmo = tmo;
3271	queue_delayed_work(nfit_wq, &acpi_desc->dwork, tmo * HZ);
3272}
3273
3274static void sched_ars(struct acpi_nfit_desc *acpi_desc)
3275{
3276	__sched_ars(acpi_desc, 0);
3277}
3278
3279static void notify_ars_done(struct acpi_nfit_desc *acpi_desc)
3280{
3281	lockdep_assert_held(&acpi_desc->init_mutex);
3282
3283	clear_bit(ARS_BUSY, &acpi_desc->scrub_flags);
3284	acpi_desc->scrub_count++;
3285	if (acpi_desc->scrub_count_state)
3286		sysfs_notify_dirent(acpi_desc->scrub_count_state);
3287}
3288
3289static void acpi_nfit_scrub(struct work_struct *work)
3290{
3291	struct acpi_nfit_desc *acpi_desc;
3292	unsigned int tmo;
3293	int query_rc;
3294
3295	acpi_desc = container_of(work, typeof(*acpi_desc), dwork.work);
3296	mutex_lock(&acpi_desc->init_mutex);
3297	query_rc = acpi_nfit_query_poison(acpi_desc);
3298	tmo = __acpi_nfit_scrub(acpi_desc, query_rc);
3299	if (tmo)
3300		__sched_ars(acpi_desc, tmo);
3301	else
3302		notify_ars_done(acpi_desc);
3303	memset(acpi_desc->ars_status, 0, acpi_desc->max_ars);
3304	clear_bit(ARS_POLL, &acpi_desc->scrub_flags);
3305	mutex_unlock(&acpi_desc->init_mutex);
3306}
3307
3308static void acpi_nfit_init_ars(struct acpi_nfit_desc *acpi_desc,
3309		struct nfit_spa *nfit_spa)
3310{
3311	int type = nfit_spa_type(nfit_spa->spa);
3312	struct nd_cmd_ars_cap ars_cap;
3313	int rc;
3314
3315	set_bit(ARS_FAILED, &nfit_spa->ars_state);
3316	memset(&ars_cap, 0, sizeof(ars_cap));
3317	rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa);
3318	if (rc < 0)
3319		return;
3320	/* check that the supported scrub types match the spa type */
3321	if (type == NFIT_SPA_VOLATILE && ((ars_cap.status >> 16)
3322				& ND_ARS_VOLATILE) == 0)
3323		return;
3324	if (type == NFIT_SPA_PM && ((ars_cap.status >> 16)
3325				& ND_ARS_PERSISTENT) == 0)
3326		return;
3327
3328	nfit_spa->max_ars = ars_cap.max_ars_out;
3329	nfit_spa->clear_err_unit = ars_cap.clear_err_unit;
3330	acpi_desc->max_ars = max(nfit_spa->max_ars, acpi_desc->max_ars);
3331	clear_bit(ARS_FAILED, &nfit_spa->ars_state);
3332}
3333
3334static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc)
3335{
3336	struct nfit_spa *nfit_spa;
3337	int rc, do_sched_ars = 0;
3338
3339	set_bit(ARS_VALID, &acpi_desc->scrub_flags);
3340	list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3341		switch (nfit_spa_type(nfit_spa->spa)) {
3342		case NFIT_SPA_VOLATILE:
3343		case NFIT_SPA_PM:
3344			acpi_nfit_init_ars(acpi_desc, nfit_spa);
3345			break;
3346		}
3347	}
3348
3349	list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3350		switch (nfit_spa_type(nfit_spa->spa)) {
3351		case NFIT_SPA_VOLATILE:
3352		case NFIT_SPA_PM:
3353			/* register regions and kick off initial ARS run */
3354			rc = ars_register(acpi_desc, nfit_spa);
3355			if (rc)
3356				return rc;
3357
3358			/*
3359			 * Kick off background ARS if at least one
3360			 * region successfully registered ARS
3361			 */
3362			if (!test_bit(ARS_FAILED, &nfit_spa->ars_state))
3363				do_sched_ars++;
3364			break;
3365		case NFIT_SPA_BDW:
3366			/* nothing to register */
3367			break;
3368		case NFIT_SPA_DCR:
3369		case NFIT_SPA_VDISK:
3370		case NFIT_SPA_VCD:
3371		case NFIT_SPA_PDISK:
3372		case NFIT_SPA_PCD:
3373			/* register known regions that don't support ARS */
3374			rc = acpi_nfit_register_region(acpi_desc, nfit_spa);
3375			if (rc)
3376				return rc;
3377			break;
3378		default:
3379			/* don't register unknown regions */
3380			break;
3381		}
3382	}
3383
3384	if (do_sched_ars)
3385		sched_ars(acpi_desc);
3386	return 0;
3387}
3388
3389static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc,
3390		struct nfit_table_prev *prev)
3391{
3392	struct device *dev = acpi_desc->dev;
3393
3394	if (!list_empty(&prev->spas) ||
3395			!list_empty(&prev->memdevs) ||
3396			!list_empty(&prev->dcrs) ||
3397			!list_empty(&prev->bdws) ||
3398			!list_empty(&prev->idts) ||
3399			!list_empty(&prev->flushes)) {
3400		dev_err(dev, "new nfit deletes entries (unsupported)\n");
3401		return -ENXIO;
3402	}
3403	return 0;
3404}
3405
3406static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc)
3407{
3408	struct device *dev = acpi_desc->dev;
3409	struct kernfs_node *nfit;
3410	struct device *bus_dev;
3411
3412	if (!ars_supported(acpi_desc->nvdimm_bus))
3413		return 0;
3414
3415	bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus);
3416	nfit = sysfs_get_dirent(bus_dev->kobj.sd, "nfit");
3417	if (!nfit) {
3418		dev_err(dev, "sysfs_get_dirent 'nfit' failed\n");
3419		return -ENODEV;
3420	}
3421	acpi_desc->scrub_count_state = sysfs_get_dirent(nfit, "scrub");
3422	sysfs_put(nfit);
3423	if (!acpi_desc->scrub_count_state) {
3424		dev_err(dev, "sysfs_get_dirent 'scrub' failed\n");
3425		return -ENODEV;
3426	}
3427
3428	return 0;
3429}
3430
3431static void acpi_nfit_unregister(void *data)
3432{
3433	struct acpi_nfit_desc *acpi_desc = data;
3434
3435	nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
3436}
3437
3438int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz)
3439{
3440	struct device *dev = acpi_desc->dev;
3441	struct nfit_table_prev prev;
3442	const void *end;
3443	int rc;
3444
3445	if (!acpi_desc->nvdimm_bus) {
3446		acpi_nfit_init_dsms(acpi_desc);
3447
3448		acpi_desc->nvdimm_bus = nvdimm_bus_register(dev,
3449				&acpi_desc->nd_desc);
3450		if (!acpi_desc->nvdimm_bus)
3451			return -ENOMEM;
3452
3453		rc = devm_add_action_or_reset(dev, acpi_nfit_unregister,
3454				acpi_desc);
3455		if (rc)
3456			return rc;
3457
3458		rc = acpi_nfit_desc_init_scrub_attr(acpi_desc);
3459		if (rc)
3460			return rc;
3461
3462		/* register this acpi_desc for mce notifications */
3463		mutex_lock(&acpi_desc_lock);
3464		list_add_tail(&acpi_desc->list, &acpi_descs);
3465		mutex_unlock(&acpi_desc_lock);
3466	}
3467
3468	mutex_lock(&acpi_desc->init_mutex);
3469
3470	INIT_LIST_HEAD(&prev.spas);
3471	INIT_LIST_HEAD(&prev.memdevs);
3472	INIT_LIST_HEAD(&prev.dcrs);
3473	INIT_LIST_HEAD(&prev.bdws);
3474	INIT_LIST_HEAD(&prev.idts);
3475	INIT_LIST_HEAD(&prev.flushes);
3476
3477	list_cut_position(&prev.spas, &acpi_desc->spas,
3478				acpi_desc->spas.prev);
3479	list_cut_position(&prev.memdevs, &acpi_desc->memdevs,
3480				acpi_desc->memdevs.prev);
3481	list_cut_position(&prev.dcrs, &acpi_desc->dcrs,
3482				acpi_desc->dcrs.prev);
3483	list_cut_position(&prev.bdws, &acpi_desc->bdws,
3484				acpi_desc->bdws.prev);
3485	list_cut_position(&prev.idts, &acpi_desc->idts,
3486				acpi_desc->idts.prev);
3487	list_cut_position(&prev.flushes, &acpi_desc->flushes,
3488				acpi_desc->flushes.prev);
3489
3490	end = data + sz;
3491	while (!IS_ERR_OR_NULL(data))
3492		data = add_table(acpi_desc, &prev, data, end);
3493
3494	if (IS_ERR(data)) {
3495		dev_dbg(dev, "nfit table parsing error: %ld\n",	PTR_ERR(data));
3496		rc = PTR_ERR(data);
3497		goto out_unlock;
3498	}
3499
3500	rc = acpi_nfit_check_deletions(acpi_desc, &prev);
3501	if (rc)
3502		goto out_unlock;
3503
3504	rc = nfit_mem_init(acpi_desc);
3505	if (rc)
3506		goto out_unlock;
3507
3508	rc = acpi_nfit_register_dimms(acpi_desc);
3509	if (rc)
3510		goto out_unlock;
3511
3512	rc = acpi_nfit_register_regions(acpi_desc);
3513
3514 out_unlock:
3515	mutex_unlock(&acpi_desc->init_mutex);
3516	return rc;
3517}
3518EXPORT_SYMBOL_GPL(acpi_nfit_init);
3519
3520static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc)
3521{
3522	struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
3523	struct device *dev = acpi_desc->dev;
3524
3525	/* Bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */
3526	nfit_device_lock(dev);
3527	nfit_device_unlock(dev);
3528
3529	/* Bounce the init_mutex to complete initial registration */
3530	mutex_lock(&acpi_desc->init_mutex);
3531	mutex_unlock(&acpi_desc->init_mutex);
3532
3533	return 0;
3534}
3535
3536static int __acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc,
3537		struct nvdimm *nvdimm, unsigned int cmd)
3538{
3539	struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
3540
3541	if (nvdimm)
3542		return 0;
3543	if (cmd != ND_CMD_ARS_START)
3544		return 0;
3545
3546	/*
3547	 * The kernel and userspace may race to initiate a scrub, but
3548	 * the scrub thread is prepared to lose that initial race.  It
3549	 * just needs guarantees that any ARS it initiates are not
3550	 * interrupted by any intervening start requests from userspace.
3551	 */
3552	if (work_busy(&acpi_desc->dwork.work))
3553		return -EBUSY;
3554
3555	return 0;
3556}
3557
3558/*
3559 * Prevent security and firmware activate commands from being issued via
3560 * ioctl.
3561 */
3562static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc,
3563		struct nvdimm *nvdimm, unsigned int cmd, void *buf)
3564{
3565	struct nd_cmd_pkg *call_pkg = buf;
3566	unsigned int func;
3567
3568	if (nvdimm && cmd == ND_CMD_CALL &&
3569			call_pkg->nd_family == NVDIMM_FAMILY_INTEL) {
3570		func = call_pkg->nd_command;
3571		if (func > NVDIMM_CMD_MAX ||
3572		    (1 << func) & NVDIMM_INTEL_DENY_CMDMASK)
3573			return -EOPNOTSUPP;
3574	}
3575
3576	/* block all non-nfit bus commands */
3577	if (!nvdimm && cmd == ND_CMD_CALL &&
3578			call_pkg->nd_family != NVDIMM_BUS_FAMILY_NFIT)
3579		return -EOPNOTSUPP;
3580
3581	return __acpi_nfit_clear_to_send(nd_desc, nvdimm, cmd);
3582}
3583
3584int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc,
3585		enum nfit_ars_state req_type)
3586{
3587	struct device *dev = acpi_desc->dev;
3588	int scheduled = 0, busy = 0;
3589	struct nfit_spa *nfit_spa;
3590
3591	mutex_lock(&acpi_desc->init_mutex);
3592	if (test_bit(ARS_CANCEL, &acpi_desc->scrub_flags)) {
3593		mutex_unlock(&acpi_desc->init_mutex);
3594		return 0;
3595	}
3596
3597	list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3598		int type = nfit_spa_type(nfit_spa->spa);
3599
3600		if (type != NFIT_SPA_PM && type != NFIT_SPA_VOLATILE)
3601			continue;
3602		if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3603			continue;
3604
3605		if (test_and_set_bit(req_type, &nfit_spa->ars_state))
3606			busy++;
3607		else
3608			scheduled++;
3609	}
3610	if (scheduled) {
3611		sched_ars(acpi_desc);
3612		dev_dbg(dev, "ars_scan triggered\n");
3613	}
3614	mutex_unlock(&acpi_desc->init_mutex);
3615
3616	if (scheduled)
3617		return 0;
3618	if (busy)
3619		return -EBUSY;
3620	return -ENOTTY;
3621}
3622
3623void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev)
3624{
3625	struct nvdimm_bus_descriptor *nd_desc;
3626
3627	dev_set_drvdata(dev, acpi_desc);
3628	acpi_desc->dev = dev;
3629	acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io;
3630	nd_desc = &acpi_desc->nd_desc;
3631	nd_desc->provider_name = "ACPI.NFIT";
3632	nd_desc->module = THIS_MODULE;
3633	nd_desc->ndctl = acpi_nfit_ctl;
3634	nd_desc->flush_probe = acpi_nfit_flush_probe;
3635	nd_desc->clear_to_send = acpi_nfit_clear_to_send;
3636	nd_desc->attr_groups = acpi_nfit_attribute_groups;
3637
3638	INIT_LIST_HEAD(&acpi_desc->spas);
3639	INIT_LIST_HEAD(&acpi_desc->dcrs);
3640	INIT_LIST_HEAD(&acpi_desc->bdws);
3641	INIT_LIST_HEAD(&acpi_desc->idts);
3642	INIT_LIST_HEAD(&acpi_desc->flushes);
3643	INIT_LIST_HEAD(&acpi_desc->memdevs);
3644	INIT_LIST_HEAD(&acpi_desc->dimms);
3645	INIT_LIST_HEAD(&acpi_desc->list);
3646	mutex_init(&acpi_desc->init_mutex);
3647	acpi_desc->scrub_tmo = 1;
3648	INIT_DELAYED_WORK(&acpi_desc->dwork, acpi_nfit_scrub);
3649}
3650EXPORT_SYMBOL_GPL(acpi_nfit_desc_init);
3651
3652static void acpi_nfit_put_table(void *table)
3653{
3654	acpi_put_table(table);
3655}
3656
3657void acpi_nfit_shutdown(void *data)
3658{
3659	struct acpi_nfit_desc *acpi_desc = data;
3660	struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus);
3661
3662	/*
3663	 * Destruct under acpi_desc_lock so that nfit_handle_mce does not
3664	 * race teardown
3665	 */
3666	mutex_lock(&acpi_desc_lock);
3667	list_del(&acpi_desc->list);
3668	mutex_unlock(&acpi_desc_lock);
3669
3670	mutex_lock(&acpi_desc->init_mutex);
3671	set_bit(ARS_CANCEL, &acpi_desc->scrub_flags);
3672	cancel_delayed_work_sync(&acpi_desc->dwork);
3673	mutex_unlock(&acpi_desc->init_mutex);
3674
3675	/*
3676	 * Bounce the nvdimm bus lock to make sure any in-flight
3677	 * acpi_nfit_ars_rescan() submissions have had a chance to
3678	 * either submit or see ->cancel set.
3679	 */
3680	nfit_device_lock(bus_dev);
3681	nfit_device_unlock(bus_dev);
3682
3683	flush_workqueue(nfit_wq);
3684}
3685EXPORT_SYMBOL_GPL(acpi_nfit_shutdown);
3686
3687static int acpi_nfit_add(struct acpi_device *adev)
3688{
3689	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
3690	struct acpi_nfit_desc *acpi_desc;
3691	struct device *dev = &adev->dev;
3692	struct acpi_table_header *tbl;
3693	acpi_status status = AE_OK;
3694	acpi_size sz;
3695	int rc = 0;
3696
3697	status = acpi_get_table(ACPI_SIG_NFIT, 0, &tbl);
3698	if (ACPI_FAILURE(status)) {
3699		/* The NVDIMM root device allows OS to trigger enumeration of
3700		 * NVDIMMs through NFIT at boot time and re-enumeration at
3701		 * root level via the _FIT method during runtime.
3702		 * This is ok to return 0 here, we could have an nvdimm
3703		 * hotplugged later and evaluate _FIT method which returns
3704		 * data in the format of a series of NFIT Structures.
3705		 */
3706		dev_dbg(dev, "failed to find NFIT at startup\n");
3707		return 0;
3708	}
3709
3710	rc = devm_add_action_or_reset(dev, acpi_nfit_put_table, tbl);
3711	if (rc)
3712		return rc;
3713	sz = tbl->length;
3714
3715	acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
3716	if (!acpi_desc)
3717		return -ENOMEM;
3718	acpi_nfit_desc_init(acpi_desc, &adev->dev);
3719
3720	/* Save the acpi header for exporting the revision via sysfs */
3721	acpi_desc->acpi_header = *tbl;
3722
3723	/* Evaluate _FIT and override with that if present */
3724	status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf);
3725	if (ACPI_SUCCESS(status) && buf.length > 0) {
3726		union acpi_object *obj = buf.pointer;
3727
3728		if (obj->type == ACPI_TYPE_BUFFER)
3729			rc = acpi_nfit_init(acpi_desc, obj->buffer.pointer,
3730					obj->buffer.length);
3731		else
3732			dev_dbg(dev, "invalid type %d, ignoring _FIT\n",
3733				(int) obj->type);
3734		kfree(buf.pointer);
3735	} else
3736		/* skip over the lead-in header table */
3737		rc = acpi_nfit_init(acpi_desc, (void *) tbl
3738				+ sizeof(struct acpi_table_nfit),
3739				sz - sizeof(struct acpi_table_nfit));
3740
3741	if (rc)
3742		return rc;
3743	return devm_add_action_or_reset(dev, acpi_nfit_shutdown, acpi_desc);
3744}
3745
3746static int acpi_nfit_remove(struct acpi_device *adev)
3747{
3748	/* see acpi_nfit_unregister */
3749	return 0;
3750}
3751
3752static void acpi_nfit_update_notify(struct device *dev, acpi_handle handle)
3753{
3754	struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev);
3755	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
3756	union acpi_object *obj;
3757	acpi_status status;
3758	int ret;
3759
3760	if (!dev->driver) {
3761		/* dev->driver may be null if we're being removed */
3762		dev_dbg(dev, "no driver found for dev\n");
3763		return;
3764	}
3765
3766	if (!acpi_desc) {
3767		acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
3768		if (!acpi_desc)
3769			return;
3770		acpi_nfit_desc_init(acpi_desc, dev);
3771	} else {
3772		/*
3773		 * Finish previous registration before considering new
3774		 * regions.
3775		 */
3776		flush_workqueue(nfit_wq);
3777	}
3778
3779	/* Evaluate _FIT */
3780	status = acpi_evaluate_object(handle, "_FIT", NULL, &buf);
3781	if (ACPI_FAILURE(status)) {
3782		dev_err(dev, "failed to evaluate _FIT\n");
3783		return;
3784	}
3785
3786	obj = buf.pointer;
3787	if (obj->type == ACPI_TYPE_BUFFER) {
3788		ret = acpi_nfit_init(acpi_desc, obj->buffer.pointer,
3789				obj->buffer.length);
3790		if (ret)
3791			dev_err(dev, "failed to merge updated NFIT\n");
3792	} else
3793		dev_err(dev, "Invalid _FIT\n");
3794	kfree(buf.pointer);
3795}
3796
3797static void acpi_nfit_uc_error_notify(struct device *dev, acpi_handle handle)
3798{
3799	struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev);
3800
3801	if (acpi_desc->scrub_mode == HW_ERROR_SCRUB_ON)
3802		acpi_nfit_ars_rescan(acpi_desc, ARS_REQ_LONG);
3803	else
3804		acpi_nfit_ars_rescan(acpi_desc, ARS_REQ_SHORT);
3805}
3806
3807void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event)
3808{
3809	dev_dbg(dev, "event: 0x%x\n", event);
3810
3811	switch (event) {
3812	case NFIT_NOTIFY_UPDATE:
3813		return acpi_nfit_update_notify(dev, handle);
3814	case NFIT_NOTIFY_UC_MEMORY_ERROR:
3815		return acpi_nfit_uc_error_notify(dev, handle);
3816	default:
3817		return;
3818	}
3819}
3820EXPORT_SYMBOL_GPL(__acpi_nfit_notify);
3821
3822static void acpi_nfit_notify(struct acpi_device *adev, u32 event)
3823{
3824	nfit_device_lock(&adev->dev);
3825	__acpi_nfit_notify(&adev->dev, adev->handle, event);
3826	nfit_device_unlock(&adev->dev);
3827}
3828
3829static const struct acpi_device_id acpi_nfit_ids[] = {
3830	{ "ACPI0012", 0 },
3831	{ "", 0 },
3832};
3833MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids);
3834
3835static struct acpi_driver acpi_nfit_driver = {
3836	.name = KBUILD_MODNAME,
3837	.ids = acpi_nfit_ids,
3838	.ops = {
3839		.add = acpi_nfit_add,
3840		.remove = acpi_nfit_remove,
3841		.notify = acpi_nfit_notify,
3842	},
3843};
3844
3845static __init int nfit_init(void)
3846{
3847	int ret;
3848
3849	BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40);
3850	BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56);
3851	BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48);
3852	BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20);
3853	BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9);
3854	BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80);
3855	BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40);
3856	BUILD_BUG_ON(sizeof(struct acpi_nfit_capabilities) != 16);
3857
3858	guid_parse(UUID_VOLATILE_MEMORY, &nfit_uuid[NFIT_SPA_VOLATILE]);
3859	guid_parse(UUID_PERSISTENT_MEMORY, &nfit_uuid[NFIT_SPA_PM]);
3860	guid_parse(UUID_CONTROL_REGION, &nfit_uuid[NFIT_SPA_DCR]);
3861	guid_parse(UUID_DATA_REGION, &nfit_uuid[NFIT_SPA_BDW]);
3862	guid_parse(UUID_VOLATILE_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_VDISK]);
3863	guid_parse(UUID_VOLATILE_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_VCD]);
3864	guid_parse(UUID_PERSISTENT_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_PDISK]);
3865	guid_parse(UUID_PERSISTENT_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_PCD]);
3866	guid_parse(UUID_NFIT_BUS, &nfit_uuid[NFIT_DEV_BUS]);
3867	guid_parse(UUID_NFIT_DIMM, &nfit_uuid[NFIT_DEV_DIMM]);
3868	guid_parse(UUID_NFIT_DIMM_N_HPE1, &nfit_uuid[NFIT_DEV_DIMM_N_HPE1]);
3869	guid_parse(UUID_NFIT_DIMM_N_HPE2, &nfit_uuid[NFIT_DEV_DIMM_N_HPE2]);
3870	guid_parse(UUID_NFIT_DIMM_N_MSFT, &nfit_uuid[NFIT_DEV_DIMM_N_MSFT]);
3871	guid_parse(UUID_NFIT_DIMM_N_HYPERV, &nfit_uuid[NFIT_DEV_DIMM_N_HYPERV]);
3872	guid_parse(UUID_INTEL_BUS, &nfit_uuid[NFIT_BUS_INTEL]);
3873
3874	nfit_wq = create_singlethread_workqueue("nfit");
3875	if (!nfit_wq)
3876		return -ENOMEM;
3877
3878	nfit_mce_register();
3879	ret = acpi_bus_register_driver(&acpi_nfit_driver);
3880	if (ret) {
3881		nfit_mce_unregister();
3882		destroy_workqueue(nfit_wq);
3883	}
3884
3885	return ret;
3886
3887}
3888
3889static __exit void nfit_exit(void)
3890{
3891	nfit_mce_unregister();
3892	acpi_bus_unregister_driver(&acpi_nfit_driver);
3893	destroy_workqueue(nfit_wq);
3894	WARN_ON(!list_empty(&acpi_descs));
3895}
3896
3897module_init(nfit_init);
3898module_exit(nfit_exit);
3899MODULE_LICENSE("GPL v2");
3900MODULE_AUTHOR("Intel Corporation");