1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Block driver for media (i.e., flash cards)
   4 *
   5 * Copyright 2002 Hewlett-Packard Company
   6 * Copyright 2005-2008 Pierre Ossman
   7 *
   8 * Use consistent with the GNU GPL is permitted,
   9 * provided that this copyright notice is
  10 * preserved in its entirety in all copies and derived works.
  11 *
  12 * HEWLETT-PACKARD COMPANY MAKES NO WARRANTIES, EXPRESSED OR IMPLIED,
  13 * AS TO THE USEFULNESS OR CORRECTNESS OF THIS CODE OR ITS
  14 * FITNESS FOR ANY PARTICULAR PURPOSE.
  15 *
  16 * Many thanks to Alessandro Rubini and Jonathan Corbet!
  17 *
  18 * Author:  Andrew Christian
  19 *          28 May 2002
  20 */
  21#include <linux/moduleparam.h>
  22#include <linux/module.h>
  23#include <linux/init.h>
  24
  25#include <linux/kernel.h>
  26#include <linux/fs.h>
  27#include <linux/slab.h>
  28#include <linux/errno.h>
  29#include <linux/hdreg.h>
  30#include <linux/kdev_t.h>
  31#include <linux/kref.h>
  32#include <linux/blkdev.h>
  33#include <linux/cdev.h>
  34#include <linux/mutex.h>
  35#include <linux/scatterlist.h>
  36#include <linux/string_helpers.h>
  37#include <linux/delay.h>
  38#include <linux/capability.h>
  39#include <linux/compat.h>
  40#include <linux/pm_runtime.h>
  41#include <linux/idr.h>
  42#include <linux/debugfs.h>
  43
  44#include <linux/mmc/ioctl.h>
  45#include <linux/mmc/card.h>
  46#include <linux/mmc/host.h>
  47#include <linux/mmc/mmc.h>
  48#include <linux/mmc/sd.h>
  49
  50#include <linux/uaccess.h>
  51
  52#include "queue.h"
  53#include "block.h"
  54#include "core.h"
  55#include "card.h"
  56#include "crypto.h"
  57#include "host.h"
  58#include "bus.h"
  59#include "mmc_ops.h"
  60#include "quirks.h"
  61#include "sd_ops.h"
  62
  63MODULE_ALIAS("mmc:block");
  64#ifdef MODULE_PARAM_PREFIX
  65#undef MODULE_PARAM_PREFIX
  66#endif
  67#define MODULE_PARAM_PREFIX "mmcblk."
  68
  69/*
  70 * Set a 10 second timeout for polling write request busy state. Note, mmc core
  71 * is setting a 3 second timeout for SD cards, and SDHCI has long had a 10
  72 * second software timer to timeout the whole request, so 10 seconds should be
  73 * ample.
  74 */
  75#define MMC_BLK_TIMEOUT_MS  (10 * 1000)
  76#define MMC_EXTRACT_INDEX_FROM_ARG(x) ((x & 0x00FF0000) >> 16)
  77#define MMC_EXTRACT_VALUE_FROM_ARG(x) ((x & 0x0000FF00) >> 8)
  78
 
 
  79static DEFINE_MUTEX(block_mutex);
  80
  81/*
  82 * The defaults come from config options but can be overriden by module
  83 * or bootarg options.
  84 */
  85static int perdev_minors = CONFIG_MMC_BLOCK_MINORS;
  86
  87/*
  88 * We've only got one major, so number of mmcblk devices is
  89 * limited to (1 << 20) / number of minors per device.  It is also
  90 * limited by the MAX_DEVICES below.
  91 */
  92static int max_devices;
  93
  94#define MAX_DEVICES 256
  95
  96static DEFINE_IDA(mmc_blk_ida);
  97static DEFINE_IDA(mmc_rpmb_ida);
  98
  99struct mmc_blk_busy_data {
 100	struct mmc_card *card;
 101	u32 status;
 102};
 103
 104/*
 105 * There is one mmc_blk_data per slot.
 106 */
 107struct mmc_blk_data {
 108	struct device	*parent;
 109	struct gendisk	*disk;
 110	struct mmc_queue queue;
 111	struct list_head part;
 112	struct list_head rpmbs;
 113
 114	unsigned int	flags;
 115#define MMC_BLK_CMD23	(1 << 0)	/* Can do SET_BLOCK_COUNT for multiblock */
 116#define MMC_BLK_REL_WR	(1 << 1)	/* MMC Reliable write support */
 117
 118	struct kref	kref;
 119	unsigned int	read_only;
 120	unsigned int	part_type;
 121	unsigned int	reset_done;
 122#define MMC_BLK_READ		BIT(0)
 123#define MMC_BLK_WRITE		BIT(1)
 124#define MMC_BLK_DISCARD		BIT(2)
 125#define MMC_BLK_SECDISCARD	BIT(3)
 126#define MMC_BLK_CQE_RECOVERY	BIT(4)
 127#define MMC_BLK_TRIM		BIT(5)
 128
 129	/*
 130	 * Only set in main mmc_blk_data associated
 131	 * with mmc_card with dev_set_drvdata, and keeps
 132	 * track of the current selected device partition.
 133	 */
 134	unsigned int	part_curr;
 135#define MMC_BLK_PART_INVALID	UINT_MAX	/* Unknown partition active */
 
 136	int	area_type;
 137
 138	/* debugfs files (only in main mmc_blk_data) */
 139	struct dentry *status_dentry;
 140	struct dentry *ext_csd_dentry;
 141};
 142
 143/* Device type for RPMB character devices */
 144static dev_t mmc_rpmb_devt;
 145
 146/* Bus type for RPMB character devices */
 147static struct bus_type mmc_rpmb_bus_type = {
 148	.name = "mmc_rpmb",
 149};
 150
 151/**
 152 * struct mmc_rpmb_data - special RPMB device type for these areas
 153 * @dev: the device for the RPMB area
 154 * @chrdev: character device for the RPMB area
 155 * @id: unique device ID number
 156 * @part_index: partition index (0 on first)
 157 * @md: parent MMC block device
 158 * @node: list item, so we can put this device on a list
 159 */
 160struct mmc_rpmb_data {
 161	struct device dev;
 162	struct cdev chrdev;
 163	int id;
 164	unsigned int part_index;
 165	struct mmc_blk_data *md;
 166	struct list_head node;
 167};
 168
 169static DEFINE_MUTEX(open_lock);
 170
 171module_param(perdev_minors, int, 0444);
 172MODULE_PARM_DESC(perdev_minors, "Minors numbers to allocate per device");
 173
 174static inline int mmc_blk_part_switch(struct mmc_card *card,
 175				      unsigned int part_type);
 176static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
 177			       struct mmc_card *card,
 178			       int recovery_mode,
 179			       struct mmc_queue *mq);
 180static void mmc_blk_hsq_req_done(struct mmc_request *mrq);
 181static int mmc_spi_err_check(struct mmc_card *card);
 182static int mmc_blk_busy_cb(void *cb_data, bool *busy);
 183
 184static struct mmc_blk_data *mmc_blk_get(struct gendisk *disk)
 185{
 186	struct mmc_blk_data *md;
 187
 188	mutex_lock(&open_lock);
 189	md = disk->private_data;
 190	if (md && !kref_get_unless_zero(&md->kref))
 191		md = NULL;
 192	mutex_unlock(&open_lock);
 193
 194	return md;
 195}
 196
 197static inline int mmc_get_devidx(struct gendisk *disk)
 198{
 199	int devidx = disk->first_minor / perdev_minors;
 200	return devidx;
 201}
 202
 203static void mmc_blk_kref_release(struct kref *ref)
 204{
 205	struct mmc_blk_data *md = container_of(ref, struct mmc_blk_data, kref);
 206	int devidx;
 207
 208	devidx = mmc_get_devidx(md->disk);
 209	ida_simple_remove(&mmc_blk_ida, devidx);
 210
 211	mutex_lock(&open_lock);
 212	md->disk->private_data = NULL;
 213	mutex_unlock(&open_lock);
 214
 215	put_disk(md->disk);
 216	kfree(md);
 217}
 218
 219static void mmc_blk_put(struct mmc_blk_data *md)
 220{
 221	kref_put(&md->kref, mmc_blk_kref_release);
 222}
 223
 224static ssize_t power_ro_lock_show(struct device *dev,
 225		struct device_attribute *attr, char *buf)
 226{
 227	int ret;
 228	struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
 229	struct mmc_card *card = md->queue.card;
 230	int locked = 0;
 231
 232	if (card->ext_csd.boot_ro_lock & EXT_CSD_BOOT_WP_B_PERM_WP_EN)
 233		locked = 2;
 234	else if (card->ext_csd.boot_ro_lock & EXT_CSD_BOOT_WP_B_PWR_WP_EN)
 235		locked = 1;
 236
 237	ret = snprintf(buf, PAGE_SIZE, "%d\n", locked);
 238
 239	mmc_blk_put(md);
 240
 241	return ret;
 242}
 243
 244static ssize_t power_ro_lock_store(struct device *dev,
 245		struct device_attribute *attr, const char *buf, size_t count)
 246{
 247	int ret;
 248	struct mmc_blk_data *md, *part_md;
 249	struct mmc_queue *mq;
 250	struct request *req;
 251	unsigned long set;
 252
 253	if (kstrtoul(buf, 0, &set))
 254		return -EINVAL;
 255
 256	if (set != 1)
 257		return count;
 258
 259	md = mmc_blk_get(dev_to_disk(dev));
 260	mq = &md->queue;
 261
 262	/* Dispatch locking to the block layer */
 263	req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_OUT, 0);
 264	if (IS_ERR(req)) {
 265		count = PTR_ERR(req);
 266		goto out_put;
 267	}
 268	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_BOOT_WP;
 269	req_to_mmc_queue_req(req)->drv_op_result = -EIO;
 270	blk_execute_rq(req, false);
 271	ret = req_to_mmc_queue_req(req)->drv_op_result;
 272	blk_mq_free_request(req);
 273
 274	if (!ret) {
 275		pr_info("%s: Locking boot partition ro until next power on\n",
 276			md->disk->disk_name);
 277		set_disk_ro(md->disk, 1);
 278
 279		list_for_each_entry(part_md, &md->part, part)
 280			if (part_md->area_type == MMC_BLK_DATA_AREA_BOOT) {
 281				pr_info("%s: Locking boot partition ro until next power on\n", part_md->disk->disk_name);
 282				set_disk_ro(part_md->disk, 1);
 283			}
 284	}
 285out_put:
 286	mmc_blk_put(md);
 287	return count;
 288}
 289
 290static DEVICE_ATTR(ro_lock_until_next_power_on, 0,
 291		power_ro_lock_show, power_ro_lock_store);
 292
 293static ssize_t force_ro_show(struct device *dev, struct device_attribute *attr,
 294			     char *buf)
 295{
 296	int ret;
 297	struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
 298
 299	ret = snprintf(buf, PAGE_SIZE, "%d\n",
 300		       get_disk_ro(dev_to_disk(dev)) ^
 301		       md->read_only);
 302	mmc_blk_put(md);
 303	return ret;
 304}
 305
 306static ssize_t force_ro_store(struct device *dev, struct device_attribute *attr,
 307			      const char *buf, size_t count)
 308{
 309	int ret;
 310	char *end;
 311	struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
 312	unsigned long set = simple_strtoul(buf, &end, 0);
 313	if (end == buf) {
 314		ret = -EINVAL;
 315		goto out;
 316	}
 317
 318	set_disk_ro(dev_to_disk(dev), set || md->read_only);
 319	ret = count;
 320out:
 321	mmc_blk_put(md);
 322	return ret;
 323}
 324
 325static DEVICE_ATTR(force_ro, 0644, force_ro_show, force_ro_store);
 326
 327static struct attribute *mmc_disk_attrs[] = {
 328	&dev_attr_force_ro.attr,
 329	&dev_attr_ro_lock_until_next_power_on.attr,
 330	NULL,
 331};
 332
 333static umode_t mmc_disk_attrs_is_visible(struct kobject *kobj,
 334		struct attribute *a, int n)
 335{
 336	struct device *dev = kobj_to_dev(kobj);
 337	struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
 338	umode_t mode = a->mode;
 339
 340	if (a == &dev_attr_ro_lock_until_next_power_on.attr &&
 341	    (md->area_type & MMC_BLK_DATA_AREA_BOOT) &&
 342	    md->queue.card->ext_csd.boot_ro_lockable) {
 343		mode = S_IRUGO;
 344		if (!(md->queue.card->ext_csd.boot_ro_lock &
 345				EXT_CSD_BOOT_WP_B_PWR_WP_DIS))
 346			mode |= S_IWUSR;
 347	}
 348
 349	mmc_blk_put(md);
 350	return mode;
 351}
 352
 353static const struct attribute_group mmc_disk_attr_group = {
 354	.is_visible	= mmc_disk_attrs_is_visible,
 355	.attrs		= mmc_disk_attrs,
 356};
 357
 358static const struct attribute_group *mmc_disk_attr_groups[] = {
 359	&mmc_disk_attr_group,
 360	NULL,
 361};
 362
 363static int mmc_blk_open(struct gendisk *disk, blk_mode_t mode)
 364{
 365	struct mmc_blk_data *md = mmc_blk_get(disk);
 366	int ret = -ENXIO;
 367
 368	mutex_lock(&block_mutex);
 369	if (md) {
 370		ret = 0;
 371		if ((mode & BLK_OPEN_WRITE) && md->read_only) {
 372			mmc_blk_put(md);
 373			ret = -EROFS;
 374		}
 375	}
 376	mutex_unlock(&block_mutex);
 377
 378	return ret;
 379}
 380
 381static void mmc_blk_release(struct gendisk *disk)
 382{
 383	struct mmc_blk_data *md = disk->private_data;
 384
 385	mutex_lock(&block_mutex);
 386	mmc_blk_put(md);
 387	mutex_unlock(&block_mutex);
 388}
 389
 390static int
 391mmc_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
 392{
 393	geo->cylinders = get_capacity(bdev->bd_disk) / (4 * 16);
 394	geo->heads = 4;
 395	geo->sectors = 16;
 396	return 0;
 397}
 398
 399struct mmc_blk_ioc_data {
 400	struct mmc_ioc_cmd ic;
 401	unsigned char *buf;
 402	u64 buf_bytes;
 403	unsigned int flags;
 404#define MMC_BLK_IOC_DROP	BIT(0)	/* drop this mrq */
 405#define MMC_BLK_IOC_SBC	BIT(1)	/* use mrq.sbc */
 406
 407	struct mmc_rpmb_data *rpmb;
 408};
 409
 410static struct mmc_blk_ioc_data *mmc_blk_ioctl_copy_from_user(
 411	struct mmc_ioc_cmd __user *user)
 412{
 413	struct mmc_blk_ioc_data *idata;
 414	int err;
 415
 416	idata = kmalloc(sizeof(*idata), GFP_KERNEL);
 417	if (!idata) {
 418		err = -ENOMEM;
 419		goto out;
 420	}
 421
 422	if (copy_from_user(&idata->ic, user, sizeof(idata->ic))) {
 423		err = -EFAULT;
 424		goto idata_err;
 425	}
 426
 427	idata->buf_bytes = (u64) idata->ic.blksz * idata->ic.blocks;
 428	if (idata->buf_bytes > MMC_IOC_MAX_BYTES) {
 429		err = -EOVERFLOW;
 430		goto idata_err;
 431	}
 432
 433	if (!idata->buf_bytes) {
 434		idata->buf = NULL;
 435		return idata;
 436	}
 437
 438	idata->buf = memdup_user((void __user *)(unsigned long)
 439				 idata->ic.data_ptr, idata->buf_bytes);
 440	if (IS_ERR(idata->buf)) {
 441		err = PTR_ERR(idata->buf);
 442		goto idata_err;
 443	}
 444
 445	return idata;
 446
 447idata_err:
 448	kfree(idata);
 449out:
 450	return ERR_PTR(err);
 451}
 452
 453static int mmc_blk_ioctl_copy_to_user(struct mmc_ioc_cmd __user *ic_ptr,
 454				      struct mmc_blk_ioc_data *idata)
 455{
 456	struct mmc_ioc_cmd *ic = &idata->ic;
 457
 458	if (copy_to_user(&(ic_ptr->response), ic->response,
 459			 sizeof(ic->response)))
 460		return -EFAULT;
 461
 462	if (!idata->ic.write_flag) {
 463		if (copy_to_user((void __user *)(unsigned long)ic->data_ptr,
 464				 idata->buf, idata->buf_bytes))
 465			return -EFAULT;
 466	}
 467
 468	return 0;
 469}
 470
 471static int __mmc_blk_ioctl_cmd(struct mmc_card *card, struct mmc_blk_data *md,
 472			       struct mmc_blk_ioc_data **idatas, int i)
 473{
 474	struct mmc_command cmd = {}, sbc = {};
 475	struct mmc_data data = {};
 476	struct mmc_request mrq = {};
 477	struct scatterlist sg;
 478	bool r1b_resp;
 479	unsigned int busy_timeout_ms;
 480	int err;
 481	unsigned int target_part;
 482	struct mmc_blk_ioc_data *idata = idatas[i];
 483	struct mmc_blk_ioc_data *prev_idata = NULL;
 484
 485	if (!card || !md || !idata)
 486		return -EINVAL;
 487
 488	if (idata->flags & MMC_BLK_IOC_DROP)
 489		return 0;
 490
 491	if (idata->flags & MMC_BLK_IOC_SBC)
 492		prev_idata = idatas[i - 1];
 493
 494	/*
 495	 * The RPMB accesses comes in from the character device, so we
 496	 * need to target these explicitly. Else we just target the
 497	 * partition type for the block device the ioctl() was issued
 498	 * on.
 499	 */
 500	if (idata->rpmb) {
 501		/* Support multiple RPMB partitions */
 502		target_part = idata->rpmb->part_index;
 503		target_part |= EXT_CSD_PART_CONFIG_ACC_RPMB;
 504	} else {
 505		target_part = md->part_type;
 506	}
 507
 508	cmd.opcode = idata->ic.opcode;
 509	cmd.arg = idata->ic.arg;
 510	cmd.flags = idata->ic.flags;
 511
 512	if (idata->buf_bytes) {
 513		data.sg = &sg;
 514		data.sg_len = 1;
 515		data.blksz = idata->ic.blksz;
 516		data.blocks = idata->ic.blocks;
 517
 518		sg_init_one(data.sg, idata->buf, idata->buf_bytes);
 519
 520		if (idata->ic.write_flag)
 521			data.flags = MMC_DATA_WRITE;
 522		else
 523			data.flags = MMC_DATA_READ;
 524
 525		/* data.flags must already be set before doing this. */
 526		mmc_set_data_timeout(&data, card);
 527
 528		/* Allow overriding the timeout_ns for empirical tuning. */
 529		if (idata->ic.data_timeout_ns)
 530			data.timeout_ns = idata->ic.data_timeout_ns;
 531
 
 
 
 
 
 
 
 
 
 
 
 
 
 532		mrq.data = &data;
 533	}
 534
 535	mrq.cmd = &cmd;
 536
 537	err = mmc_blk_part_switch(card, target_part);
 538	if (err)
 539		return err;
 540
 541	if (idata->ic.is_acmd) {
 542		err = mmc_app_cmd(card->host, card);
 543		if (err)
 544			return err;
 545	}
 546
 547	if (idata->rpmb || prev_idata) {
 548		sbc.opcode = MMC_SET_BLOCK_COUNT;
 549		/*
 550		 * We don't do any blockcount validation because the max size
 551		 * may be increased by a future standard. We just copy the
 552		 * 'Reliable Write' bit here.
 553		 */
 554		sbc.arg = data.blocks | (idata->ic.write_flag & BIT(31));
 555		if (prev_idata)
 556			sbc.arg = prev_idata->ic.arg;
 557		sbc.flags = MMC_RSP_R1 | MMC_CMD_AC;
 558		mrq.sbc = &sbc;
 559	}
 560
 561	if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_SANITIZE_START) &&
 562	    (cmd.opcode == MMC_SWITCH))
 563		return mmc_sanitize(card, idata->ic.cmd_timeout_ms);
 564
 565	/* If it's an R1B response we need some more preparations. */
 566	busy_timeout_ms = idata->ic.cmd_timeout_ms ? : MMC_BLK_TIMEOUT_MS;
 567	r1b_resp = (cmd.flags & MMC_RSP_R1B) == MMC_RSP_R1B;
 568	if (r1b_resp)
 569		mmc_prepare_busy_cmd(card->host, &cmd, busy_timeout_ms);
 570
 571	mmc_wait_for_req(card->host, &mrq);
 572	memcpy(&idata->ic.response, cmd.resp, sizeof(cmd.resp));
 573
 574	if (prev_idata) {
 575		memcpy(&prev_idata->ic.response, sbc.resp, sizeof(sbc.resp));
 576		if (sbc.error) {
 577			dev_err(mmc_dev(card->host), "%s: sbc error %d\n",
 578							__func__, sbc.error);
 579			return sbc.error;
 580		}
 581	}
 582
 583	if (cmd.error) {
 584		dev_err(mmc_dev(card->host), "%s: cmd error %d\n",
 585						__func__, cmd.error);
 586		return cmd.error;
 587	}
 588	if (data.error) {
 589		dev_err(mmc_dev(card->host), "%s: data error %d\n",
 590						__func__, data.error);
 591		return data.error;
 592	}
 593
 594	/*
 595	 * Make sure the cache of the PARTITION_CONFIG register and
 596	 * PARTITION_ACCESS bits is updated in case the ioctl ext_csd write
 597	 * changed it successfully.
 598	 */
 599	if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_PART_CONFIG) &&
 600	    (cmd.opcode == MMC_SWITCH)) {
 601		struct mmc_blk_data *main_md = dev_get_drvdata(&card->dev);
 602		u8 value = MMC_EXTRACT_VALUE_FROM_ARG(cmd.arg);
 603
 604		/*
 605		 * Update cache so the next mmc_blk_part_switch call operates
 606		 * on up-to-date data.
 607		 */
 608		card->ext_csd.part_config = value;
 609		main_md->part_curr = value & EXT_CSD_PART_CONFIG_ACC_MASK;
 610	}
 611
 612	/*
 613	 * Make sure to update CACHE_CTRL in case it was changed. The cache
 614	 * will get turned back on if the card is re-initialized, e.g.
 615	 * suspend/resume or hw reset in recovery.
 616	 */
 617	if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_CACHE_CTRL) &&
 618	    (cmd.opcode == MMC_SWITCH)) {
 619		u8 value = MMC_EXTRACT_VALUE_FROM_ARG(cmd.arg) & 1;
 620
 621		card->ext_csd.cache_ctrl = value;
 622	}
 623
 624	/*
 625	 * According to the SD specs, some commands require a delay after
 626	 * issuing the command.
 627	 */
 628	if (idata->ic.postsleep_min_us)
 629		usleep_range(idata->ic.postsleep_min_us, idata->ic.postsleep_max_us);
 630
 631	if (mmc_host_is_spi(card->host)) {
 632		if (idata->ic.write_flag || r1b_resp || cmd.flags & MMC_RSP_SPI_BUSY)
 633			return mmc_spi_err_check(card);
 634		return err;
 635	}
 636
 637	/*
 638	 * Ensure RPMB, writes and R1B responses are completed by polling with
 639	 * CMD13. Note that, usually we don't need to poll when using HW busy
 640	 * detection, but here it's needed since some commands may indicate the
 641	 * error through the R1 status bits.
 642	 */
 643	if (idata->rpmb || idata->ic.write_flag || r1b_resp) {
 644		struct mmc_blk_busy_data cb_data = {
 645			.card = card,
 646		};
 647
 648		err = __mmc_poll_for_busy(card->host, 0, busy_timeout_ms,
 649					  &mmc_blk_busy_cb, &cb_data);
 650
 651		idata->ic.response[0] = cb_data.status;
 652	}
 653
 654	return err;
 655}
 656
 657static int mmc_blk_ioctl_cmd(struct mmc_blk_data *md,
 658			     struct mmc_ioc_cmd __user *ic_ptr,
 659			     struct mmc_rpmb_data *rpmb)
 660{
 661	struct mmc_blk_ioc_data *idata;
 662	struct mmc_blk_ioc_data *idatas[1];
 663	struct mmc_queue *mq;
 664	struct mmc_card *card;
 665	int err = 0, ioc_err = 0;
 666	struct request *req;
 667
 668	idata = mmc_blk_ioctl_copy_from_user(ic_ptr);
 669	if (IS_ERR(idata))
 670		return PTR_ERR(idata);
 671	/* This will be NULL on non-RPMB ioctl():s */
 672	idata->rpmb = rpmb;
 673
 674	card = md->queue.card;
 675	if (IS_ERR(card)) {
 676		err = PTR_ERR(card);
 677		goto cmd_done;
 678	}
 679
 680	/*
 681	 * Dispatch the ioctl() into the block request queue.
 682	 */
 683	mq = &md->queue;
 684	req = blk_mq_alloc_request(mq->queue,
 685		idata->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0);
 686	if (IS_ERR(req)) {
 687		err = PTR_ERR(req);
 688		goto cmd_done;
 689	}
 690	idatas[0] = idata;
 691	req_to_mmc_queue_req(req)->drv_op =
 692		rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;
 693	req_to_mmc_queue_req(req)->drv_op_result = -EIO;
 694	req_to_mmc_queue_req(req)->drv_op_data = idatas;
 695	req_to_mmc_queue_req(req)->ioc_count = 1;
 696	blk_execute_rq(req, false);
 697	ioc_err = req_to_mmc_queue_req(req)->drv_op_result;
 698	err = mmc_blk_ioctl_copy_to_user(ic_ptr, idata);
 699	blk_mq_free_request(req);
 700
 701cmd_done:
 702	kfree(idata->buf);
 703	kfree(idata);
 704	return ioc_err ? ioc_err : err;
 705}
 706
 707static int mmc_blk_ioctl_multi_cmd(struct mmc_blk_data *md,
 708				   struct mmc_ioc_multi_cmd __user *user,
 709				   struct mmc_rpmb_data *rpmb)
 710{
 711	struct mmc_blk_ioc_data **idata = NULL;
 712	struct mmc_ioc_cmd __user *cmds = user->cmds;
 713	struct mmc_card *card;
 714	struct mmc_queue *mq;
 715	int err = 0, ioc_err = 0;
 716	__u64 num_of_cmds;
 717	unsigned int i, n;
 718	struct request *req;
 719
 720	if (copy_from_user(&num_of_cmds, &user->num_of_cmds,
 721			   sizeof(num_of_cmds)))
 722		return -EFAULT;
 723
 724	if (!num_of_cmds)
 725		return 0;
 726
 727	if (num_of_cmds > MMC_IOC_MAX_CMDS)
 728		return -EINVAL;
 729
 730	n = num_of_cmds;
 731	idata = kcalloc(n, sizeof(*idata), GFP_KERNEL);
 732	if (!idata)
 733		return -ENOMEM;
 734
 735	for (i = 0; i < n; i++) {
 736		idata[i] = mmc_blk_ioctl_copy_from_user(&cmds[i]);
 737		if (IS_ERR(idata[i])) {
 738			err = PTR_ERR(idata[i]);
 739			n = i;
 740			goto cmd_err;
 741		}
 742		/* This will be NULL on non-RPMB ioctl():s */
 743		idata[i]->rpmb = rpmb;
 744	}
 745
 746	card = md->queue.card;
 747	if (IS_ERR(card)) {
 748		err = PTR_ERR(card);
 749		goto cmd_err;
 750	}
 751
 752
 753	/*
 754	 * Dispatch the ioctl()s into the block request queue.
 755	 */
 756	mq = &md->queue;
 757	req = blk_mq_alloc_request(mq->queue,
 758		idata[0]->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0);
 759	if (IS_ERR(req)) {
 760		err = PTR_ERR(req);
 761		goto cmd_err;
 762	}
 763	req_to_mmc_queue_req(req)->drv_op =
 764		rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;
 765	req_to_mmc_queue_req(req)->drv_op_result = -EIO;
 766	req_to_mmc_queue_req(req)->drv_op_data = idata;
 767	req_to_mmc_queue_req(req)->ioc_count = n;
 768	blk_execute_rq(req, false);
 769	ioc_err = req_to_mmc_queue_req(req)->drv_op_result;
 770
 771	/* copy to user if data and response */
 772	for (i = 0; i < n && !err; i++)
 773		err = mmc_blk_ioctl_copy_to_user(&cmds[i], idata[i]);
 774
 775	blk_mq_free_request(req);
 776
 777cmd_err:
 778	for (i = 0; i < n; i++) {
 779		kfree(idata[i]->buf);
 780		kfree(idata[i]);
 781	}
 782	kfree(idata);
 783	return ioc_err ? ioc_err : err;
 784}
 785
 786static int mmc_blk_check_blkdev(struct block_device *bdev)
 787{
 788	/*
 789	 * The caller must have CAP_SYS_RAWIO, and must be calling this on the
 790	 * whole block device, not on a partition.  This prevents overspray
 791	 * between sibling partitions.
 792	 */
 793	if (!capable(CAP_SYS_RAWIO) || bdev_is_partition(bdev))
 794		return -EPERM;
 795	return 0;
 796}
 797
 798static int mmc_blk_ioctl(struct block_device *bdev, blk_mode_t mode,
 799	unsigned int cmd, unsigned long arg)
 800{
 801	struct mmc_blk_data *md;
 802	int ret;
 803
 804	switch (cmd) {
 805	case MMC_IOC_CMD:
 806		ret = mmc_blk_check_blkdev(bdev);
 807		if (ret)
 808			return ret;
 809		md = mmc_blk_get(bdev->bd_disk);
 810		if (!md)
 811			return -EINVAL;
 812		ret = mmc_blk_ioctl_cmd(md,
 813					(struct mmc_ioc_cmd __user *)arg,
 814					NULL);
 815		mmc_blk_put(md);
 816		return ret;
 817	case MMC_IOC_MULTI_CMD:
 818		ret = mmc_blk_check_blkdev(bdev);
 819		if (ret)
 820			return ret;
 821		md = mmc_blk_get(bdev->bd_disk);
 822		if (!md)
 823			return -EINVAL;
 824		ret = mmc_blk_ioctl_multi_cmd(md,
 825					(struct mmc_ioc_multi_cmd __user *)arg,
 826					NULL);
 827		mmc_blk_put(md);
 828		return ret;
 829	default:
 830		return -EINVAL;
 831	}
 832}
 833
 834#ifdef CONFIG_COMPAT
 835static int mmc_blk_compat_ioctl(struct block_device *bdev, blk_mode_t mode,
 836	unsigned int cmd, unsigned long arg)
 837{
 838	return mmc_blk_ioctl(bdev, mode, cmd, (unsigned long) compat_ptr(arg));
 839}
 840#endif
 841
 842static int mmc_blk_alternative_gpt_sector(struct gendisk *disk,
 843					  sector_t *sector)
 844{
 845	struct mmc_blk_data *md;
 846	int ret;
 847
 848	md = mmc_blk_get(disk);
 849	if (!md)
 850		return -EINVAL;
 851
 852	if (md->queue.card)
 853		ret = mmc_card_alternative_gpt_sector(md->queue.card, sector);
 854	else
 855		ret = -ENODEV;
 856
 857	mmc_blk_put(md);
 858
 859	return ret;
 860}
 861
 862static const struct block_device_operations mmc_bdops = {
 863	.open			= mmc_blk_open,
 864	.release		= mmc_blk_release,
 865	.getgeo			= mmc_blk_getgeo,
 866	.owner			= THIS_MODULE,
 867	.ioctl			= mmc_blk_ioctl,
 868#ifdef CONFIG_COMPAT
 869	.compat_ioctl		= mmc_blk_compat_ioctl,
 870#endif
 871	.alternative_gpt_sector	= mmc_blk_alternative_gpt_sector,
 872};
 873
 874static int mmc_blk_part_switch_pre(struct mmc_card *card,
 875				   unsigned int part_type)
 876{
 877	const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_RPMB;
 878	int ret = 0;
 879
 880	if ((part_type & mask) == mask) {
 881		if (card->ext_csd.cmdq_en) {
 882			ret = mmc_cmdq_disable(card);
 883			if (ret)
 884				return ret;
 885		}
 886		mmc_retune_pause(card->host);
 887	}
 888
 889	return ret;
 890}
 891
 892static int mmc_blk_part_switch_post(struct mmc_card *card,
 893				    unsigned int part_type)
 894{
 895	const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_RPMB;
 896	int ret = 0;
 897
 898	if ((part_type & mask) == mask) {
 899		mmc_retune_unpause(card->host);
 900		if (card->reenable_cmdq && !card->ext_csd.cmdq_en)
 901			ret = mmc_cmdq_enable(card);
 902	}
 903
 904	return ret;
 905}
 906
 907static inline int mmc_blk_part_switch(struct mmc_card *card,
 908				      unsigned int part_type)
 909{
 910	int ret = 0;
 911	struct mmc_blk_data *main_md = dev_get_drvdata(&card->dev);
 912
 913	if (main_md->part_curr == part_type)
 914		return 0;
 915
 916	if (mmc_card_mmc(card)) {
 917		u8 part_config = card->ext_csd.part_config;
 918
 919		ret = mmc_blk_part_switch_pre(card, part_type);
 920		if (ret)
 921			return ret;
 922
 923		part_config &= ~EXT_CSD_PART_CONFIG_ACC_MASK;
 924		part_config |= part_type;
 925
 926		ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
 927				 EXT_CSD_PART_CONFIG, part_config,
 928				 card->ext_csd.part_time);
 929		if (ret) {
 930			mmc_blk_part_switch_post(card, part_type);
 931			return ret;
 932		}
 933
 934		card->ext_csd.part_config = part_config;
 935
 936		ret = mmc_blk_part_switch_post(card, main_md->part_curr);
 937	}
 938
 939	main_md->part_curr = part_type;
 940	return ret;
 941}
 942
 943static int mmc_sd_num_wr_blocks(struct mmc_card *card, u32 *written_blocks)
 944{
 945	int err;
 946	u32 result;
 947	__be32 *blocks;
 948
 949	struct mmc_request mrq = {};
 950	struct mmc_command cmd = {};
 951	struct mmc_data data = {};
 952
 953	struct scatterlist sg;
 954
 955	err = mmc_app_cmd(card->host, card);
 
 
 
 
 956	if (err)
 957		return err;
 
 
 
 
 958
 959	cmd.opcode = SD_APP_SEND_NUM_WR_BLKS;
 960	cmd.arg = 0;
 961	cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_ADTC;
 962
 963	data.blksz = 4;
 964	data.blocks = 1;
 965	data.flags = MMC_DATA_READ;
 966	data.sg = &sg;
 967	data.sg_len = 1;
 968	mmc_set_data_timeout(&data, card);
 969
 970	mrq.cmd = &cmd;
 971	mrq.data = &data;
 972
 973	blocks = kmalloc(4, GFP_KERNEL);
 974	if (!blocks)
 975		return -ENOMEM;
 976
 977	sg_init_one(&sg, blocks, 4);
 978
 979	mmc_wait_for_req(card->host, &mrq);
 980
 981	result = ntohl(*blocks);
 982	kfree(blocks);
 983
 984	if (cmd.error || data.error)
 985		return -EIO;
 986
 987	*written_blocks = result;
 988
 989	return 0;
 990}
 991
 992static unsigned int mmc_blk_clock_khz(struct mmc_host *host)
 993{
 994	if (host->actual_clock)
 995		return host->actual_clock / 1000;
 996
 997	/* Clock may be subject to a divisor, fudge it by a factor of 2. */
 998	if (host->ios.clock)
 999		return host->ios.clock / 2000;
1000
1001	/* How can there be no clock */
1002	WARN_ON_ONCE(1);
1003	return 100; /* 100 kHz is minimum possible value */
1004}
1005
1006static unsigned int mmc_blk_data_timeout_ms(struct mmc_host *host,
1007					    struct mmc_data *data)
1008{
1009	unsigned int ms = DIV_ROUND_UP(data->timeout_ns, 1000000);
1010	unsigned int khz;
1011
1012	if (data->timeout_clks) {
1013		khz = mmc_blk_clock_khz(host);
1014		ms += DIV_ROUND_UP(data->timeout_clks, khz);
1015	}
1016
1017	return ms;
1018}
1019
1020/*
1021 * Attempts to reset the card and get back to the requested partition.
1022 * Therefore any error here must result in cancelling the block layer
1023 * request, it must not be reattempted without going through the mmc_blk
1024 * partition sanity checks.
1025 */
1026static int mmc_blk_reset(struct mmc_blk_data *md, struct mmc_host *host,
1027			 int type)
1028{
1029	int err;
1030	struct mmc_blk_data *main_md = dev_get_drvdata(&host->card->dev);
1031
1032	if (md->reset_done & type)
1033		return -EEXIST;
1034
1035	md->reset_done |= type;
1036	err = mmc_hw_reset(host->card);
1037	/*
1038	 * A successful reset will leave the card in the main partition, but
1039	 * upon failure it might not be, so set it to MMC_BLK_PART_INVALID
1040	 * in that case.
1041	 */
1042	main_md->part_curr = err ? MMC_BLK_PART_INVALID : main_md->part_type;
1043	if (err)
1044		return err;
1045	/* Ensure we switch back to the correct partition */
1046	if (mmc_blk_part_switch(host->card, md->part_type))
1047		/*
1048		 * We have failed to get back into the correct
1049		 * partition, so we need to abort the whole request.
1050		 */
1051		return -ENODEV;
1052	return 0;
 
 
 
 
 
 
 
 
 
1053}
1054
1055static inline void mmc_blk_reset_success(struct mmc_blk_data *md, int type)
1056{
1057	md->reset_done &= ~type;
1058}
1059
1060static void mmc_blk_check_sbc(struct mmc_queue_req *mq_rq)
1061{
1062	struct mmc_blk_ioc_data **idata = mq_rq->drv_op_data;
1063	int i;
1064
1065	for (i = 1; i < mq_rq->ioc_count; i++) {
1066		if (idata[i - 1]->ic.opcode == MMC_SET_BLOCK_COUNT &&
1067		    mmc_op_multi(idata[i]->ic.opcode)) {
1068			idata[i - 1]->flags |= MMC_BLK_IOC_DROP;
1069			idata[i]->flags |= MMC_BLK_IOC_SBC;
1070		}
1071	}
1072}
1073
1074/*
1075 * The non-block commands come back from the block layer after it queued it and
1076 * processed it with all other requests and then they get issued in this
1077 * function.
1078 */
1079static void mmc_blk_issue_drv_op(struct mmc_queue *mq, struct request *req)
1080{
1081	struct mmc_queue_req *mq_rq;
1082	struct mmc_card *card = mq->card;
1083	struct mmc_blk_data *md = mq->blkdata;
1084	struct mmc_blk_ioc_data **idata;
1085	bool rpmb_ioctl;
1086	u8 **ext_csd;
1087	u32 status;
1088	int ret;
1089	int i;
1090
1091	mq_rq = req_to_mmc_queue_req(req);
1092	rpmb_ioctl = (mq_rq->drv_op == MMC_DRV_OP_IOCTL_RPMB);
1093
1094	switch (mq_rq->drv_op) {
1095	case MMC_DRV_OP_IOCTL:
1096		if (card->ext_csd.cmdq_en) {
1097			ret = mmc_cmdq_disable(card);
1098			if (ret)
1099				break;
1100		}
1101
1102		mmc_blk_check_sbc(mq_rq);
1103
1104		fallthrough;
1105	case MMC_DRV_OP_IOCTL_RPMB:
1106		idata = mq_rq->drv_op_data;
1107		for (i = 0, ret = 0; i < mq_rq->ioc_count; i++) {
1108			ret = __mmc_blk_ioctl_cmd(card, md, idata, i);
1109			if (ret)
1110				break;
1111		}
1112		/* Always switch back to main area after RPMB access */
1113		if (rpmb_ioctl)
1114			mmc_blk_part_switch(card, 0);
1115		else if (card->reenable_cmdq && !card->ext_csd.cmdq_en)
1116			mmc_cmdq_enable(card);
1117		break;
1118	case MMC_DRV_OP_BOOT_WP:
1119		ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_BOOT_WP,
1120				 card->ext_csd.boot_ro_lock |
1121				 EXT_CSD_BOOT_WP_B_PWR_WP_EN,
1122				 card->ext_csd.part_time);
1123		if (ret)
1124			pr_err("%s: Locking boot partition ro until next power on failed: %d\n",
1125			       md->disk->disk_name, ret);
1126		else
1127			card->ext_csd.boot_ro_lock |=
1128				EXT_CSD_BOOT_WP_B_PWR_WP_EN;
1129		break;
1130	case MMC_DRV_OP_GET_CARD_STATUS:
1131		ret = mmc_send_status(card, &status);
1132		if (!ret)
1133			ret = status;
1134		break;
1135	case MMC_DRV_OP_GET_EXT_CSD:
1136		ext_csd = mq_rq->drv_op_data;
1137		ret = mmc_get_ext_csd(card, ext_csd);
1138		break;
1139	default:
1140		pr_err("%s: unknown driver specific operation\n",
1141		       md->disk->disk_name);
1142		ret = -EINVAL;
1143		break;
1144	}
1145	mq_rq->drv_op_result = ret;
1146	blk_mq_end_request(req, ret ? BLK_STS_IOERR : BLK_STS_OK);
1147}
1148
1149static void mmc_blk_issue_erase_rq(struct mmc_queue *mq, struct request *req,
1150				   int type, unsigned int erase_arg)
1151{
1152	struct mmc_blk_data *md = mq->blkdata;
1153	struct mmc_card *card = md->queue.card;
1154	unsigned int from, nr;
1155	int err = 0;
1156	blk_status_t status = BLK_STS_OK;
1157
1158	if (!mmc_can_erase(card)) {
1159		status = BLK_STS_NOTSUPP;
1160		goto fail;
1161	}
1162
1163	from = blk_rq_pos(req);
1164	nr = blk_rq_sectors(req);
1165
1166	do {
1167		err = 0;
1168		if (card->quirks & MMC_QUIRK_INAND_CMD38) {
1169			err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
1170					 INAND_CMD38_ARG_EXT_CSD,
1171					 erase_arg == MMC_TRIM_ARG ?
1172					 INAND_CMD38_ARG_TRIM :
1173					 INAND_CMD38_ARG_ERASE,
1174					 card->ext_csd.generic_cmd6_time);
1175		}
1176		if (!err)
1177			err = mmc_erase(card, from, nr, erase_arg);
1178	} while (err == -EIO && !mmc_blk_reset(md, card->host, type));
1179	if (err)
1180		status = BLK_STS_IOERR;
1181	else
1182		mmc_blk_reset_success(md, type);
1183fail:
1184	blk_mq_end_request(req, status);
1185}
1186
1187static void mmc_blk_issue_trim_rq(struct mmc_queue *mq, struct request *req)
1188{
1189	mmc_blk_issue_erase_rq(mq, req, MMC_BLK_TRIM, MMC_TRIM_ARG);
1190}
1191
1192static void mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req)
1193{
1194	struct mmc_blk_data *md = mq->blkdata;
1195	struct mmc_card *card = md->queue.card;
1196	unsigned int arg = card->erase_arg;
1197
1198	if (mmc_card_broken_sd_discard(card))
1199		arg = SD_ERASE_ARG;
1200
1201	mmc_blk_issue_erase_rq(mq, req, MMC_BLK_DISCARD, arg);
1202}
1203
1204static void mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,
1205				       struct request *req)
1206{
1207	struct mmc_blk_data *md = mq->blkdata;
1208	struct mmc_card *card = md->queue.card;
1209	unsigned int from, nr, arg;
1210	int err = 0, type = MMC_BLK_SECDISCARD;
1211	blk_status_t status = BLK_STS_OK;
1212
1213	if (!(mmc_can_secure_erase_trim(card))) {
1214		status = BLK_STS_NOTSUPP;
1215		goto out;
1216	}
1217
1218	from = blk_rq_pos(req);
1219	nr = blk_rq_sectors(req);
1220
1221	if (mmc_can_trim(card) && !mmc_erase_group_aligned(card, from, nr))
1222		arg = MMC_SECURE_TRIM1_ARG;
1223	else
1224		arg = MMC_SECURE_ERASE_ARG;
1225
1226retry:
1227	if (card->quirks & MMC_QUIRK_INAND_CMD38) {
1228		err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
1229				 INAND_CMD38_ARG_EXT_CSD,
1230				 arg == MMC_SECURE_TRIM1_ARG ?
1231				 INAND_CMD38_ARG_SECTRIM1 :
1232				 INAND_CMD38_ARG_SECERASE,
1233				 card->ext_csd.generic_cmd6_time);
1234		if (err)
1235			goto out_retry;
1236	}
1237
1238	err = mmc_erase(card, from, nr, arg);
1239	if (err == -EIO)
1240		goto out_retry;
1241	if (err) {
1242		status = BLK_STS_IOERR;
1243		goto out;
1244	}
1245
1246	if (arg == MMC_SECURE_TRIM1_ARG) {
1247		if (card->quirks & MMC_QUIRK_INAND_CMD38) {
1248			err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
1249					 INAND_CMD38_ARG_EXT_CSD,
1250					 INAND_CMD38_ARG_SECTRIM2,
1251					 card->ext_csd.generic_cmd6_time);
1252			if (err)
1253				goto out_retry;
1254		}
1255
1256		err = mmc_erase(card, from, nr, MMC_SECURE_TRIM2_ARG);
1257		if (err == -EIO)
1258			goto out_retry;
1259		if (err) {
1260			status = BLK_STS_IOERR;
1261			goto out;
1262		}
1263	}
1264
1265out_retry:
1266	if (err && !mmc_blk_reset(md, card->host, type))
1267		goto retry;
1268	if (!err)
1269		mmc_blk_reset_success(md, type);
1270out:
1271	blk_mq_end_request(req, status);
1272}
1273
1274static void mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req)
1275{
1276	struct mmc_blk_data *md = mq->blkdata;
1277	struct mmc_card *card = md->queue.card;
1278	int ret = 0;
1279
1280	ret = mmc_flush_cache(card->host);
1281	blk_mq_end_request(req, ret ? BLK_STS_IOERR : BLK_STS_OK);
1282}
1283
1284/*
1285 * Reformat current write as a reliable write, supporting
1286 * both legacy and the enhanced reliable write MMC cards.
1287 * In each transfer we'll handle only as much as a single
1288 * reliable write can handle, thus finish the request in
1289 * partial completions.
1290 */
1291static inline void mmc_apply_rel_rw(struct mmc_blk_request *brq,
1292				    struct mmc_card *card,
1293				    struct request *req)
1294{
1295	if (!(card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN)) {
1296		/* Legacy mode imposes restrictions on transfers. */
1297		if (!IS_ALIGNED(blk_rq_pos(req), card->ext_csd.rel_sectors))
1298			brq->data.blocks = 1;
1299
1300		if (brq->data.blocks > card->ext_csd.rel_sectors)
1301			brq->data.blocks = card->ext_csd.rel_sectors;
1302		else if (brq->data.blocks < card->ext_csd.rel_sectors)
1303			brq->data.blocks = 1;
1304	}
1305}
1306
1307#define CMD_ERRORS_EXCL_OOR						\
1308	(R1_ADDRESS_ERROR |	/* Misaligned address */		\
1309	 R1_BLOCK_LEN_ERROR |	/* Transferred block length incorrect */\
1310	 R1_WP_VIOLATION |	/* Tried to write to protected block */	\
1311	 R1_CARD_ECC_FAILED |	/* Card ECC failed */			\
1312	 R1_CC_ERROR |		/* Card controller error */		\
1313	 R1_ERROR)		/* General/unknown error */
1314
1315#define CMD_ERRORS							\
1316	(CMD_ERRORS_EXCL_OOR |						\
1317	 R1_OUT_OF_RANGE)	/* Command argument out of range */	\
1318
1319static void mmc_blk_eval_resp_error(struct mmc_blk_request *brq)
1320{
1321	u32 val;
1322
1323	/*
1324	 * Per the SD specification(physical layer version 4.10)[1],
1325	 * section 4.3.3, it explicitly states that "When the last
1326	 * block of user area is read using CMD18, the host should
1327	 * ignore OUT_OF_RANGE error that may occur even the sequence
1328	 * is correct". And JESD84-B51 for eMMC also has a similar
1329	 * statement on section 6.8.3.
1330	 *
1331	 * Multiple block read/write could be done by either predefined
1332	 * method, namely CMD23, or open-ending mode. For open-ending mode,
1333	 * we should ignore the OUT_OF_RANGE error as it's normal behaviour.
1334	 *
1335	 * However the spec[1] doesn't tell us whether we should also
1336	 * ignore that for predefined method. But per the spec[1], section
1337	 * 4.15 Set Block Count Command, it says"If illegal block count
1338	 * is set, out of range error will be indicated during read/write
1339	 * operation (For example, data transfer is stopped at user area
1340	 * boundary)." In another word, we could expect a out of range error
1341	 * in the response for the following CMD18/25. And if argument of
1342	 * CMD23 + the argument of CMD18/25 exceed the max number of blocks,
1343	 * we could also expect to get a -ETIMEDOUT or any error number from
1344	 * the host drivers due to missing data response(for write)/data(for
1345	 * read), as the cards will stop the data transfer by itself per the
1346	 * spec. So we only need to check R1_OUT_OF_RANGE for open-ending mode.
1347	 */
1348
1349	if (!brq->stop.error) {
1350		bool oor_with_open_end;
1351		/* If there is no error yet, check R1 response */
1352
1353		val = brq->stop.resp[0] & CMD_ERRORS;
1354		oor_with_open_end = val & R1_OUT_OF_RANGE && !brq->mrq.sbc;
1355
1356		if (val && !oor_with_open_end)
1357			brq->stop.error = -EIO;
1358	}
1359}
1360
1361static void mmc_blk_data_prep(struct mmc_queue *mq, struct mmc_queue_req *mqrq,
1362			      int recovery_mode, bool *do_rel_wr_p,
1363			      bool *do_data_tag_p)
1364{
1365	struct mmc_blk_data *md = mq->blkdata;
1366	struct mmc_card *card = md->queue.card;
1367	struct mmc_blk_request *brq = &mqrq->brq;
1368	struct request *req = mmc_queue_req_to_req(mqrq);
1369	bool do_rel_wr, do_data_tag;
1370
1371	/*
1372	 * Reliable writes are used to implement Forced Unit Access and
1373	 * are supported only on MMCs.
1374	 */
1375	do_rel_wr = (req->cmd_flags & REQ_FUA) &&
1376		    rq_data_dir(req) == WRITE &&
1377		    (md->flags & MMC_BLK_REL_WR);
1378
1379	memset(brq, 0, sizeof(struct mmc_blk_request));
1380
1381	mmc_crypto_prepare_req(mqrq);
1382
1383	brq->mrq.data = &brq->data;
1384	brq->mrq.tag = req->tag;
1385
1386	brq->stop.opcode = MMC_STOP_TRANSMISSION;
1387	brq->stop.arg = 0;
1388
1389	if (rq_data_dir(req) == READ) {
1390		brq->data.flags = MMC_DATA_READ;
1391		brq->stop.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC;
1392	} else {
1393		brq->data.flags = MMC_DATA_WRITE;
1394		brq->stop.flags = MMC_RSP_SPI_R1B | MMC_RSP_R1B | MMC_CMD_AC;
1395	}
1396
1397	brq->data.blksz = 512;
1398	brq->data.blocks = blk_rq_sectors(req);
1399	brq->data.blk_addr = blk_rq_pos(req);
1400
1401	/*
1402	 * The command queue supports 2 priorities: "high" (1) and "simple" (0).
1403	 * The eMMC will give "high" priority tasks priority over "simple"
1404	 * priority tasks. Here we always set "simple" priority by not setting
1405	 * MMC_DATA_PRIO.
1406	 */
1407
1408	/*
1409	 * The block layer doesn't support all sector count
1410	 * restrictions, so we need to be prepared for too big
1411	 * requests.
1412	 */
1413	if (brq->data.blocks > card->host->max_blk_count)
1414		brq->data.blocks = card->host->max_blk_count;
1415
1416	if (brq->data.blocks > 1) {
1417		/*
1418		 * Some SD cards in SPI mode return a CRC error or even lock up
1419		 * completely when trying to read the last block using a
1420		 * multiblock read command.
1421		 */
1422		if (mmc_host_is_spi(card->host) && (rq_data_dir(req) == READ) &&
1423		    (blk_rq_pos(req) + blk_rq_sectors(req) ==
1424		     get_capacity(md->disk)))
1425			brq->data.blocks--;
1426
1427		/*
1428		 * After a read error, we redo the request one (native) sector
1429		 * at a time in order to accurately determine which
1430		 * sectors can be read successfully.
1431		 */
1432		if (recovery_mode)
1433			brq->data.blocks = queue_physical_block_size(mq->queue) >> 9;
1434
1435		/*
1436		 * Some controllers have HW issues while operating
1437		 * in multiple I/O mode
1438		 */
1439		if (card->host->ops->multi_io_quirk)
1440			brq->data.blocks = card->host->ops->multi_io_quirk(card,
1441						(rq_data_dir(req) == READ) ?
1442						MMC_DATA_READ : MMC_DATA_WRITE,
1443						brq->data.blocks);
1444	}
1445
1446	if (do_rel_wr) {
1447		mmc_apply_rel_rw(brq, card, req);
1448		brq->data.flags |= MMC_DATA_REL_WR;
1449	}
1450
1451	/*
1452	 * Data tag is used only during writing meta data to speed
1453	 * up write and any subsequent read of this meta data
1454	 */
1455	do_data_tag = card->ext_csd.data_tag_unit_size &&
1456		      (req->cmd_flags & REQ_META) &&
1457		      (rq_data_dir(req) == WRITE) &&
1458		      ((brq->data.blocks * brq->data.blksz) >=
1459		       card->ext_csd.data_tag_unit_size);
1460
1461	if (do_data_tag)
1462		brq->data.flags |= MMC_DATA_DAT_TAG;
1463
1464	mmc_set_data_timeout(&brq->data, card);
1465
1466	brq->data.sg = mqrq->sg;
1467	brq->data.sg_len = mmc_queue_map_sg(mq, mqrq);
1468
1469	/*
1470	 * Adjust the sg list so it is the same size as the
1471	 * request.
1472	 */
1473	if (brq->data.blocks != blk_rq_sectors(req)) {
1474		int i, data_size = brq->data.blocks << 9;
1475		struct scatterlist *sg;
1476
1477		for_each_sg(brq->data.sg, sg, brq->data.sg_len, i) {
1478			data_size -= sg->length;
1479			if (data_size <= 0) {
1480				sg->length += data_size;
1481				i++;
1482				break;
1483			}
1484		}
1485		brq->data.sg_len = i;
1486	}
1487
1488	if (do_rel_wr_p)
1489		*do_rel_wr_p = do_rel_wr;
1490
1491	if (do_data_tag_p)
1492		*do_data_tag_p = do_data_tag;
1493}
1494
1495#define MMC_CQE_RETRIES 2
1496
1497static void mmc_blk_cqe_complete_rq(struct mmc_queue *mq, struct request *req)
1498{
1499	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1500	struct mmc_request *mrq = &mqrq->brq.mrq;
1501	struct request_queue *q = req->q;
1502	struct mmc_host *host = mq->card->host;
1503	enum mmc_issue_type issue_type = mmc_issue_type(mq, req);
1504	unsigned long flags;
1505	bool put_card;
1506	int err;
1507
1508	mmc_cqe_post_req(host, mrq);
1509
1510	if (mrq->cmd && mrq->cmd->error)
1511		err = mrq->cmd->error;
1512	else if (mrq->data && mrq->data->error)
1513		err = mrq->data->error;
1514	else
1515		err = 0;
1516
1517	if (err) {
1518		if (mqrq->retries++ < MMC_CQE_RETRIES)
1519			blk_mq_requeue_request(req, true);
1520		else
1521			blk_mq_end_request(req, BLK_STS_IOERR);
1522	} else if (mrq->data) {
1523		if (blk_update_request(req, BLK_STS_OK, mrq->data->bytes_xfered))
1524			blk_mq_requeue_request(req, true);
1525		else
1526			__blk_mq_end_request(req, BLK_STS_OK);
1527	} else if (mq->in_recovery) {
1528		blk_mq_requeue_request(req, true);
1529	} else {
1530		blk_mq_end_request(req, BLK_STS_OK);
1531	}
1532
1533	spin_lock_irqsave(&mq->lock, flags);
1534
1535	mq->in_flight[issue_type] -= 1;
1536
1537	put_card = (mmc_tot_in_flight(mq) == 0);
1538
1539	mmc_cqe_check_busy(mq);
1540
1541	spin_unlock_irqrestore(&mq->lock, flags);
1542
1543	if (!mq->cqe_busy)
1544		blk_mq_run_hw_queues(q, true);
1545
1546	if (put_card)
1547		mmc_put_card(mq->card, &mq->ctx);
1548}
1549
1550void mmc_blk_cqe_recovery(struct mmc_queue *mq)
1551{
1552	struct mmc_card *card = mq->card;
1553	struct mmc_host *host = card->host;
1554	int err;
1555
1556	pr_debug("%s: CQE recovery start\n", mmc_hostname(host));
1557
1558	err = mmc_cqe_recovery(host);
1559	if (err)
1560		mmc_blk_reset(mq->blkdata, host, MMC_BLK_CQE_RECOVERY);
1561	mmc_blk_reset_success(mq->blkdata, MMC_BLK_CQE_RECOVERY);
 
1562
1563	pr_debug("%s: CQE recovery done\n", mmc_hostname(host));
1564}
1565
1566static void mmc_blk_cqe_req_done(struct mmc_request *mrq)
1567{
1568	struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req,
1569						  brq.mrq);
1570	struct request *req = mmc_queue_req_to_req(mqrq);
1571	struct request_queue *q = req->q;
1572	struct mmc_queue *mq = q->queuedata;
1573
1574	/*
1575	 * Block layer timeouts race with completions which means the normal
1576	 * completion path cannot be used during recovery.
1577	 */
1578	if (mq->in_recovery)
1579		mmc_blk_cqe_complete_rq(mq, req);
1580	else if (likely(!blk_should_fake_timeout(req->q)))
1581		blk_mq_complete_request(req);
1582}
1583
1584static int mmc_blk_cqe_start_req(struct mmc_host *host, struct mmc_request *mrq)
1585{
1586	mrq->done		= mmc_blk_cqe_req_done;
1587	mrq->recovery_notifier	= mmc_cqe_recovery_notifier;
1588
1589	return mmc_cqe_start_req(host, mrq);
1590}
1591
1592static struct mmc_request *mmc_blk_cqe_prep_dcmd(struct mmc_queue_req *mqrq,
1593						 struct request *req)
1594{
1595	struct mmc_blk_request *brq = &mqrq->brq;
1596
1597	memset(brq, 0, sizeof(*brq));
1598
1599	brq->mrq.cmd = &brq->cmd;
1600	brq->mrq.tag = req->tag;
1601
1602	return &brq->mrq;
1603}
1604
1605static int mmc_blk_cqe_issue_flush(struct mmc_queue *mq, struct request *req)
1606{
1607	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1608	struct mmc_request *mrq = mmc_blk_cqe_prep_dcmd(mqrq, req);
1609
1610	mrq->cmd->opcode = MMC_SWITCH;
1611	mrq->cmd->arg = (MMC_SWITCH_MODE_WRITE_BYTE << 24) |
1612			(EXT_CSD_FLUSH_CACHE << 16) |
1613			(1 << 8) |
1614			EXT_CSD_CMD_SET_NORMAL;
1615	mrq->cmd->flags = MMC_CMD_AC | MMC_RSP_R1B;
1616
1617	return mmc_blk_cqe_start_req(mq->card->host, mrq);
1618}
1619
1620static int mmc_blk_hsq_issue_rw_rq(struct mmc_queue *mq, struct request *req)
1621{
1622	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1623	struct mmc_host *host = mq->card->host;
1624	int err;
1625
1626	mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq);
1627	mqrq->brq.mrq.done = mmc_blk_hsq_req_done;
1628	mmc_pre_req(host, &mqrq->brq.mrq);
1629
1630	err = mmc_cqe_start_req(host, &mqrq->brq.mrq);
1631	if (err)
1632		mmc_post_req(host, &mqrq->brq.mrq, err);
1633
1634	return err;
1635}
1636
1637static int mmc_blk_cqe_issue_rw_rq(struct mmc_queue *mq, struct request *req)
1638{
1639	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1640	struct mmc_host *host = mq->card->host;
1641
1642	if (host->hsq_enabled)
1643		return mmc_blk_hsq_issue_rw_rq(mq, req);
1644
1645	mmc_blk_data_prep(mq, mqrq, 0, NULL, NULL);
1646
1647	return mmc_blk_cqe_start_req(mq->card->host, &mqrq->brq.mrq);
1648}
1649
1650static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
1651			       struct mmc_card *card,
1652			       int recovery_mode,
1653			       struct mmc_queue *mq)
1654{
1655	u32 readcmd, writecmd;
1656	struct mmc_blk_request *brq = &mqrq->brq;
1657	struct request *req = mmc_queue_req_to_req(mqrq);
1658	struct mmc_blk_data *md = mq->blkdata;
1659	bool do_rel_wr, do_data_tag;
1660
1661	mmc_blk_data_prep(mq, mqrq, recovery_mode, &do_rel_wr, &do_data_tag);
1662
1663	brq->mrq.cmd = &brq->cmd;
1664
1665	brq->cmd.arg = blk_rq_pos(req);
1666	if (!mmc_card_blockaddr(card))
1667		brq->cmd.arg <<= 9;
1668	brq->cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_ADTC;
1669
1670	if (brq->data.blocks > 1 || do_rel_wr) {
1671		/* SPI multiblock writes terminate using a special
1672		 * token, not a STOP_TRANSMISSION request.
1673		 */
1674		if (!mmc_host_is_spi(card->host) ||
1675		    rq_data_dir(req) == READ)
1676			brq->mrq.stop = &brq->stop;
1677		readcmd = MMC_READ_MULTIPLE_BLOCK;
1678		writecmd = MMC_WRITE_MULTIPLE_BLOCK;
1679	} else {
1680		brq->mrq.stop = NULL;
1681		readcmd = MMC_READ_SINGLE_BLOCK;
1682		writecmd = MMC_WRITE_BLOCK;
1683	}
1684	brq->cmd.opcode = rq_data_dir(req) == READ ? readcmd : writecmd;
1685
1686	/*
1687	 * Pre-defined multi-block transfers are preferable to
1688	 * open ended-ones (and necessary for reliable writes).
1689	 * However, it is not sufficient to just send CMD23,
1690	 * and avoid the final CMD12, as on an error condition
1691	 * CMD12 (stop) needs to be sent anyway. This, coupled
1692	 * with Auto-CMD23 enhancements provided by some
1693	 * hosts, means that the complexity of dealing
1694	 * with this is best left to the host. If CMD23 is
1695	 * supported by card and host, we'll fill sbc in and let
1696	 * the host deal with handling it correctly. This means
1697	 * that for hosts that don't expose MMC_CAP_CMD23, no
1698	 * change of behavior will be observed.
1699	 *
1700	 * N.B: Some MMC cards experience perf degradation.
1701	 * We'll avoid using CMD23-bounded multiblock writes for
1702	 * these, while retaining features like reliable writes.
1703	 */
1704	if ((md->flags & MMC_BLK_CMD23) && mmc_op_multi(brq->cmd.opcode) &&
1705	    (do_rel_wr || !(card->quirks & MMC_QUIRK_BLK_NO_CMD23) ||
1706	     do_data_tag)) {
1707		brq->sbc.opcode = MMC_SET_BLOCK_COUNT;
1708		brq->sbc.arg = brq->data.blocks |
1709			(do_rel_wr ? (1 << 31) : 0) |
1710			(do_data_tag ? (1 << 29) : 0);
1711		brq->sbc.flags = MMC_RSP_R1 | MMC_CMD_AC;
1712		brq->mrq.sbc = &brq->sbc;
1713	}
1714}
1715
1716#define MMC_MAX_RETRIES		5
1717#define MMC_DATA_RETRIES	2
1718#define MMC_NO_RETRIES		(MMC_MAX_RETRIES + 1)
1719
1720static int mmc_blk_send_stop(struct mmc_card *card, unsigned int timeout)
1721{
1722	struct mmc_command cmd = {
1723		.opcode = MMC_STOP_TRANSMISSION,
1724		.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC,
1725		/* Some hosts wait for busy anyway, so provide a busy timeout */
1726		.busy_timeout = timeout,
1727	};
1728
1729	return mmc_wait_for_cmd(card->host, &cmd, 5);
1730}
1731
1732static int mmc_blk_fix_state(struct mmc_card *card, struct request *req)
1733{
1734	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1735	struct mmc_blk_request *brq = &mqrq->brq;
1736	unsigned int timeout = mmc_blk_data_timeout_ms(card->host, &brq->data);
1737	int err;
1738
1739	mmc_retune_hold_now(card->host);
1740
1741	mmc_blk_send_stop(card, timeout);
1742
1743	err = mmc_poll_for_busy(card, timeout, false, MMC_BUSY_IO);
1744
1745	mmc_retune_release(card->host);
1746
1747	return err;
1748}
1749
1750#define MMC_READ_SINGLE_RETRIES	2
1751
1752/* Single (native) sector read during recovery */
1753static void mmc_blk_read_single(struct mmc_queue *mq, struct request *req)
1754{
1755	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1756	struct mmc_request *mrq = &mqrq->brq.mrq;
1757	struct mmc_card *card = mq->card;
1758	struct mmc_host *host = card->host;
1759	blk_status_t error = BLK_STS_OK;
1760	size_t bytes_per_read = queue_physical_block_size(mq->queue);
1761
1762	do {
1763		u32 status;
1764		int err;
1765		int retries = 0;
1766
1767		while (retries++ <= MMC_READ_SINGLE_RETRIES) {
1768			mmc_blk_rw_rq_prep(mqrq, card, 1, mq);
 
1769
1770			mmc_wait_for_req(host, mrq);
 
 
1771
1772			err = mmc_send_status(card, &status);
 
 
1773			if (err)
1774				goto error_exit;
 
1775
1776			if (!mmc_host_is_spi(host) &&
1777			    !mmc_ready_for_data(status)) {
1778				err = mmc_blk_fix_state(card, req);
1779				if (err)
1780					goto error_exit;
1781			}
1782
1783			if (!mrq->cmd->error)
1784				break;
1785		}
1786
1787		if (mrq->cmd->error ||
1788		    mrq->data->error ||
1789		    (!mmc_host_is_spi(host) &&
1790		     (mrq->cmd->resp[0] & CMD_ERRORS || status & CMD_ERRORS)))
1791			error = BLK_STS_IOERR;
1792		else
1793			error = BLK_STS_OK;
1794
1795	} while (blk_update_request(req, error, bytes_per_read));
1796
1797	return;
1798
1799error_exit:
1800	mrq->data->bytes_xfered = 0;
1801	blk_update_request(req, BLK_STS_IOERR, bytes_per_read);
1802	/* Let it try the remaining request again */
1803	if (mqrq->retries > MMC_MAX_RETRIES - 1)
1804		mqrq->retries = MMC_MAX_RETRIES - 1;
1805}
1806
1807static inline bool mmc_blk_oor_valid(struct mmc_blk_request *brq)
1808{
1809	return !!brq->mrq.sbc;
1810}
1811
1812static inline u32 mmc_blk_stop_err_bits(struct mmc_blk_request *brq)
1813{
1814	return mmc_blk_oor_valid(brq) ? CMD_ERRORS : CMD_ERRORS_EXCL_OOR;
1815}
1816
1817/*
1818 * Check for errors the host controller driver might not have seen such as
1819 * response mode errors or invalid card state.
1820 */
1821static bool mmc_blk_status_error(struct request *req, u32 status)
1822{
1823	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1824	struct mmc_blk_request *brq = &mqrq->brq;
1825	struct mmc_queue *mq = req->q->queuedata;
1826	u32 stop_err_bits;
1827
1828	if (mmc_host_is_spi(mq->card->host))
1829		return false;
1830
1831	stop_err_bits = mmc_blk_stop_err_bits(brq);
1832
1833	return brq->cmd.resp[0]  & CMD_ERRORS    ||
1834	       brq->stop.resp[0] & stop_err_bits ||
1835	       status            & stop_err_bits ||
1836	       (rq_data_dir(req) == WRITE && !mmc_ready_for_data(status));
1837}
1838
1839static inline bool mmc_blk_cmd_started(struct mmc_blk_request *brq)
1840{
1841	return !brq->sbc.error && !brq->cmd.error &&
1842	       !(brq->cmd.resp[0] & CMD_ERRORS);
1843}
1844
1845/*
1846 * Requests are completed by mmc_blk_mq_complete_rq() which sets simple
1847 * policy:
1848 * 1. A request that has transferred at least some data is considered
1849 * successful and will be requeued if there is remaining data to
1850 * transfer.
1851 * 2. Otherwise the number of retries is incremented and the request
1852 * will be requeued if there are remaining retries.
1853 * 3. Otherwise the request will be errored out.
1854 * That means mmc_blk_mq_complete_rq() is controlled by bytes_xfered and
1855 * mqrq->retries. So there are only 4 possible actions here:
1856 *	1. do not accept the bytes_xfered value i.e. set it to zero
1857 *	2. change mqrq->retries to determine the number of retries
1858 *	3. try to reset the card
1859 *	4. read one sector at a time
1860 */
1861static void mmc_blk_mq_rw_recovery(struct mmc_queue *mq, struct request *req)
1862{
1863	int type = rq_data_dir(req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE;
1864	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1865	struct mmc_blk_request *brq = &mqrq->brq;
1866	struct mmc_blk_data *md = mq->blkdata;
1867	struct mmc_card *card = mq->card;
1868	u32 status;
1869	u32 blocks;
1870	int err;
1871
1872	/*
1873	 * Some errors the host driver might not have seen. Set the number of
1874	 * bytes transferred to zero in that case.
1875	 */
1876	err = __mmc_send_status(card, &status, 0);
1877	if (err || mmc_blk_status_error(req, status))
1878		brq->data.bytes_xfered = 0;
1879
1880	mmc_retune_release(card->host);
1881
1882	/*
1883	 * Try again to get the status. This also provides an opportunity for
1884	 * re-tuning.
1885	 */
1886	if (err)
1887		err = __mmc_send_status(card, &status, 0);
1888
1889	/*
1890	 * Nothing more to do after the number of bytes transferred has been
1891	 * updated and there is no card.
1892	 */
1893	if (err && mmc_detect_card_removed(card->host))
1894		return;
1895
1896	/* Try to get back to "tran" state */
1897	if (!mmc_host_is_spi(mq->card->host) &&
1898	    (err || !mmc_ready_for_data(status)))
1899		err = mmc_blk_fix_state(mq->card, req);
1900
1901	/*
1902	 * Special case for SD cards where the card might record the number of
1903	 * blocks written.
1904	 */
1905	if (!err && mmc_blk_cmd_started(brq) && mmc_card_sd(card) &&
1906	    rq_data_dir(req) == WRITE) {
1907		if (mmc_sd_num_wr_blocks(card, &blocks))
1908			brq->data.bytes_xfered = 0;
1909		else
1910			brq->data.bytes_xfered = blocks << 9;
1911	}
1912
1913	/* Reset if the card is in a bad state */
1914	if (!mmc_host_is_spi(mq->card->host) &&
1915	    err && mmc_blk_reset(md, card->host, type)) {
1916		pr_err("%s: recovery failed!\n", req->q->disk->disk_name);
1917		mqrq->retries = MMC_NO_RETRIES;
1918		return;
1919	}
1920
1921	/*
1922	 * If anything was done, just return and if there is anything remaining
1923	 * on the request it will get requeued.
1924	 */
1925	if (brq->data.bytes_xfered)
1926		return;
1927
1928	/* Reset before last retry */
1929	if (mqrq->retries + 1 == MMC_MAX_RETRIES &&
1930	    mmc_blk_reset(md, card->host, type))
1931		return;
1932
1933	/* Command errors fail fast, so use all MMC_MAX_RETRIES */
1934	if (brq->sbc.error || brq->cmd.error)
1935		return;
1936
1937	/* Reduce the remaining retries for data errors */
1938	if (mqrq->retries < MMC_MAX_RETRIES - MMC_DATA_RETRIES) {
1939		mqrq->retries = MMC_MAX_RETRIES - MMC_DATA_RETRIES;
1940		return;
1941	}
1942
1943	if (rq_data_dir(req) == READ && brq->data.blocks >
1944			queue_physical_block_size(mq->queue) >> 9) {
1945		/* Read one (native) sector at a time */
 
1946		mmc_blk_read_single(mq, req);
1947		return;
1948	}
1949}
1950
1951static inline bool mmc_blk_rq_error(struct mmc_blk_request *brq)
1952{
1953	mmc_blk_eval_resp_error(brq);
1954
1955	return brq->sbc.error || brq->cmd.error || brq->stop.error ||
1956	       brq->data.error || brq->cmd.resp[0] & CMD_ERRORS;
1957}
1958
1959static int mmc_spi_err_check(struct mmc_card *card)
1960{
1961	u32 status = 0;
1962	int err;
1963
1964	/*
1965	 * SPI does not have a TRAN state we have to wait on, instead the
1966	 * card is ready again when it no longer holds the line LOW.
1967	 * We still have to ensure two things here before we know the write
1968	 * was successful:
1969	 * 1. The card has not disconnected during busy and we actually read our
1970	 * own pull-up, thinking it was still connected, so ensure it
1971	 * still responds.
1972	 * 2. Check for any error bits, in particular R1_SPI_IDLE to catch a
1973	 * just reconnected card after being disconnected during busy.
1974	 */
1975	err = __mmc_send_status(card, &status, 0);
1976	if (err)
1977		return err;
1978	/* All R1 and R2 bits of SPI are errors in our case */
1979	if (status)
1980		return -EIO;
1981	return 0;
1982}
1983
1984static int mmc_blk_busy_cb(void *cb_data, bool *busy)
1985{
1986	struct mmc_blk_busy_data *data = cb_data;
1987	u32 status = 0;
1988	int err;
1989
1990	err = mmc_send_status(data->card, &status);
1991	if (err)
1992		return err;
1993
1994	/* Accumulate response error bits. */
1995	data->status |= status;
1996
1997	*busy = !mmc_ready_for_data(status);
1998	return 0;
1999}
2000
2001static int mmc_blk_card_busy(struct mmc_card *card, struct request *req)
2002{
2003	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
2004	struct mmc_blk_busy_data cb_data;
2005	int err;
2006
2007	if (rq_data_dir(req) == READ)
2008		return 0;
2009
2010	if (mmc_host_is_spi(card->host)) {
2011		err = mmc_spi_err_check(card);
2012		if (err)
2013			mqrq->brq.data.bytes_xfered = 0;
2014		return err;
2015	}
2016
2017	cb_data.card = card;
2018	cb_data.status = 0;
2019	err = __mmc_poll_for_busy(card->host, 0, MMC_BLK_TIMEOUT_MS,
2020				  &mmc_blk_busy_cb, &cb_data);
2021
2022	/*
2023	 * Do not assume data transferred correctly if there are any error bits
2024	 * set.
2025	 */
2026	if (cb_data.status & mmc_blk_stop_err_bits(&mqrq->brq)) {
2027		mqrq->brq.data.bytes_xfered = 0;
2028		err = err ? err : -EIO;
2029	}
2030
2031	/* Copy the exception bit so it will be seen later on */
2032	if (mmc_card_mmc(card) && cb_data.status & R1_EXCEPTION_EVENT)
2033		mqrq->brq.cmd.resp[0] |= R1_EXCEPTION_EVENT;
2034
2035	return err;
2036}
2037
2038static inline void mmc_blk_rw_reset_success(struct mmc_queue *mq,
2039					    struct request *req)
2040{
2041	int type = rq_data_dir(req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE;
2042
2043	mmc_blk_reset_success(mq->blkdata, type);
2044}
2045
2046static void mmc_blk_mq_complete_rq(struct mmc_queue *mq, struct request *req)
2047{
2048	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
2049	unsigned int nr_bytes = mqrq->brq.data.bytes_xfered;
2050
2051	if (nr_bytes) {
2052		if (blk_update_request(req, BLK_STS_OK, nr_bytes))
2053			blk_mq_requeue_request(req, true);
2054		else
2055			__blk_mq_end_request(req, BLK_STS_OK);
2056	} else if (!blk_rq_bytes(req)) {
2057		__blk_mq_end_request(req, BLK_STS_IOERR);
2058	} else if (mqrq->retries++ < MMC_MAX_RETRIES) {
2059		blk_mq_requeue_request(req, true);
2060	} else {
2061		if (mmc_card_removed(mq->card))
2062			req->rq_flags |= RQF_QUIET;
2063		blk_mq_end_request(req, BLK_STS_IOERR);
2064	}
2065}
2066
2067static bool mmc_blk_urgent_bkops_needed(struct mmc_queue *mq,
2068					struct mmc_queue_req *mqrq)
2069{
2070	return mmc_card_mmc(mq->card) && !mmc_host_is_spi(mq->card->host) &&
2071	       (mqrq->brq.cmd.resp[0] & R1_EXCEPTION_EVENT ||
2072		mqrq->brq.stop.resp[0] & R1_EXCEPTION_EVENT);
2073}
2074
2075static void mmc_blk_urgent_bkops(struct mmc_queue *mq,
2076				 struct mmc_queue_req *mqrq)
2077{
2078	if (mmc_blk_urgent_bkops_needed(mq, mqrq))
2079		mmc_run_bkops(mq->card);
2080}
2081
2082static void mmc_blk_hsq_req_done(struct mmc_request *mrq)
2083{
2084	struct mmc_queue_req *mqrq =
2085		container_of(mrq, struct mmc_queue_req, brq.mrq);
2086	struct request *req = mmc_queue_req_to_req(mqrq);
2087	struct request_queue *q = req->q;
2088	struct mmc_queue *mq = q->queuedata;
2089	struct mmc_host *host = mq->card->host;
2090	unsigned long flags;
2091
2092	if (mmc_blk_rq_error(&mqrq->brq) ||
2093	    mmc_blk_urgent_bkops_needed(mq, mqrq)) {
2094		spin_lock_irqsave(&mq->lock, flags);
2095		mq->recovery_needed = true;
2096		mq->recovery_req = req;
2097		spin_unlock_irqrestore(&mq->lock, flags);
2098
2099		host->cqe_ops->cqe_recovery_start(host);
2100
2101		schedule_work(&mq->recovery_work);
2102		return;
2103	}
2104
2105	mmc_blk_rw_reset_success(mq, req);
2106
2107	/*
2108	 * Block layer timeouts race with completions which means the normal
2109	 * completion path cannot be used during recovery.
2110	 */
2111	if (mq->in_recovery)
2112		mmc_blk_cqe_complete_rq(mq, req);
2113	else if (likely(!blk_should_fake_timeout(req->q)))
2114		blk_mq_complete_request(req);
2115}
2116
2117void mmc_blk_mq_complete(struct request *req)
2118{
2119	struct mmc_queue *mq = req->q->queuedata;
2120	struct mmc_host *host = mq->card->host;
2121
2122	if (host->cqe_enabled)
2123		mmc_blk_cqe_complete_rq(mq, req);
2124	else if (likely(!blk_should_fake_timeout(req->q)))
2125		mmc_blk_mq_complete_rq(mq, req);
2126}
2127
2128static void mmc_blk_mq_poll_completion(struct mmc_queue *mq,
2129				       struct request *req)
2130{
2131	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
2132	struct mmc_host *host = mq->card->host;
2133
2134	if (mmc_blk_rq_error(&mqrq->brq) ||
2135	    mmc_blk_card_busy(mq->card, req)) {
2136		mmc_blk_mq_rw_recovery(mq, req);
2137	} else {
2138		mmc_blk_rw_reset_success(mq, req);
2139		mmc_retune_release(host);
2140	}
2141
2142	mmc_blk_urgent_bkops(mq, mqrq);
2143}
2144
2145static void mmc_blk_mq_dec_in_flight(struct mmc_queue *mq, enum mmc_issue_type issue_type)
2146{
2147	unsigned long flags;
2148	bool put_card;
2149
2150	spin_lock_irqsave(&mq->lock, flags);
2151
2152	mq->in_flight[issue_type] -= 1;
2153
2154	put_card = (mmc_tot_in_flight(mq) == 0);
2155
2156	spin_unlock_irqrestore(&mq->lock, flags);
2157
2158	if (put_card)
2159		mmc_put_card(mq->card, &mq->ctx);
2160}
2161
2162static void mmc_blk_mq_post_req(struct mmc_queue *mq, struct request *req,
2163				bool can_sleep)
2164{
2165	enum mmc_issue_type issue_type = mmc_issue_type(mq, req);
2166	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
2167	struct mmc_request *mrq = &mqrq->brq.mrq;
2168	struct mmc_host *host = mq->card->host;
2169
2170	mmc_post_req(host, mrq, 0);
2171
2172	/*
2173	 * Block layer timeouts race with completions which means the normal
2174	 * completion path cannot be used during recovery.
2175	 */
2176	if (mq->in_recovery) {
2177		mmc_blk_mq_complete_rq(mq, req);
2178	} else if (likely(!blk_should_fake_timeout(req->q))) {
2179		if (can_sleep)
2180			blk_mq_complete_request_direct(req, mmc_blk_mq_complete);
2181		else
2182			blk_mq_complete_request(req);
2183	}
2184
2185	mmc_blk_mq_dec_in_flight(mq, issue_type);
2186}
2187
2188void mmc_blk_mq_recovery(struct mmc_queue *mq)
2189{
2190	struct request *req = mq->recovery_req;
2191	struct mmc_host *host = mq->card->host;
2192	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
2193
2194	mq->recovery_req = NULL;
2195	mq->rw_wait = false;
2196
2197	if (mmc_blk_rq_error(&mqrq->brq)) {
2198		mmc_retune_hold_now(host);
2199		mmc_blk_mq_rw_recovery(mq, req);
2200	}
2201
2202	mmc_blk_urgent_bkops(mq, mqrq);
2203
2204	mmc_blk_mq_post_req(mq, req, true);
2205}
2206
2207static void mmc_blk_mq_complete_prev_req(struct mmc_queue *mq,
2208					 struct request **prev_req)
2209{
2210	if (mmc_host_done_complete(mq->card->host))
2211		return;
2212
2213	mutex_lock(&mq->complete_lock);
2214
2215	if (!mq->complete_req)
2216		goto out_unlock;
2217
2218	mmc_blk_mq_poll_completion(mq, mq->complete_req);
2219
2220	if (prev_req)
2221		*prev_req = mq->complete_req;
2222	else
2223		mmc_blk_mq_post_req(mq, mq->complete_req, true);
2224
2225	mq->complete_req = NULL;
2226
2227out_unlock:
2228	mutex_unlock(&mq->complete_lock);
2229}
2230
2231void mmc_blk_mq_complete_work(struct work_struct *work)
2232{
2233	struct mmc_queue *mq = container_of(work, struct mmc_queue,
2234					    complete_work);
2235
2236	mmc_blk_mq_complete_prev_req(mq, NULL);
2237}
2238
2239static void mmc_blk_mq_req_done(struct mmc_request *mrq)
2240{
2241	struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req,
2242						  brq.mrq);
2243	struct request *req = mmc_queue_req_to_req(mqrq);
2244	struct request_queue *q = req->q;
2245	struct mmc_queue *mq = q->queuedata;
2246	struct mmc_host *host = mq->card->host;
2247	unsigned long flags;
2248
2249	if (!mmc_host_done_complete(host)) {
2250		bool waiting;
2251
2252		/*
2253		 * We cannot complete the request in this context, so record
2254		 * that there is a request to complete, and that a following
2255		 * request does not need to wait (although it does need to
2256		 * complete complete_req first).
2257		 */
2258		spin_lock_irqsave(&mq->lock, flags);
2259		mq->complete_req = req;
2260		mq->rw_wait = false;
2261		waiting = mq->waiting;
2262		spin_unlock_irqrestore(&mq->lock, flags);
2263
2264		/*
2265		 * If 'waiting' then the waiting task will complete this
2266		 * request, otherwise queue a work to do it. Note that
2267		 * complete_work may still race with the dispatch of a following
2268		 * request.
2269		 */
2270		if (waiting)
2271			wake_up(&mq->wait);
2272		else
2273			queue_work(mq->card->complete_wq, &mq->complete_work);
2274
2275		return;
2276	}
2277
2278	/* Take the recovery path for errors or urgent background operations */
2279	if (mmc_blk_rq_error(&mqrq->brq) ||
2280	    mmc_blk_urgent_bkops_needed(mq, mqrq)) {
2281		spin_lock_irqsave(&mq->lock, flags);
2282		mq->recovery_needed = true;
2283		mq->recovery_req = req;
2284		spin_unlock_irqrestore(&mq->lock, flags);
2285		wake_up(&mq->wait);
2286		schedule_work(&mq->recovery_work);
2287		return;
2288	}
2289
2290	mmc_blk_rw_reset_success(mq, req);
2291
2292	mq->rw_wait = false;
2293	wake_up(&mq->wait);
2294
2295	/* context unknown */
2296	mmc_blk_mq_post_req(mq, req, false);
2297}
2298
2299static bool mmc_blk_rw_wait_cond(struct mmc_queue *mq, int *err)
2300{
2301	unsigned long flags;
2302	bool done;
2303
2304	/*
2305	 * Wait while there is another request in progress, but not if recovery
2306	 * is needed. Also indicate whether there is a request waiting to start.
2307	 */
2308	spin_lock_irqsave(&mq->lock, flags);
2309	if (mq->recovery_needed) {
2310		*err = -EBUSY;
2311		done = true;
2312	} else {
2313		done = !mq->rw_wait;
2314	}
2315	mq->waiting = !done;
2316	spin_unlock_irqrestore(&mq->lock, flags);
2317
2318	return done;
2319}
2320
2321static int mmc_blk_rw_wait(struct mmc_queue *mq, struct request **prev_req)
2322{
2323	int err = 0;
2324
2325	wait_event(mq->wait, mmc_blk_rw_wait_cond(mq, &err));
2326
2327	/* Always complete the previous request if there is one */
2328	mmc_blk_mq_complete_prev_req(mq, prev_req);
2329
2330	return err;
2331}
2332
2333static int mmc_blk_mq_issue_rw_rq(struct mmc_queue *mq,
2334				  struct request *req)
2335{
2336	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
2337	struct mmc_host *host = mq->card->host;
2338	struct request *prev_req = NULL;
2339	int err = 0;
2340
2341	mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq);
2342
2343	mqrq->brq.mrq.done = mmc_blk_mq_req_done;
2344
2345	mmc_pre_req(host, &mqrq->brq.mrq);
2346
2347	err = mmc_blk_rw_wait(mq, &prev_req);
2348	if (err)
2349		goto out_post_req;
2350
2351	mq->rw_wait = true;
2352
2353	err = mmc_start_request(host, &mqrq->brq.mrq);
2354
2355	if (prev_req)
2356		mmc_blk_mq_post_req(mq, prev_req, true);
2357
2358	if (err)
2359		mq->rw_wait = false;
2360
2361	/* Release re-tuning here where there is no synchronization required */
2362	if (err || mmc_host_done_complete(host))
2363		mmc_retune_release(host);
2364
2365out_post_req:
2366	if (err)
2367		mmc_post_req(host, &mqrq->brq.mrq, err);
2368
2369	return err;
2370}
2371
2372static int mmc_blk_wait_for_idle(struct mmc_queue *mq, struct mmc_host *host)
2373{
2374	if (host->cqe_enabled)
2375		return host->cqe_ops->cqe_wait_for_idle(host);
2376
2377	return mmc_blk_rw_wait(mq, NULL);
2378}
2379
2380enum mmc_issued mmc_blk_mq_issue_rq(struct mmc_queue *mq, struct request *req)
2381{
2382	struct mmc_blk_data *md = mq->blkdata;
2383	struct mmc_card *card = md->queue.card;
2384	struct mmc_host *host = card->host;
2385	int ret;
2386
2387	ret = mmc_blk_part_switch(card, md->part_type);
2388	if (ret)
2389		return MMC_REQ_FAILED_TO_START;
2390
2391	switch (mmc_issue_type(mq, req)) {
2392	case MMC_ISSUE_SYNC:
2393		ret = mmc_blk_wait_for_idle(mq, host);
2394		if (ret)
2395			return MMC_REQ_BUSY;
2396		switch (req_op(req)) {
2397		case REQ_OP_DRV_IN:
2398		case REQ_OP_DRV_OUT:
2399			mmc_blk_issue_drv_op(mq, req);
2400			break;
2401		case REQ_OP_DISCARD:
2402			mmc_blk_issue_discard_rq(mq, req);
2403			break;
2404		case REQ_OP_SECURE_ERASE:
2405			mmc_blk_issue_secdiscard_rq(mq, req);
2406			break;
2407		case REQ_OP_WRITE_ZEROES:
2408			mmc_blk_issue_trim_rq(mq, req);
2409			break;
2410		case REQ_OP_FLUSH:
2411			mmc_blk_issue_flush(mq, req);
2412			break;
2413		default:
2414			WARN_ON_ONCE(1);
2415			return MMC_REQ_FAILED_TO_START;
2416		}
2417		return MMC_REQ_FINISHED;
2418	case MMC_ISSUE_DCMD:
2419	case MMC_ISSUE_ASYNC:
2420		switch (req_op(req)) {
2421		case REQ_OP_FLUSH:
2422			if (!mmc_cache_enabled(host)) {
2423				blk_mq_end_request(req, BLK_STS_OK);
2424				return MMC_REQ_FINISHED;
2425			}
2426			ret = mmc_blk_cqe_issue_flush(mq, req);
2427			break;
2428		case REQ_OP_WRITE:
2429			card->written_flag = true;
2430			fallthrough;
2431		case REQ_OP_READ:
 
2432			if (host->cqe_enabled)
2433				ret = mmc_blk_cqe_issue_rw_rq(mq, req);
2434			else
2435				ret = mmc_blk_mq_issue_rw_rq(mq, req);
2436			break;
2437		default:
2438			WARN_ON_ONCE(1);
2439			ret = -EINVAL;
2440		}
2441		if (!ret)
2442			return MMC_REQ_STARTED;
2443		return ret == -EBUSY ? MMC_REQ_BUSY : MMC_REQ_FAILED_TO_START;
2444	default:
2445		WARN_ON_ONCE(1);
2446		return MMC_REQ_FAILED_TO_START;
2447	}
2448}
2449
2450static inline int mmc_blk_readonly(struct mmc_card *card)
2451{
2452	return mmc_card_readonly(card) ||
2453	       !(card->csd.cmdclass & CCC_BLOCK_WRITE);
2454}
2455
2456static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card,
2457					      struct device *parent,
2458					      sector_t size,
2459					      bool default_ro,
2460					      const char *subname,
2461					      int area_type,
2462					      unsigned int part_type)
2463{
2464	struct mmc_blk_data *md;
2465	int devidx, ret;
2466	char cap_str[10];
2467	bool cache_enabled = false;
2468	bool fua_enabled = false;
2469
2470	devidx = ida_simple_get(&mmc_blk_ida, 0, max_devices, GFP_KERNEL);
2471	if (devidx < 0) {
2472		/*
2473		 * We get -ENOSPC because there are no more any available
2474		 * devidx. The reason may be that, either userspace haven't yet
2475		 * unmounted the partitions, which postpones mmc_blk_release()
2476		 * from being called, or the device has more partitions than
2477		 * what we support.
2478		 */
2479		if (devidx == -ENOSPC)
2480			dev_err(mmc_dev(card->host),
2481				"no more device IDs available\n");
2482
2483		return ERR_PTR(devidx);
2484	}
2485
2486	md = kzalloc(sizeof(struct mmc_blk_data), GFP_KERNEL);
2487	if (!md) {
2488		ret = -ENOMEM;
2489		goto out;
2490	}
2491
2492	md->area_type = area_type;
2493
2494	/*
2495	 * Set the read-only status based on the supported commands
2496	 * and the write protect switch.
2497	 */
2498	md->read_only = mmc_blk_readonly(card);
2499
2500	md->disk = mmc_init_queue(&md->queue, card);
2501	if (IS_ERR(md->disk)) {
2502		ret = PTR_ERR(md->disk);
2503		goto err_kfree;
2504	}
2505
2506	INIT_LIST_HEAD(&md->part);
2507	INIT_LIST_HEAD(&md->rpmbs);
2508	kref_init(&md->kref);
2509
2510	md->queue.blkdata = md;
2511	md->part_type = part_type;
2512
2513	md->disk->major	= MMC_BLOCK_MAJOR;
2514	md->disk->minors = perdev_minors;
2515	md->disk->first_minor = devidx * perdev_minors;
2516	md->disk->fops = &mmc_bdops;
2517	md->disk->private_data = md;
2518	md->parent = parent;
2519	set_disk_ro(md->disk, md->read_only || default_ro);
 
2520	if (area_type & (MMC_BLK_DATA_AREA_RPMB | MMC_BLK_DATA_AREA_BOOT))
2521		md->disk->flags |= GENHD_FL_NO_PART;
 
2522
2523	/*
2524	 * As discussed on lkml, GENHD_FL_REMOVABLE should:
2525	 *
2526	 * - be set for removable media with permanent block devices
2527	 * - be unset for removable block devices with permanent media
2528	 *
2529	 * Since MMC block devices clearly fall under the second
2530	 * case, we do not set GENHD_FL_REMOVABLE.  Userspace
2531	 * should use the block device creation/destruction hotplug
2532	 * messages to tell when the card is present.
2533	 */
2534
2535	snprintf(md->disk->disk_name, sizeof(md->disk->disk_name),
2536		 "mmcblk%u%s", card->host->index, subname ? subname : "");
2537
2538	set_capacity(md->disk, size);
2539
2540	if (mmc_host_cmd23(card->host)) {
2541		if ((mmc_card_mmc(card) &&
2542		     card->csd.mmca_vsn >= CSD_SPEC_VER_3) ||
2543		    (mmc_card_sd(card) &&
2544		     card->scr.cmds & SD_SCR_CMD23_SUPPORT))
2545			md->flags |= MMC_BLK_CMD23;
2546	}
2547
2548	if (md->flags & MMC_BLK_CMD23 &&
 
2549	    ((card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN) ||
2550	     card->ext_csd.rel_sectors)) {
2551		md->flags |= MMC_BLK_REL_WR;
2552		fua_enabled = true;
2553		cache_enabled = true;
2554	}
2555	if (mmc_cache_enabled(card->host))
2556		cache_enabled  = true;
2557
2558	blk_queue_write_cache(md->queue.queue, cache_enabled, fua_enabled);
2559
2560	string_get_size((u64)size, 512, STRING_UNITS_2,
2561			cap_str, sizeof(cap_str));
2562	pr_info("%s: %s %s %s%s\n",
2563		md->disk->disk_name, mmc_card_id(card), mmc_card_name(card),
2564		cap_str, md->read_only ? " (ro)" : "");
2565
2566	/* used in ->open, must be set before add_disk: */
2567	if (area_type == MMC_BLK_DATA_AREA_MAIN)
2568		dev_set_drvdata(&card->dev, md);
2569	ret = device_add_disk(md->parent, md->disk, mmc_disk_attr_groups);
2570	if (ret)
2571		goto err_put_disk;
2572	return md;
2573
2574 err_put_disk:
2575	put_disk(md->disk);
2576	blk_mq_free_tag_set(&md->queue.tag_set);
2577 err_kfree:
2578	kfree(md);
2579 out:
2580	ida_simple_remove(&mmc_blk_ida, devidx);
2581	return ERR_PTR(ret);
2582}
2583
2584static struct mmc_blk_data *mmc_blk_alloc(struct mmc_card *card)
2585{
2586	sector_t size;
2587
2588	if (!mmc_card_sd(card) && mmc_card_blockaddr(card)) {
2589		/*
2590		 * The EXT_CSD sector count is in number or 512 byte
2591		 * sectors.
2592		 */
2593		size = card->ext_csd.sectors;
2594	} else {
2595		/*
2596		 * The CSD capacity field is in units of read_blkbits.
2597		 * set_capacity takes units of 512 bytes.
2598		 */
2599		size = (typeof(sector_t))card->csd.capacity
2600			<< (card->csd.read_blkbits - 9);
2601	}
2602
2603	return mmc_blk_alloc_req(card, &card->dev, size, false, NULL,
2604					MMC_BLK_DATA_AREA_MAIN, 0);
2605}
2606
2607static int mmc_blk_alloc_part(struct mmc_card *card,
2608			      struct mmc_blk_data *md,
2609			      unsigned int part_type,
2610			      sector_t size,
2611			      bool default_ro,
2612			      const char *subname,
2613			      int area_type)
2614{
2615	struct mmc_blk_data *part_md;
2616
2617	part_md = mmc_blk_alloc_req(card, disk_to_dev(md->disk), size, default_ro,
2618				    subname, area_type, part_type);
2619	if (IS_ERR(part_md))
2620		return PTR_ERR(part_md);
 
2621	list_add(&part_md->part, &md->part);
2622
2623	return 0;
2624}
2625
2626/**
2627 * mmc_rpmb_ioctl() - ioctl handler for the RPMB chardev
2628 * @filp: the character device file
2629 * @cmd: the ioctl() command
2630 * @arg: the argument from userspace
2631 *
2632 * This will essentially just redirect the ioctl()s coming in over to
2633 * the main block device spawning the RPMB character device.
2634 */
2635static long mmc_rpmb_ioctl(struct file *filp, unsigned int cmd,
2636			   unsigned long arg)
2637{
2638	struct mmc_rpmb_data *rpmb = filp->private_data;
2639	int ret;
2640
2641	switch (cmd) {
2642	case MMC_IOC_CMD:
2643		ret = mmc_blk_ioctl_cmd(rpmb->md,
2644					(struct mmc_ioc_cmd __user *)arg,
2645					rpmb);
2646		break;
2647	case MMC_IOC_MULTI_CMD:
2648		ret = mmc_blk_ioctl_multi_cmd(rpmb->md,
2649					(struct mmc_ioc_multi_cmd __user *)arg,
2650					rpmb);
2651		break;
2652	default:
2653		ret = -EINVAL;
2654		break;
2655	}
2656
2657	return ret;
2658}
2659
2660#ifdef CONFIG_COMPAT
2661static long mmc_rpmb_ioctl_compat(struct file *filp, unsigned int cmd,
2662			      unsigned long arg)
2663{
2664	return mmc_rpmb_ioctl(filp, cmd, (unsigned long)compat_ptr(arg));
2665}
2666#endif
2667
2668static int mmc_rpmb_chrdev_open(struct inode *inode, struct file *filp)
2669{
2670	struct mmc_rpmb_data *rpmb = container_of(inode->i_cdev,
2671						  struct mmc_rpmb_data, chrdev);
2672
2673	get_device(&rpmb->dev);
2674	filp->private_data = rpmb;
2675	mmc_blk_get(rpmb->md->disk);
2676
2677	return nonseekable_open(inode, filp);
2678}
2679
2680static int mmc_rpmb_chrdev_release(struct inode *inode, struct file *filp)
2681{
2682	struct mmc_rpmb_data *rpmb = container_of(inode->i_cdev,
2683						  struct mmc_rpmb_data, chrdev);
2684
2685	mmc_blk_put(rpmb->md);
2686	put_device(&rpmb->dev);
2687
2688	return 0;
2689}
2690
2691static const struct file_operations mmc_rpmb_fileops = {
2692	.release = mmc_rpmb_chrdev_release,
2693	.open = mmc_rpmb_chrdev_open,
2694	.owner = THIS_MODULE,
2695	.llseek = no_llseek,
2696	.unlocked_ioctl = mmc_rpmb_ioctl,
2697#ifdef CONFIG_COMPAT
2698	.compat_ioctl = mmc_rpmb_ioctl_compat,
2699#endif
2700};
2701
2702static void mmc_blk_rpmb_device_release(struct device *dev)
2703{
2704	struct mmc_rpmb_data *rpmb = dev_get_drvdata(dev);
2705
2706	ida_simple_remove(&mmc_rpmb_ida, rpmb->id);
2707	kfree(rpmb);
2708}
2709
2710static int mmc_blk_alloc_rpmb_part(struct mmc_card *card,
2711				   struct mmc_blk_data *md,
2712				   unsigned int part_index,
2713				   sector_t size,
2714				   const char *subname)
2715{
2716	int devidx, ret;
2717	char rpmb_name[DISK_NAME_LEN];
2718	char cap_str[10];
2719	struct mmc_rpmb_data *rpmb;
2720
2721	/* This creates the minor number for the RPMB char device */
2722	devidx = ida_simple_get(&mmc_rpmb_ida, 0, max_devices, GFP_KERNEL);
2723	if (devidx < 0)
2724		return devidx;
2725
2726	rpmb = kzalloc(sizeof(*rpmb), GFP_KERNEL);
2727	if (!rpmb) {
2728		ida_simple_remove(&mmc_rpmb_ida, devidx);
2729		return -ENOMEM;
2730	}
2731
2732	snprintf(rpmb_name, sizeof(rpmb_name),
2733		 "mmcblk%u%s", card->host->index, subname ? subname : "");
2734
2735	rpmb->id = devidx;
2736	rpmb->part_index = part_index;
2737	rpmb->dev.init_name = rpmb_name;
2738	rpmb->dev.bus = &mmc_rpmb_bus_type;
2739	rpmb->dev.devt = MKDEV(MAJOR(mmc_rpmb_devt), rpmb->id);
2740	rpmb->dev.parent = &card->dev;
2741	rpmb->dev.release = mmc_blk_rpmb_device_release;
2742	device_initialize(&rpmb->dev);
2743	dev_set_drvdata(&rpmb->dev, rpmb);
2744	rpmb->md = md;
2745
2746	cdev_init(&rpmb->chrdev, &mmc_rpmb_fileops);
2747	rpmb->chrdev.owner = THIS_MODULE;
2748	ret = cdev_device_add(&rpmb->chrdev, &rpmb->dev);
2749	if (ret) {
2750		pr_err("%s: could not add character device\n", rpmb_name);
2751		goto out_put_device;
2752	}
2753
2754	list_add(&rpmb->node, &md->rpmbs);
2755
2756	string_get_size((u64)size, 512, STRING_UNITS_2,
2757			cap_str, sizeof(cap_str));
2758
2759	pr_info("%s: %s %s %s, chardev (%d:%d)\n",
2760		rpmb_name, mmc_card_id(card), mmc_card_name(card), cap_str,
2761		MAJOR(mmc_rpmb_devt), rpmb->id);
2762
2763	return 0;
2764
2765out_put_device:
2766	put_device(&rpmb->dev);
2767	return ret;
2768}
2769
2770static void mmc_blk_remove_rpmb_part(struct mmc_rpmb_data *rpmb)
2771
2772{
2773	cdev_device_del(&rpmb->chrdev, &rpmb->dev);
2774	put_device(&rpmb->dev);
2775}
2776
2777/* MMC Physical partitions consist of two boot partitions and
2778 * up to four general purpose partitions.
2779 * For each partition enabled in EXT_CSD a block device will be allocatedi
2780 * to provide access to the partition.
2781 */
2782
2783static int mmc_blk_alloc_parts(struct mmc_card *card, struct mmc_blk_data *md)
2784{
2785	int idx, ret;
2786
2787	if (!mmc_card_mmc(card))
2788		return 0;
2789
2790	for (idx = 0; idx < card->nr_parts; idx++) {
2791		if (card->part[idx].area_type & MMC_BLK_DATA_AREA_RPMB) {
2792			/*
2793			 * RPMB partitions does not provide block access, they
2794			 * are only accessed using ioctl():s. Thus create
2795			 * special RPMB block devices that do not have a
2796			 * backing block queue for these.
2797			 */
2798			ret = mmc_blk_alloc_rpmb_part(card, md,
2799				card->part[idx].part_cfg,
2800				card->part[idx].size >> 9,
2801				card->part[idx].name);
2802			if (ret)
2803				return ret;
2804		} else if (card->part[idx].size) {
2805			ret = mmc_blk_alloc_part(card, md,
2806				card->part[idx].part_cfg,
2807				card->part[idx].size >> 9,
2808				card->part[idx].force_ro,
2809				card->part[idx].name,
2810				card->part[idx].area_type);
2811			if (ret)
2812				return ret;
2813		}
2814	}
2815
2816	return 0;
2817}
2818
2819static void mmc_blk_remove_req(struct mmc_blk_data *md)
2820{
2821	/*
2822	 * Flush remaining requests and free queues. It is freeing the queue
2823	 * that stops new requests from being accepted.
2824	 */
2825	del_gendisk(md->disk);
2826	mmc_cleanup_queue(&md->queue);
2827	mmc_blk_put(md);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2828}
2829
2830static void mmc_blk_remove_parts(struct mmc_card *card,
2831				 struct mmc_blk_data *md)
2832{
2833	struct list_head *pos, *q;
2834	struct mmc_blk_data *part_md;
2835	struct mmc_rpmb_data *rpmb;
2836
2837	/* Remove RPMB partitions */
2838	list_for_each_safe(pos, q, &md->rpmbs) {
2839		rpmb = list_entry(pos, struct mmc_rpmb_data, node);
2840		list_del(pos);
2841		mmc_blk_remove_rpmb_part(rpmb);
2842	}
2843	/* Remove block partitions */
2844	list_for_each_safe(pos, q, &md->part) {
2845		part_md = list_entry(pos, struct mmc_blk_data, part);
2846		list_del(pos);
2847		mmc_blk_remove_req(part_md);
2848	}
2849}
2850
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2851#ifdef CONFIG_DEBUG_FS
2852
2853static int mmc_dbg_card_status_get(void *data, u64 *val)
2854{
2855	struct mmc_card *card = data;
2856	struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
2857	struct mmc_queue *mq = &md->queue;
2858	struct request *req;
2859	int ret;
2860
2861	/* Ask the block layer about the card status */
2862	req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_IN, 0);
2863	if (IS_ERR(req))
2864		return PTR_ERR(req);
2865	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_CARD_STATUS;
2866	req_to_mmc_queue_req(req)->drv_op_result = -EIO;
2867	blk_execute_rq(req, false);
2868	ret = req_to_mmc_queue_req(req)->drv_op_result;
2869	if (ret >= 0) {
2870		*val = ret;
2871		ret = 0;
2872	}
2873	blk_mq_free_request(req);
2874
2875	return ret;
2876}
2877DEFINE_DEBUGFS_ATTRIBUTE(mmc_dbg_card_status_fops, mmc_dbg_card_status_get,
2878			 NULL, "%08llx\n");
2879
2880/* That is two digits * 512 + 1 for newline */
2881#define EXT_CSD_STR_LEN 1025
2882
2883static int mmc_ext_csd_open(struct inode *inode, struct file *filp)
2884{
2885	struct mmc_card *card = inode->i_private;
2886	struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
2887	struct mmc_queue *mq = &md->queue;
2888	struct request *req;
2889	char *buf;
2890	ssize_t n = 0;
2891	u8 *ext_csd;
2892	int err, i;
2893
2894	buf = kmalloc(EXT_CSD_STR_LEN + 1, GFP_KERNEL);
2895	if (!buf)
2896		return -ENOMEM;
2897
2898	/* Ask the block layer for the EXT CSD */
2899	req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_IN, 0);
2900	if (IS_ERR(req)) {
2901		err = PTR_ERR(req);
2902		goto out_free;
2903	}
2904	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_EXT_CSD;
2905	req_to_mmc_queue_req(req)->drv_op_result = -EIO;
2906	req_to_mmc_queue_req(req)->drv_op_data = &ext_csd;
2907	blk_execute_rq(req, false);
2908	err = req_to_mmc_queue_req(req)->drv_op_result;
2909	blk_mq_free_request(req);
2910	if (err) {
2911		pr_err("FAILED %d\n", err);
2912		goto out_free;
2913	}
2914
2915	for (i = 0; i < 512; i++)
2916		n += sprintf(buf + n, "%02x", ext_csd[i]);
2917	n += sprintf(buf + n, "\n");
2918
2919	if (n != EXT_CSD_STR_LEN) {
2920		err = -EINVAL;
2921		kfree(ext_csd);
2922		goto out_free;
2923	}
2924
2925	filp->private_data = buf;
2926	kfree(ext_csd);
2927	return 0;
2928
2929out_free:
2930	kfree(buf);
2931	return err;
2932}
2933
2934static ssize_t mmc_ext_csd_read(struct file *filp, char __user *ubuf,
2935				size_t cnt, loff_t *ppos)
2936{
2937	char *buf = filp->private_data;
2938
2939	return simple_read_from_buffer(ubuf, cnt, ppos,
2940				       buf, EXT_CSD_STR_LEN);
2941}
2942
2943static int mmc_ext_csd_release(struct inode *inode, struct file *file)
2944{
2945	kfree(file->private_data);
2946	return 0;
2947}
2948
2949static const struct file_operations mmc_dbg_ext_csd_fops = {
2950	.open		= mmc_ext_csd_open,
2951	.read		= mmc_ext_csd_read,
2952	.release	= mmc_ext_csd_release,
2953	.llseek		= default_llseek,
2954};
2955
2956static void mmc_blk_add_debugfs(struct mmc_card *card, struct mmc_blk_data *md)
2957{
2958	struct dentry *root;
2959
2960	if (!card->debugfs_root)
2961		return;
2962
2963	root = card->debugfs_root;
2964
2965	if (mmc_card_mmc(card) || mmc_card_sd(card)) {
2966		md->status_dentry =
2967			debugfs_create_file_unsafe("status", 0400, root,
2968						   card,
2969						   &mmc_dbg_card_status_fops);
 
 
2970	}
2971
2972	if (mmc_card_mmc(card)) {
2973		md->ext_csd_dentry =
2974			debugfs_create_file("ext_csd", S_IRUSR, root, card,
2975					    &mmc_dbg_ext_csd_fops);
 
 
2976	}
 
 
2977}
2978
2979static void mmc_blk_remove_debugfs(struct mmc_card *card,
2980				   struct mmc_blk_data *md)
2981{
2982	if (!card->debugfs_root)
2983		return;
2984
2985	debugfs_remove(md->status_dentry);
2986	md->status_dentry = NULL;
 
 
2987
2988	debugfs_remove(md->ext_csd_dentry);
2989	md->ext_csd_dentry = NULL;
 
 
2990}
2991
2992#else
2993
2994static void mmc_blk_add_debugfs(struct mmc_card *card, struct mmc_blk_data *md)
2995{
 
2996}
2997
2998static void mmc_blk_remove_debugfs(struct mmc_card *card,
2999				   struct mmc_blk_data *md)
3000{
3001}
3002
3003#endif /* CONFIG_DEBUG_FS */
3004
3005static int mmc_blk_probe(struct mmc_card *card)
3006{
3007	struct mmc_blk_data *md;
3008	int ret = 0;
3009
3010	/*
3011	 * Check that the card supports the command class(es) we need.
3012	 */
3013	if (!(card->csd.cmdclass & CCC_BLOCK_READ))
3014		return -ENODEV;
3015
3016	mmc_fixup_device(card, mmc_blk_fixups);
3017
3018	card->complete_wq = alloc_workqueue("mmc_complete",
3019					WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
3020	if (!card->complete_wq) {
3021		pr_err("Failed to create mmc completion workqueue");
3022		return -ENOMEM;
3023	}
3024
3025	md = mmc_blk_alloc(card);
3026	if (IS_ERR(md)) {
3027		ret = PTR_ERR(md);
3028		goto out_free;
3029	}
3030
3031	ret = mmc_blk_alloc_parts(card, md);
3032	if (ret)
3033		goto out;
3034
 
 
 
 
 
 
 
 
 
 
 
 
3035	/* Add two debugfs entries */
3036	mmc_blk_add_debugfs(card, md);
3037
3038	pm_runtime_set_autosuspend_delay(&card->dev, 3000);
3039	pm_runtime_use_autosuspend(&card->dev);
3040
3041	/*
3042	 * Don't enable runtime PM for SD-combo cards here. Leave that
3043	 * decision to be taken during the SDIO init sequence instead.
3044	 */
3045	if (!mmc_card_sd_combo(card)) {
3046		pm_runtime_set_active(&card->dev);
3047		pm_runtime_enable(&card->dev);
3048	}
3049
3050	return 0;
3051
3052out:
3053	mmc_blk_remove_parts(card, md);
3054	mmc_blk_remove_req(md);
3055out_free:
3056	destroy_workqueue(card->complete_wq);
3057	return ret;
3058}
3059
3060static void mmc_blk_remove(struct mmc_card *card)
3061{
3062	struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
3063
3064	mmc_blk_remove_debugfs(card, md);
3065	mmc_blk_remove_parts(card, md);
3066	pm_runtime_get_sync(&card->dev);
3067	if (md->part_curr != md->part_type) {
3068		mmc_claim_host(card->host);
3069		mmc_blk_part_switch(card, md->part_type);
3070		mmc_release_host(card->host);
3071	}
3072	if (!mmc_card_sd_combo(card))
3073		pm_runtime_disable(&card->dev);
3074	pm_runtime_put_noidle(&card->dev);
3075	mmc_blk_remove_req(md);
 
3076	destroy_workqueue(card->complete_wq);
3077}
3078
3079static int _mmc_blk_suspend(struct mmc_card *card)
3080{
3081	struct mmc_blk_data *part_md;
3082	struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
3083
3084	if (md) {
3085		mmc_queue_suspend(&md->queue);
3086		list_for_each_entry(part_md, &md->part, part) {
3087			mmc_queue_suspend(&part_md->queue);
3088		}
3089	}
3090	return 0;
3091}
3092
3093static void mmc_blk_shutdown(struct mmc_card *card)
3094{
3095	_mmc_blk_suspend(card);
3096}
3097
3098#ifdef CONFIG_PM_SLEEP
3099static int mmc_blk_suspend(struct device *dev)
3100{
3101	struct mmc_card *card = mmc_dev_to_card(dev);
3102
3103	return _mmc_blk_suspend(card);
3104}
3105
3106static int mmc_blk_resume(struct device *dev)
3107{
3108	struct mmc_blk_data *part_md;
3109	struct mmc_blk_data *md = dev_get_drvdata(dev);
3110
3111	if (md) {
3112		/*
3113		 * Resume involves the card going into idle state,
3114		 * so current partition is always the main one.
3115		 */
3116		md->part_curr = md->part_type;
3117		mmc_queue_resume(&md->queue);
3118		list_for_each_entry(part_md, &md->part, part) {
3119			mmc_queue_resume(&part_md->queue);
3120		}
3121	}
3122	return 0;
3123}
3124#endif
3125
3126static SIMPLE_DEV_PM_OPS(mmc_blk_pm_ops, mmc_blk_suspend, mmc_blk_resume);
3127
3128static struct mmc_driver mmc_driver = {
3129	.drv		= {
3130		.name	= "mmcblk",
3131		.pm	= &mmc_blk_pm_ops,
3132	},
3133	.probe		= mmc_blk_probe,
3134	.remove		= mmc_blk_remove,
3135	.shutdown	= mmc_blk_shutdown,
3136};
3137
3138static int __init mmc_blk_init(void)
3139{
3140	int res;
3141
3142	res  = bus_register(&mmc_rpmb_bus_type);
3143	if (res < 0) {
3144		pr_err("mmcblk: could not register RPMB bus type\n");
3145		return res;
3146	}
3147	res = alloc_chrdev_region(&mmc_rpmb_devt, 0, MAX_DEVICES, "rpmb");
3148	if (res < 0) {
3149		pr_err("mmcblk: failed to allocate rpmb chrdev region\n");
3150		goto out_bus_unreg;
3151	}
3152
3153	if (perdev_minors != CONFIG_MMC_BLOCK_MINORS)
3154		pr_info("mmcblk: using %d minors per device\n", perdev_minors);
3155
3156	max_devices = min(MAX_DEVICES, (1 << MINORBITS) / perdev_minors);
3157
3158	res = register_blkdev(MMC_BLOCK_MAJOR, "mmc");
3159	if (res)
3160		goto out_chrdev_unreg;
3161
3162	res = mmc_register_driver(&mmc_driver);
3163	if (res)
3164		goto out_blkdev_unreg;
3165
3166	return 0;
3167
3168out_blkdev_unreg:
3169	unregister_blkdev(MMC_BLOCK_MAJOR, "mmc");
3170out_chrdev_unreg:
3171	unregister_chrdev_region(mmc_rpmb_devt, MAX_DEVICES);
3172out_bus_unreg:
3173	bus_unregister(&mmc_rpmb_bus_type);
3174	return res;
3175}
3176
3177static void __exit mmc_blk_exit(void)
3178{
3179	mmc_unregister_driver(&mmc_driver);
3180	unregister_blkdev(MMC_BLOCK_MAJOR, "mmc");
3181	unregister_chrdev_region(mmc_rpmb_devt, MAX_DEVICES);
3182	bus_unregister(&mmc_rpmb_bus_type);
3183}
3184
3185module_init(mmc_blk_init);
3186module_exit(mmc_blk_exit);
3187
3188MODULE_LICENSE("GPL");
3189MODULE_DESCRIPTION("Multimedia Card (MMC) block device driver");