1/* SPDX-License-Identifier: GPL-2.0 */
   2#ifndef _LINUX_BLKDEV_H
   3#define _LINUX_BLKDEV_H
   4
   5#include <linux/sched.h>
   6#include <linux/sched/clock.h>
 
 
   7#include <linux/major.h>
   8#include <linux/genhd.h>
   9#include <linux/list.h>
  10#include <linux/llist.h>
  11#include <linux/timer.h>
  12#include <linux/workqueue.h>
  13#include <linux/pagemap.h>
  14#include <linux/backing-dev-defs.h>
  15#include <linux/wait.h>
  16#include <linux/mempool.h>
  17#include <linux/pfn.h>
  18#include <linux/bio.h>
  19#include <linux/stringify.h>
  20#include <linux/gfp.h>
  21#include <linux/bsg.h>
  22#include <linux/smp.h>
  23#include <linux/rcupdate.h>
  24#include <linux/percpu-refcount.h>
  25#include <linux/scatterlist.h>
  26#include <linux/blkzoned.h>
  27
  28struct module;
  29struct scsi_ioctl_command;
  30
  31struct request_queue;
  32struct elevator_queue;
 
  33struct blk_trace;
  34struct request;
  35struct sg_io_hdr;
  36struct bsg_job;
  37struct blkcg_gq;
  38struct blk_flush_queue;
  39struct pr_ops;
  40struct rq_qos;
  41struct blk_queue_stats;
  42struct blk_stat_callback;
  43struct blk_keyslot_manager;
  44
  45#define BLKDEV_MIN_RQ	4
  46#define BLKDEV_MAX_RQ	128	/* Default maximum */
  47
  48/* Must be consistent with blk_mq_poll_stats_bkt() */
  49#define BLK_MQ_POLL_STATS_BKTS 16
  50
  51/* Doing classic polling */
  52#define BLK_MQ_POLL_CLASSIC -1
  53
  54/*
  55 * Maximum number of blkcg policies allowed to be registered concurrently.
  56 * Defined here to simplify include dependency.
  57 */
  58#define BLKCG_MAX_POLS		5
  59
  60typedef void (rq_end_io_fn)(struct request *, blk_status_t);
  61
  62/*
  63 * request flags */
  64typedef __u32 __bitwise req_flags_t;
  65
  66/* elevator knows about this request */
  67#define RQF_SORTED		((__force req_flags_t)(1 << 0))
  68/* drive already may have started this one */
  69#define RQF_STARTED		((__force req_flags_t)(1 << 1))
  70/* may not be passed by ioscheduler */
  71#define RQF_SOFTBARRIER		((__force req_flags_t)(1 << 3))
  72/* request for flush sequence */
  73#define RQF_FLUSH_SEQ		((__force req_flags_t)(1 << 4))
  74/* merge of different types, fail separately */
  75#define RQF_MIXED_MERGE		((__force req_flags_t)(1 << 5))
  76/* track inflight for MQ */
  77#define RQF_MQ_INFLIGHT		((__force req_flags_t)(1 << 6))
  78/* don't call prep for this one */
  79#define RQF_DONTPREP		((__force req_flags_t)(1 << 7))
  80/* set for "ide_preempt" requests and also for requests for which the SCSI
  81   "quiesce" state must be ignored. */
  82#define RQF_PREEMPT		((__force req_flags_t)(1 << 8))
  83/* vaguely specified driver internal error.  Ignored by the block layer */
  84#define RQF_FAILED		((__force req_flags_t)(1 << 10))
  85/* don't warn about errors */
  86#define RQF_QUIET		((__force req_flags_t)(1 << 11))
  87/* elevator private data attached */
  88#define RQF_ELVPRIV		((__force req_flags_t)(1 << 12))
  89/* account into disk and partition IO statistics */
  90#define RQF_IO_STAT		((__force req_flags_t)(1 << 13))
  91/* request came from our alloc pool */
  92#define RQF_ALLOCED		((__force req_flags_t)(1 << 14))
  93/* runtime pm request */
  94#define RQF_PM			((__force req_flags_t)(1 << 15))
  95/* on IO scheduler merge hash */
  96#define RQF_HASHED		((__force req_flags_t)(1 << 16))
  97/* track IO completion time */
  98#define RQF_STATS		((__force req_flags_t)(1 << 17))
  99/* Look at ->special_vec for the actual data payload instead of the
 100   bio chain. */
 101#define RQF_SPECIAL_PAYLOAD	((__force req_flags_t)(1 << 18))
 102/* The per-zone write lock is held for this request */
 103#define RQF_ZONE_WRITE_LOCKED	((__force req_flags_t)(1 << 19))
 104/* already slept for hybrid poll */
 105#define RQF_MQ_POLL_SLEPT	((__force req_flags_t)(1 << 20))
 106/* ->timeout has been called, don't expire again */
 107#define RQF_TIMED_OUT		((__force req_flags_t)(1 << 21))
 108
 109/* flags that prevent us from merging requests: */
 110#define RQF_NOMERGE_FLAGS \
 111	(RQF_STARTED | RQF_SOFTBARRIER | RQF_FLUSH_SEQ | RQF_SPECIAL_PAYLOAD)
 112
 113/*
 114 * Request state for blk-mq.
 115 */
 116enum mq_rq_state {
 117	MQ_RQ_IDLE		= 0,
 118	MQ_RQ_IN_FLIGHT		= 1,
 119	MQ_RQ_COMPLETE		= 2,
 
 
 
 
 
 
 
 
 
 
 
 120};
 121
 
 
 122/*
 123 * Try to put the fields that are referenced together in the same cacheline.
 124 *
 125 * If you modify this structure, make sure to update blk_rq_init() and
 126 * especially blk_mq_rq_ctx_init() to take care of the added fields.
 127 */
 128struct request {
 
 
 
 129	struct request_queue *q;
 130	struct blk_mq_ctx *mq_ctx;
 131	struct blk_mq_hw_ctx *mq_hctx;
 132
 133	unsigned int cmd_flags;		/* op and common flags */
 134	req_flags_t rq_flags;
 
 135
 136	int tag;
 137	int internal_tag;
 138
 139	/* the following two fields are internal, NEVER access directly */
 140	unsigned int __data_len;	/* total data len */
 141	sector_t __sector;		/* sector cursor */
 142
 143	struct bio *bio;
 144	struct bio *biotail;
 145
 146	struct list_head queuelist;
 147
 148	/*
 149	 * The hash is used inside the scheduler, and killed once the
 150	 * request reaches the dispatch list. The ipi_list is only used
 151	 * to queue the request for softirq completion, which is long
 152	 * after the request has been unhashed (and even removed from
 153	 * the dispatch list).
 154	 */
 155	union {
 156		struct hlist_node hash;	/* merge hash */
 157		struct list_head ipi_list;
 158	};
 159
 160	/*
 161	 * The rb_node is only used inside the io scheduler, requests
 162	 * are pruned when moved to the dispatch queue. So let the
 163	 * completion_data share space with the rb_node.
 164	 */
 165	union {
 166		struct rb_node rb_node;	/* sort/lookup */
 167		struct bio_vec special_vec;
 168		void *completion_data;
 169		int error_count; /* for legacy drivers, don't use */
 170	};
 171
 172	/*
 173	 * Three pointers are available for the IO schedulers, if they need
 174	 * more they have to dynamically allocate it.  Flush requests are
 175	 * never put on the IO scheduler. So let the flush fields share
 176	 * space with the elevator data.
 177	 */
 178	union {
 179		struct {
 180			struct io_cq		*icq;
 181			void			*priv[2];
 182		} elv;
 183
 184		struct {
 185			unsigned int		seq;
 186			struct list_head	list;
 187			rq_end_io_fn		*saved_end_io;
 188		} flush;
 189	};
 190
 191	struct gendisk *rq_disk;
 192	struct hd_struct *part;
 193#ifdef CONFIG_BLK_RQ_ALLOC_TIME
 194	/* Time that the first bio started allocating this request. */
 195	u64 alloc_time_ns;
 196#endif
 197	/* Time that this request was allocated for this IO. */
 198	u64 start_time_ns;
 199	/* Time that I/O was submitted to the device. */
 200	u64 io_start_time_ns;
 201
 202#ifdef CONFIG_BLK_WBT
 203	unsigned short wbt_flags;
 204#endif
 205	/*
 206	 * rq sectors used for blk stats. It has the same value
 207	 * with blk_rq_sectors(rq), except that it never be zeroed
 208	 * by completion.
 209	 */
 210	unsigned short stats_sectors;
 211
 212	/*
 213	 * Number of scatter-gather DMA addr+len pairs after
 214	 * physical address coalescing is performed.
 215	 */
 216	unsigned short nr_phys_segments;
 217
 218#if defined(CONFIG_BLK_DEV_INTEGRITY)
 219	unsigned short nr_integrity_segments;
 220#endif
 221
 222#ifdef CONFIG_BLK_INLINE_ENCRYPTION
 223	struct bio_crypt_ctx *crypt_ctx;
 224	struct blk_ksm_keyslot *crypt_keyslot;
 225#endif
 226
 227	unsigned short write_hint;
 228	unsigned short ioprio;
 229
 230	enum mq_rq_state state;
 231	refcount_t ref;
 232
 233	unsigned int timeout;
 234	unsigned long deadline;
 235
 236	union {
 237		struct __call_single_data csd;
 238		u64 fifo_time;
 239	};
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 240
 241	/*
 242	 * completion callback.
 243	 */
 244	rq_end_io_fn *end_io;
 245	void *end_io_data;
 246};
 247
 248static inline bool blk_op_is_scsi(unsigned int op)
 249{
 250	return op == REQ_OP_SCSI_IN || op == REQ_OP_SCSI_OUT;
 251}
 252
 253static inline bool blk_op_is_private(unsigned int op)
 254{
 255	return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT;
 256}
 257
 258static inline bool blk_rq_is_scsi(struct request *rq)
 259{
 260	return blk_op_is_scsi(req_op(rq));
 261}
 262
 263static inline bool blk_rq_is_private(struct request *rq)
 264{
 265	return blk_op_is_private(req_op(rq));
 266}
 267
 268static inline bool blk_rq_is_passthrough(struct request *rq)
 269{
 270	return blk_rq_is_scsi(rq) || blk_rq_is_private(rq);
 271}
 272
 273static inline bool bio_is_passthrough(struct bio *bio)
 274{
 275	unsigned op = bio_op(bio);
 276
 277	return blk_op_is_scsi(op) || blk_op_is_private(op);
 278}
 279
 280static inline unsigned short req_get_ioprio(struct request *req)
 281{
 282	return req->ioprio;
 283}
 284
 
 
 
 
 
 
 
 
 
 
 
 
 
 285#include <linux/elevator.h>
 286
 287struct blk_queue_ctx;
 
 
 
 288
 289struct bio_vec;
 
 
 
 
 
 
 
 
 
 
 
 
 290
 291enum blk_eh_timer_return {
 292	BLK_EH_DONE,		/* drivers has completed the command */
 293	BLK_EH_RESET_TIMER,	/* reset timer and try again */
 
 294};
 295
 
 
 296enum blk_queue_state {
 297	Queue_down,
 298	Queue_up,
 299};
 300
 301#define BLK_TAG_ALLOC_FIFO 0 /* allocate starting from 0 */
 302#define BLK_TAG_ALLOC_RR 1 /* allocate starting from last allocated tag */
 
 
 
 
 
 
 303
 304#define BLK_SCSI_MAX_CMDS	(256)
 305#define BLK_SCSI_CMD_PER_LONG	(BLK_SCSI_MAX_CMDS / (sizeof(long) * 8))
 306
 307/*
 308 * Zoned block device models (zoned limit).
 309 *
 310 * Note: This needs to be ordered from the least to the most severe
 311 * restrictions for the inheritance in blk_stack_limits() to work.
 312 */
 313enum blk_zoned_model {
 314	BLK_ZONED_NONE = 0,	/* Regular block device */
 315	BLK_ZONED_HA,		/* Host-aware zoned block device */
 316	BLK_ZONED_HM,		/* Host-managed zoned block device */
 317};
 318
 319struct queue_limits {
 320	unsigned long		bounce_pfn;
 321	unsigned long		seg_boundary_mask;
 322	unsigned long		virt_boundary_mask;
 323
 324	unsigned int		max_hw_sectors;
 325	unsigned int		max_dev_sectors;
 326	unsigned int		chunk_sectors;
 327	unsigned int		max_sectors;
 328	unsigned int		max_segment_size;
 329	unsigned int		physical_block_size;
 330	unsigned int		logical_block_size;
 331	unsigned int		alignment_offset;
 332	unsigned int		io_min;
 333	unsigned int		io_opt;
 334	unsigned int		max_discard_sectors;
 335	unsigned int		max_hw_discard_sectors;
 336	unsigned int		max_write_same_sectors;
 337	unsigned int		max_write_zeroes_sectors;
 338	unsigned int		max_zone_append_sectors;
 339	unsigned int		discard_granularity;
 340	unsigned int		discard_alignment;
 341
 
 342	unsigned short		max_segments;
 343	unsigned short		max_integrity_segments;
 344	unsigned short		max_discard_segments;
 345
 346	unsigned char		misaligned;
 347	unsigned char		discard_misaligned;
 348	unsigned char		raid_partial_stripes_expensive;
 349	enum blk_zoned_model	zoned;
 350};
 351
 352typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx,
 353			       void *data);
 354
 355void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model);
 356
 357#ifdef CONFIG_BLK_DEV_ZONED
 358
 359#define BLK_ALL_ZONES  ((unsigned int)-1)
 360int blkdev_report_zones(struct block_device *bdev, sector_t sector,
 361			unsigned int nr_zones, report_zones_cb cb, void *data);
 362unsigned int blkdev_nr_zones(struct gendisk *disk);
 363extern int blkdev_zone_mgmt(struct block_device *bdev, enum req_opf op,
 364			    sector_t sectors, sector_t nr_sectors,
 365			    gfp_t gfp_mask);
 366int blk_revalidate_disk_zones(struct gendisk *disk,
 367			      void (*update_driver_data)(struct gendisk *disk));
 368
 369extern int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
 370				     unsigned int cmd, unsigned long arg);
 371extern int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode,
 372				  unsigned int cmd, unsigned long arg);
 373
 374#else /* CONFIG_BLK_DEV_ZONED */
 375
 376static inline unsigned int blkdev_nr_zones(struct gendisk *disk)
 377{
 378	return 0;
 379}
 380
 381static inline int blkdev_report_zones_ioctl(struct block_device *bdev,
 382					    fmode_t mode, unsigned int cmd,
 383					    unsigned long arg)
 384{
 385	return -ENOTTY;
 386}
 387
 388static inline int blkdev_zone_mgmt_ioctl(struct block_device *bdev,
 389					 fmode_t mode, unsigned int cmd,
 390					 unsigned long arg)
 391{
 392	return -ENOTTY;
 393}
 394
 395#endif /* CONFIG_BLK_DEV_ZONED */
 396
 397struct request_queue {
 
 
 
 
 398	struct request		*last_merge;
 399	struct elevator_queue	*elevator;
 400
 401	struct blk_queue_stats	*stats;
 402	struct rq_qos		*rq_qos;
 403
 404	const struct blk_mq_ops	*mq_ops;
 405
 406	/* sw queues */
 407	struct blk_mq_ctx __percpu	*queue_ctx;
 
 
 
 
 
 
 
 408
 409	unsigned int		queue_depth;
 
 
 
 
 410
 411	/* hw dispatch queues */
 412	struct blk_mq_hw_ctx	**queue_hw_ctx;
 413	unsigned int		nr_hw_queues;
 
 414
 415	struct backing_dev_info	*backing_dev_info;
 416
 417	/*
 418	 * The queue owner gets to use this for whatever they like.
 419	 * ll_rw_blk doesn't touch it.
 420	 */
 421	void			*queuedata;
 422
 423	/*
 424	 * various queue flags, see QUEUE_* below
 425	 */
 426	unsigned long		queue_flags;
 427	/*
 428	 * Number of contexts that have called blk_set_pm_only(). If this
 429	 * counter is above zero then only RQF_PM and RQF_PREEMPT requests are
 430	 * processed.
 431	 */
 432	atomic_t		pm_only;
 433
 434	/*
 435	 * ida allocated id for this queue.  Used to index queues from
 436	 * ioctx.
 437	 */
 438	int			id;
 439
 440	/*
 441	 * queue needs bounce pages for pages above this limit
 442	 */
 443	gfp_t			bounce_gfp;
 444
 445	spinlock_t		queue_lock;
 446
 447	/*
 448	 * queue kobject
 
 
 449	 */
 450	struct kobject kobj;
 
 451
 452	/*
 453	 * mq queue kobject
 454	 */
 455	struct kobject *mq_kobj;
 456
 457#ifdef  CONFIG_BLK_DEV_INTEGRITY
 458	struct blk_integrity integrity;
 459#endif	/* CONFIG_BLK_DEV_INTEGRITY */
 460
 461#ifdef CONFIG_PM
 462	struct device		*dev;
 463	int			rpm_status;
 464	unsigned int		nr_pending;
 465#endif
 466
 467	/*
 468	 * queue settings
 469	 */
 470	unsigned long		nr_requests;	/* Max # of requests */
 
 
 
 471
 
 
 472	unsigned int		dma_pad_mask;
 473	unsigned int		dma_alignment;
 474
 475#ifdef CONFIG_BLK_INLINE_ENCRYPTION
 476	/* Inline crypto capabilities */
 477	struct blk_keyslot_manager *ksm;
 478#endif
 479
 480	unsigned int		rq_timeout;
 481	int			poll_nsec;
 482
 483	struct blk_stat_callback	*poll_cb;
 484	struct blk_rq_stat	poll_stat[BLK_MQ_POLL_STATS_BKTS];
 485
 
 486	struct timer_list	timeout;
 487	struct work_struct	timeout_work;
 488
 489	struct list_head	icq_list;
 490#ifdef CONFIG_BLK_CGROUP
 491	DECLARE_BITMAP		(blkcg_pols, BLKCG_MAX_POLS);
 492	struct blkcg_gq		*root_blkg;
 493	struct list_head	blkg_list;
 494#endif
 495
 496	struct queue_limits	limits;
 497
 498	unsigned int		required_elevator_features;
 499
 500#ifdef CONFIG_BLK_DEV_ZONED
 501	/*
 502	 * Zoned block device information for request dispatch control.
 503	 * nr_zones is the total number of zones of the device. This is always
 504	 * 0 for regular block devices. conv_zones_bitmap is a bitmap of nr_zones
 505	 * bits which indicates if a zone is conventional (bit set) or
 506	 * sequential (bit clear). seq_zones_wlock is a bitmap of nr_zones
 507	 * bits which indicates if a zone is write locked, that is, if a write
 508	 * request targeting the zone was dispatched. All three fields are
 509	 * initialized by the low level device driver (e.g. scsi/sd.c).
 510	 * Stacking drivers (device mappers) may or may not initialize
 511	 * these fields.
 512	 *
 513	 * Reads of this information must be protected with blk_queue_enter() /
 514	 * blk_queue_exit(). Modifying this information is only allowed while
 515	 * no requests are being processed. See also blk_mq_freeze_queue() and
 516	 * blk_mq_unfreeze_queue().
 517	 */
 518	unsigned int		nr_zones;
 519	unsigned long		*conv_zones_bitmap;
 520	unsigned long		*seq_zones_wlock;
 521	unsigned int		max_open_zones;
 522	unsigned int		max_active_zones;
 523#endif /* CONFIG_BLK_DEV_ZONED */
 524
 525	/*
 526	 * sg stuff
 527	 */
 528	unsigned int		sg_timeout;
 529	unsigned int		sg_reserved_size;
 530	int			node;
 531	struct mutex		debugfs_mutex;
 532#ifdef CONFIG_BLK_DEV_IO_TRACE
 533	struct blk_trace __rcu	*blk_trace;
 534#endif
 535	/*
 536	 * for flush operations
 537	 */
 538	struct blk_flush_queue	*fq;
 539
 540	struct list_head	requeue_list;
 541	spinlock_t		requeue_lock;
 542	struct delayed_work	requeue_work;
 
 
 
 
 543
 544	struct mutex		sysfs_lock;
 545	struct mutex		sysfs_dir_lock;
 546
 547	/*
 548	 * for reusing dead hctx instance in case of updating
 549	 * nr_hw_queues
 550	 */
 551	struct list_head	unused_hctx_list;
 552	spinlock_t		unused_hctx_lock;
 553
 554	int			mq_freeze_depth;
 555
 556#if defined(CONFIG_BLK_DEV_BSG)
 
 
 557	struct bsg_class_device bsg_dev;
 558#endif
 559
 
 
 
 560#ifdef CONFIG_BLK_DEV_THROTTLING
 561	/* Throttle data */
 562	struct throtl_data *td;
 563#endif
 564	struct rcu_head		rcu_head;
 565	wait_queue_head_t	mq_freeze_wq;
 566	/*
 567	 * Protect concurrent access to q_usage_counter by
 568	 * percpu_ref_kill() and percpu_ref_reinit().
 569	 */
 570	struct mutex		mq_freeze_lock;
 571	struct percpu_ref	q_usage_counter;
 572
 573	struct blk_mq_tag_set	*tag_set;
 574	struct list_head	tag_set_list;
 575	struct bio_set		bio_split;
 576
 577	struct dentry		*debugfs_dir;
 578
 579#ifdef CONFIG_BLK_DEBUG_FS
 580	struct dentry		*sched_debugfs_dir;
 581	struct dentry		*rqos_debugfs_dir;
 582#endif
 583
 584	bool			mq_sysfs_init_done;
 585
 586	size_t			cmd_size;
 587
 588#define BLK_MAX_WRITE_HINTS	5
 589	u64			write_hints[BLK_MAX_WRITE_HINTS];
 590};
 591
 592/* Keep blk_queue_flag_name[] in sync with the definitions below */
 593#define QUEUE_FLAG_STOPPED	0	/* queue is stopped */
 594#define QUEUE_FLAG_DYING	1	/* queue being torn down */
 595#define QUEUE_FLAG_NOMERGES     3	/* disable merge attempts */
 596#define QUEUE_FLAG_SAME_COMP	4	/* complete on same CPU-group */
 597#define QUEUE_FLAG_FAIL_IO	5	/* fake timeout */
 598#define QUEUE_FLAG_NONROT	6	/* non-rotational device (SSD) */
 599#define QUEUE_FLAG_VIRT		QUEUE_FLAG_NONROT /* paravirt device */
 600#define QUEUE_FLAG_IO_STAT	7	/* do disk/partitions IO accounting */
 601#define QUEUE_FLAG_DISCARD	8	/* supports DISCARD */
 602#define QUEUE_FLAG_NOXMERGES	9	/* No extended merges */
 603#define QUEUE_FLAG_ADD_RANDOM	10	/* Contributes to random pool */
 604#define QUEUE_FLAG_SECERASE	11	/* supports secure erase */
 605#define QUEUE_FLAG_SAME_FORCE	12	/* force complete on same CPU */
 606#define QUEUE_FLAG_DEAD		13	/* queue tear-down finished */
 607#define QUEUE_FLAG_INIT_DONE	14	/* queue is initialized */
 608#define QUEUE_FLAG_POLL		16	/* IO polling enabled if set */
 609#define QUEUE_FLAG_WC		17	/* Write back caching */
 610#define QUEUE_FLAG_FUA		18	/* device supports FUA writes */
 611#define QUEUE_FLAG_DAX		19	/* device supports DAX */
 612#define QUEUE_FLAG_STATS	20	/* track IO start and completion times */
 613#define QUEUE_FLAG_POLL_STATS	21	/* collecting stats for hybrid polling */
 614#define QUEUE_FLAG_REGISTERED	22	/* queue has been registered to a disk */
 615#define QUEUE_FLAG_SCSI_PASSTHROUGH 23	/* queue supports SCSI commands */
 616#define QUEUE_FLAG_QUIESCED	24	/* queue has been quiesced */
 617#define QUEUE_FLAG_PCI_P2PDMA	25	/* device supports PCI p2p requests */
 618#define QUEUE_FLAG_ZONE_RESETALL 26	/* supports Zone Reset All */
 619#define QUEUE_FLAG_RQ_ALLOC_TIME 27	/* record rq->alloc_time_ns */
 620
 621#define QUEUE_FLAG_MQ_DEFAULT	((1 << QUEUE_FLAG_IO_STAT) |		\
 622				 (1 << QUEUE_FLAG_SAME_COMP))
 623
 624void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
 625void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
 626bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
 627
 628#define blk_queue_stopped(q)	test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
 629#define blk_queue_dying(q)	test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
 630#define blk_queue_dead(q)	test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
 631#define blk_queue_init_done(q)	test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
 632#define blk_queue_nomerges(q)	test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
 633#define blk_queue_noxmerges(q)	\
 634	test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
 635#define blk_queue_nonrot(q)	test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
 636#define blk_queue_io_stat(q)	test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
 637#define blk_queue_add_random(q)	test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
 638#define blk_queue_discard(q)	test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
 639#define blk_queue_zone_resetall(q)	\
 640	test_bit(QUEUE_FLAG_ZONE_RESETALL, &(q)->queue_flags)
 641#define blk_queue_secure_erase(q) \
 642	(test_bit(QUEUE_FLAG_SECERASE, &(q)->queue_flags))
 643#define blk_queue_dax(q)	test_bit(QUEUE_FLAG_DAX, &(q)->queue_flags)
 644#define blk_queue_scsi_passthrough(q)	\
 645	test_bit(QUEUE_FLAG_SCSI_PASSTHROUGH, &(q)->queue_flags)
 646#define blk_queue_pci_p2pdma(q)	\
 647	test_bit(QUEUE_FLAG_PCI_P2PDMA, &(q)->queue_flags)
 648#ifdef CONFIG_BLK_RQ_ALLOC_TIME
 649#define blk_queue_rq_alloc_time(q)	\
 650	test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags)
 651#else
 652#define blk_queue_rq_alloc_time(q)	false
 653#endif
 654
 655#define blk_noretry_request(rq) \
 656	((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
 657			     REQ_FAILFAST_DRIVER))
 658#define blk_queue_quiesced(q)	test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags)
 659#define blk_queue_pm_only(q)	atomic_read(&(q)->pm_only)
 660#define blk_queue_fua(q)	test_bit(QUEUE_FLAG_FUA, &(q)->queue_flags)
 661#define blk_queue_registered(q)	test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags)
 662
 663extern void blk_set_pm_only(struct request_queue *q);
 664extern void blk_clear_pm_only(struct request_queue *q);
 665
 666static inline bool blk_account_rq(struct request *rq)
 667{
 668	return (rq->rq_flags & RQF_STARTED) && !blk_rq_is_passthrough(rq);
 669}
 670
 671#define list_entry_rq(ptr)	list_entry((ptr), struct request, queuelist)
 672
 673#define rq_data_dir(rq)		(op_is_write(req_op(rq)) ? WRITE : READ)
 674
 675#define rq_dma_dir(rq) \
 676	(op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE)
 677
 678#define dma_map_bvec(dev, bv, dir, attrs) \
 679	dma_map_page_attrs(dev, (bv)->bv_page, (bv)->bv_offset, (bv)->bv_len, \
 680	(dir), (attrs))
 
 681
 682static inline bool queue_is_mq(struct request_queue *q)
 683{
 684	return q->mq_ops;
 685}
 686
 687static inline enum blk_zoned_model
 688blk_queue_zoned_model(struct request_queue *q)
 689{
 690	return q->limits.zoned;
 691}
 692
 693static inline bool blk_queue_is_zoned(struct request_queue *q)
 694{
 695	switch (blk_queue_zoned_model(q)) {
 696	case BLK_ZONED_HA:
 697	case BLK_ZONED_HM:
 698		return true;
 699	default:
 700		return false;
 701	}
 702}
 703
 704static inline sector_t blk_queue_zone_sectors(struct request_queue *q)
 705{
 706	return blk_queue_is_zoned(q) ? q->limits.chunk_sectors : 0;
 707}
 708
 709#ifdef CONFIG_BLK_DEV_ZONED
 710static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
 711{
 712	return blk_queue_is_zoned(q) ? q->nr_zones : 0;
 
 713}
 714
 715static inline unsigned int blk_queue_zone_no(struct request_queue *q,
 716					     sector_t sector)
 717{
 718	if (!blk_queue_is_zoned(q))
 719		return 0;
 720	return sector >> ilog2(q->limits.chunk_sectors);
 721}
 722
 723static inline bool blk_queue_zone_is_seq(struct request_queue *q,
 724					 sector_t sector)
 725{
 726	if (!blk_queue_is_zoned(q))
 727		return false;
 728	if (!q->conv_zones_bitmap)
 729		return true;
 730	return !test_bit(blk_queue_zone_no(q, sector), q->conv_zones_bitmap);
 731}
 732
 733static inline void blk_queue_max_open_zones(struct request_queue *q,
 734		unsigned int max_open_zones)
 735{
 736	q->max_open_zones = max_open_zones;
 
 737}
 738
 739static inline unsigned int queue_max_open_zones(const struct request_queue *q)
 740{
 741	return q->max_open_zones;
 742}
 
 
 
 
 
 
 
 
 
 
 
 743
 744static inline void blk_queue_max_active_zones(struct request_queue *q,
 745		unsigned int max_active_zones)
 746{
 747	q->max_active_zones = max_active_zones;
 748}
 749
 750static inline unsigned int queue_max_active_zones(const struct request_queue *q)
 751{
 752	return q->max_active_zones;
 753}
 754#else /* CONFIG_BLK_DEV_ZONED */
 755static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
 756{
 757	return 0;
 758}
 759static inline bool blk_queue_zone_is_seq(struct request_queue *q,
 760					 sector_t sector)
 761{
 762	return false;
 763}
 764static inline unsigned int blk_queue_zone_no(struct request_queue *q,
 765					     sector_t sector)
 766{
 767	return 0;
 768}
 769static inline unsigned int queue_max_open_zones(const struct request_queue *q)
 770{
 771	return 0;
 772}
 773static inline unsigned int queue_max_active_zones(const struct request_queue *q)
 
 
 
 
 774{
 775	return 0;
 776}
 777#endif /* CONFIG_BLK_DEV_ZONED */
 778
 779static inline bool rq_is_sync(struct request *rq)
 780{
 781	return op_is_sync(rq->cmd_flags);
 782}
 783
 784static inline bool rq_mergeable(struct request *rq)
 785{
 786	if (blk_rq_is_passthrough(rq))
 787		return false;
 788
 789	if (req_op(rq) == REQ_OP_FLUSH)
 790		return false;
 791
 792	if (req_op(rq) == REQ_OP_WRITE_ZEROES)
 793		return false;
 794
 795	if (req_op(rq) == REQ_OP_ZONE_APPEND)
 796		return false;
 797
 798	if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
 799		return false;
 800	if (rq->rq_flags & RQF_NOMERGE_FLAGS)
 801		return false;
 802
 803	return true;
 804}
 805
 806static inline bool blk_write_same_mergeable(struct bio *a, struct bio *b)
 807{
 808	if (bio_page(a) == bio_page(b) &&
 809	    bio_offset(a) == bio_offset(b))
 810		return true;
 811
 812	return false;
 813}
 814
 815static inline unsigned int blk_queue_depth(struct request_queue *q)
 816{
 817	if (q->queue_depth)
 818		return q->queue_depth;
 819
 820	return q->nr_requests;
 821}
 822
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 823extern unsigned long blk_max_low_pfn, blk_max_pfn;
 824
 825/*
 826 * standard bounce addresses:
 827 *
 828 * BLK_BOUNCE_HIGH	: bounce all highmem pages
 829 * BLK_BOUNCE_ANY	: don't bounce anything
 830 * BLK_BOUNCE_ISA	: bounce pages above ISA DMA boundary
 831 */
 832
 833#if BITS_PER_LONG == 32
 834#define BLK_BOUNCE_HIGH		((u64)blk_max_low_pfn << PAGE_SHIFT)
 835#else
 836#define BLK_BOUNCE_HIGH		-1ULL
 837#endif
 838#define BLK_BOUNCE_ANY		(-1ULL)
 839#define BLK_BOUNCE_ISA		(DMA_BIT_MASK(24))
 840
 841/*
 842 * default timeout for SG_IO if none specified
 843 */
 844#define BLK_DEFAULT_SG_TIMEOUT	(60 * HZ)
 845#define BLK_MIN_SG_TIMEOUT	(7 * HZ)
 846
 
 
 
 
 
 
 
 
 
 
 
 
 
 847struct rq_map_data {
 848	struct page **pages;
 849	int page_order;
 850	int nr_entries;
 851	unsigned long offset;
 852	int null_mapped;
 853	int from_user;
 854};
 855
 856struct req_iterator {
 857	struct bvec_iter iter;
 858	struct bio *bio;
 859};
 860
 861/* This should not be used directly - use rq_for_each_segment */
 862#define for_each_bio(_bio)		\
 863	for (; _bio; _bio = _bio->bi_next)
 864#define __rq_for_each_bio(_bio, rq)	\
 865	if ((rq->bio))			\
 866		for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)
 867
 868#define rq_for_each_segment(bvl, _rq, _iter)			\
 869	__rq_for_each_bio(_iter.bio, _rq)			\
 870		bio_for_each_segment(bvl, _iter.bio, _iter.iter)
 871
 872#define rq_for_each_bvec(bvl, _rq, _iter)			\
 873	__rq_for_each_bio(_iter.bio, _rq)			\
 874		bio_for_each_bvec(bvl, _iter.bio, _iter.iter)
 875
 876#define rq_iter_last(bvec, _iter)				\
 877		(_iter.bio->bi_next == NULL &&			\
 878		 bio_iter_last(bvec, _iter.iter))
 879
 880#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
 881# error	"You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
 882#endif
 883#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
 884extern void rq_flush_dcache_pages(struct request *rq);
 885#else
 886static inline void rq_flush_dcache_pages(struct request *rq)
 887{
 888}
 889#endif
 890
 891extern int blk_register_queue(struct gendisk *disk);
 892extern void blk_unregister_queue(struct gendisk *disk);
 893blk_qc_t submit_bio_noacct(struct bio *bio);
 894extern void blk_rq_init(struct request_queue *q, struct request *rq);
 895extern void blk_put_request(struct request *);
 896extern struct request *blk_get_request(struct request_queue *, unsigned int op,
 897				       blk_mq_req_flags_t flags);
 
 
 
 
 
 
 898extern int blk_lld_busy(struct request_queue *q);
 899extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
 900			     struct bio_set *bs, gfp_t gfp_mask,
 901			     int (*bio_ctr)(struct bio *, struct bio *, void *),
 902			     void *data);
 903extern void blk_rq_unprep_clone(struct request *rq);
 904extern blk_status_t blk_insert_cloned_request(struct request_queue *q,
 905				     struct request *rq);
 906extern int blk_rq_append_bio(struct request *rq, struct bio **bio);
 907extern void blk_queue_split(struct bio **);
 908extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
 909extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
 910			      unsigned int, void __user *);
 911extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t,
 912			  unsigned int, void __user *);
 913extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
 914			 struct scsi_ioctl_command __user *);
 915extern int get_sg_io_hdr(struct sg_io_hdr *hdr, const void __user *argp);
 916extern int put_sg_io_hdr(const struct sg_io_hdr *hdr, void __user *argp);
 917
 918extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
 919extern void blk_queue_exit(struct request_queue *q);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 920extern void blk_sync_queue(struct request_queue *q);
 
 
 
 
 921extern int blk_rq_map_user(struct request_queue *, struct request *,
 922			   struct rq_map_data *, void __user *, unsigned long,
 923			   gfp_t);
 924extern int blk_rq_unmap_user(struct bio *);
 925extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t);
 926extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
 927			       struct rq_map_data *, const struct iov_iter *,
 928			       gfp_t);
 929extern void blk_execute_rq(struct request_queue *, struct gendisk *,
 930			  struct request *, int);
 931extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
 932				  struct request *, int, rq_end_io_fn *);
 933
 934/* Helper to convert REQ_OP_XXX to its string format XXX */
 935extern const char *blk_op_str(unsigned int op);
 936
 937int blk_status_to_errno(blk_status_t status);
 938blk_status_t errno_to_blk_status(int errno);
 939
 940int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin);
 941
 942static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
 943{
 944	return bdev->bd_disk->queue;	/* this is never NULL */
 945}
 946
 947/*
 948 * The basic unit of block I/O is a sector. It is used in a number of contexts
 949 * in Linux (blk, bio, genhd). The size of one sector is 512 = 2**9
 950 * bytes. Variables of type sector_t represent an offset or size that is a
 951 * multiple of 512 bytes. Hence these two constants.
 952 */
 953#ifndef SECTOR_SHIFT
 954#define SECTOR_SHIFT 9
 955#endif
 956#ifndef SECTOR_SIZE
 957#define SECTOR_SIZE (1 << SECTOR_SHIFT)
 958#endif
 959
 960/*
 961 * blk_rq_pos()			: the current sector
 962 * blk_rq_bytes()		: bytes left in the entire request
 963 * blk_rq_cur_bytes()		: bytes left in the current segment
 964 * blk_rq_err_bytes()		: bytes left till the next error boundary
 965 * blk_rq_sectors()		: sectors left in the entire request
 966 * blk_rq_cur_sectors()		: sectors left in the current segment
 967 * blk_rq_stats_sectors()	: sectors of the entire request used for stats
 968 */
 969static inline sector_t blk_rq_pos(const struct request *rq)
 970{
 971	return rq->__sector;
 972}
 973
 974static inline unsigned int blk_rq_bytes(const struct request *rq)
 975{
 976	return rq->__data_len;
 977}
 978
 979static inline int blk_rq_cur_bytes(const struct request *rq)
 980{
 981	return rq->bio ? bio_cur_bytes(rq->bio) : 0;
 982}
 983
 984extern unsigned int blk_rq_err_bytes(const struct request *rq);
 985
 986static inline unsigned int blk_rq_sectors(const struct request *rq)
 987{
 988	return blk_rq_bytes(rq) >> SECTOR_SHIFT;
 989}
 990
 991static inline unsigned int blk_rq_cur_sectors(const struct request *rq)
 992{
 993	return blk_rq_cur_bytes(rq) >> SECTOR_SHIFT;
 994}
 995
 996static inline unsigned int blk_rq_stats_sectors(const struct request *rq)
 997{
 998	return rq->stats_sectors;
 999}
1000
1001#ifdef CONFIG_BLK_DEV_ZONED
1002
1003/* Helper to convert BLK_ZONE_ZONE_XXX to its string format XXX */
1004const char *blk_zone_cond_str(enum blk_zone_cond zone_cond);
1005
1006static inline unsigned int blk_rq_zone_no(struct request *rq)
1007{
1008	return blk_queue_zone_no(rq->q, blk_rq_pos(rq));
1009}
1010
1011static inline unsigned int blk_rq_zone_is_seq(struct request *rq)
1012{
1013	return blk_queue_zone_is_seq(rq->q, blk_rq_pos(rq));
1014}
1015#endif /* CONFIG_BLK_DEV_ZONED */
1016
1017/*
1018 * Some commands like WRITE SAME have a payload or data transfer size which
1019 * is different from the size of the request.  Any driver that supports such
1020 * commands using the RQF_SPECIAL_PAYLOAD flag needs to use this helper to
1021 * calculate the data transfer size.
1022 */
1023static inline unsigned int blk_rq_payload_bytes(struct request *rq)
1024{
1025	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
1026		return rq->special_vec.bv_len;
1027	return blk_rq_bytes(rq);
1028}
1029
1030/*
1031 * Return the first full biovec in the request.  The caller needs to check that
1032 * there are any bvecs before calling this helper.
1033 */
1034static inline struct bio_vec req_bvec(struct request *rq)
1035{
1036	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
1037		return rq->special_vec;
1038	return mp_bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter);
1039}
1040
1041static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
1042						     int op)
1043{
1044	if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE))
1045		return min(q->limits.max_discard_sectors,
1046			   UINT_MAX >> SECTOR_SHIFT);
1047
1048	if (unlikely(op == REQ_OP_WRITE_SAME))
1049		return q->limits.max_write_same_sectors;
1050
1051	if (unlikely(op == REQ_OP_WRITE_ZEROES))
1052		return q->limits.max_write_zeroes_sectors;
1053
1054	return q->limits.max_sectors;
1055}
1056
1057/*
1058 * Return maximum size of a request at given offset. Only valid for
1059 * file system requests.
1060 */
1061static inline unsigned int blk_max_size_offset(struct request_queue *q,
1062					       sector_t offset)
1063{
1064	if (!q->limits.chunk_sectors)
1065		return q->limits.max_sectors;
1066
1067	return min(q->limits.max_sectors, (unsigned int)(q->limits.chunk_sectors -
1068			(offset & (q->limits.chunk_sectors - 1))));
1069}
1070
1071static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
1072						  sector_t offset)
1073{
1074	struct request_queue *q = rq->q;
1075
1076	if (blk_rq_is_passthrough(rq))
1077		return q->limits.max_hw_sectors;
1078
1079	if (!q->limits.chunk_sectors ||
1080	    req_op(rq) == REQ_OP_DISCARD ||
1081	    req_op(rq) == REQ_OP_SECURE_ERASE)
1082		return blk_queue_get_max_sectors(q, req_op(rq));
1083
1084	return min(blk_max_size_offset(q, offset),
1085			blk_queue_get_max_sectors(q, req_op(rq)));
1086}
1087
1088static inline unsigned int blk_rq_count_bios(struct request *rq)
1089{
1090	unsigned int nr_bios = 0;
1091	struct bio *bio;
1092
1093	__rq_for_each_bio(bio, rq)
1094		nr_bios++;
1095
1096	return nr_bios;
1097}
1098
1099void blk_steal_bios(struct bio_list *list, struct request *rq);
1100
1101/*
1102 * Request completion related functions.
1103 *
1104 * blk_update_request() completes given number of bytes and updates
1105 * the request without completing it.
 
 
 
 
 
 
 
1106 */
1107extern bool blk_update_request(struct request *rq, blk_status_t error,
1108			       unsigned int nr_bytes);
 
 
 
 
 
 
 
 
 
 
1109
 
 
1110extern void blk_abort_request(struct request *);
 
1111
1112/*
1113 * Access functions for manipulating queue properties
1114 */
 
 
 
 
 
1115extern void blk_cleanup_queue(struct request_queue *);
 
1116extern void blk_queue_bounce_limit(struct request_queue *, u64);
 
1117extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
1118extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
1119extern void blk_queue_max_segments(struct request_queue *, unsigned short);
1120extern void blk_queue_max_discard_segments(struct request_queue *,
1121		unsigned short);
1122extern void blk_queue_max_segment_size(struct request_queue *, unsigned int);
1123extern void blk_queue_max_discard_sectors(struct request_queue *q,
1124		unsigned int max_discard_sectors);
1125extern void blk_queue_max_write_same_sectors(struct request_queue *q,
1126		unsigned int max_write_same_sectors);
1127extern void blk_queue_max_write_zeroes_sectors(struct request_queue *q,
1128		unsigned int max_write_same_sectors);
1129extern void blk_queue_logical_block_size(struct request_queue *, unsigned int);
1130extern void blk_queue_max_zone_append_sectors(struct request_queue *q,
1131		unsigned int max_zone_append_sectors);
1132extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
1133extern void blk_queue_alignment_offset(struct request_queue *q,
1134				       unsigned int alignment);
1135extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
1136extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
1137extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
1138extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt);
1139extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth);
1140extern void blk_set_default_limits(struct queue_limits *lim);
1141extern void blk_set_stacking_limits(struct queue_limits *lim);
1142extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
1143			    sector_t offset);
 
 
1144extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
1145			      sector_t offset);
 
 
1146extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
 
 
 
 
1147extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
1148extern void blk_queue_virt_boundary(struct request_queue *, unsigned long);
 
 
1149extern void blk_queue_dma_alignment(struct request_queue *, int);
1150extern void blk_queue_update_dma_alignment(struct request_queue *, int);
 
 
1151extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
1152extern void blk_queue_write_cache(struct request_queue *q, bool enabled, bool fua);
1153extern void blk_queue_required_elevator_features(struct request_queue *q,
1154						 unsigned int features);
1155extern bool blk_queue_can_use_dma_map_merging(struct request_queue *q,
1156					      struct device *dev);
1157
1158/*
1159 * Number of physical segments as sent to the device.
1160 *
1161 * Normally this is the number of discontiguous data segments sent by the
1162 * submitter.  But for data-less command like discard we might have no
1163 * actual data segments submitted, but the driver might have to add it's
1164 * own special payload.  In that case we still return 1 here so that this
1165 * special payload will be mapped.
1166 */
1167static inline unsigned short blk_rq_nr_phys_segments(struct request *rq)
1168{
1169	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
1170		return 1;
1171	return rq->nr_phys_segments;
1172}
1173
1174/*
1175 * Number of discard segments (or ranges) the driver needs to fill in.
1176 * Each discard bio merged into a request is counted as one segment.
1177 */
1178static inline unsigned short blk_rq_nr_discard_segments(struct request *rq)
1179{
1180	return max_t(unsigned short, rq->nr_phys_segments, 1);
1181}
1182
1183int __blk_rq_map_sg(struct request_queue *q, struct request *rq,
1184		struct scatterlist *sglist, struct scatterlist **last_sg);
1185static inline int blk_rq_map_sg(struct request_queue *q, struct request *rq,
1186		struct scatterlist *sglist)
1187{
1188	struct scatterlist *last_sg = NULL;
1189
1190	return __blk_rq_map_sg(q, rq, sglist, &last_sg);
1191}
1192extern void blk_dump_rq_flags(struct request *, char *);
 
1193
1194bool __must_check blk_get_queue(struct request_queue *);
1195struct request_queue *blk_alloc_queue(int node_id);
 
1196extern void blk_put_queue(struct request_queue *);
1197extern void blk_set_queue_dying(struct request_queue *);
1198
1199#ifdef CONFIG_BLOCK
1200/*
1201 * blk_plug permits building a queue of related requests by holding the I/O
1202 * fragments for a short period. This allows merging of sequential requests
1203 * into single larger request. As the requests are moved from a per-task list to
1204 * the device's request_queue in a batch, this results in improved scalability
1205 * as the lock contention for request_queue lock is reduced.
1206 *
1207 * It is ok not to disable preemption when adding the request to the plug list
1208 * or when attempting a merge, because blk_schedule_flush_list() will only flush
1209 * the plug list when the task sleeps by itself. For details, please see
1210 * schedule() where blk_schedule_flush_plug() is called.
1211 */
1212struct blk_plug {
1213	struct list_head mq_list; /* blk-mq requests */
 
1214	struct list_head cb_list; /* md requires an unplug callback */
1215	unsigned short rq_count;
1216	bool multiple_queues;
1217	bool nowait;
1218};
1219#define BLK_MAX_REQUEST_COUNT 16
1220#define BLK_PLUG_FLUSH_SIZE (128 * 1024)
1221
1222struct blk_plug_cb;
1223typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
1224struct blk_plug_cb {
1225	struct list_head list;
1226	blk_plug_cb_fn callback;
1227	void *data;
1228};
1229extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug,
1230					     void *data, int size);
1231extern void blk_start_plug(struct blk_plug *);
1232extern void blk_finish_plug(struct blk_plug *);
1233extern void blk_flush_plug_list(struct blk_plug *, bool);
1234
1235static inline void blk_flush_plug(struct task_struct *tsk)
1236{
1237	struct blk_plug *plug = tsk->plug;
1238
1239	if (plug)
1240		blk_flush_plug_list(plug, false);
1241}
1242
1243static inline void blk_schedule_flush_plug(struct task_struct *tsk)
1244{
1245	struct blk_plug *plug = tsk->plug;
1246
1247	if (plug)
1248		blk_flush_plug_list(plug, true);
1249}
1250
1251static inline bool blk_needs_flush_plug(struct task_struct *tsk)
1252{
1253	struct blk_plug *plug = tsk->plug;
1254
1255	return plug &&
1256		 (!list_empty(&plug->mq_list) ||
1257		 !list_empty(&plug->cb_list));
1258}
1259
1260int blkdev_issue_flush(struct block_device *, gfp_t);
1261long nr_blockdev_pages(void);
1262#else /* CONFIG_BLOCK */
1263struct blk_plug {
1264};
1265
1266static inline void blk_start_plug(struct blk_plug *plug)
1267{
1268}
1269
1270static inline void blk_finish_plug(struct blk_plug *plug)
1271{
1272}
1273
1274static inline void blk_flush_plug(struct task_struct *task)
1275{
1276}
1277
1278static inline void blk_schedule_flush_plug(struct task_struct *task)
1279{
1280}
1281
1282
1283static inline bool blk_needs_flush_plug(struct task_struct *tsk)
1284{
1285	return false;
1286}
1287
1288static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask)
1289{
1290	return 0;
1291}
1292
1293static inline long nr_blockdev_pages(void)
 
1294{
1295	return 0;
 
 
1296}
1297#endif /* CONFIG_BLOCK */
1298
1299extern void blk_io_schedule(void);
1300
1301extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
1302		sector_t nr_sects, gfp_t gfp_mask, struct page *page);
1303
1304#define BLKDEV_DISCARD_SECURE	(1 << 0)	/* issue a secure erase */
1305
 
1306extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1307		sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
1308extern int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1309		sector_t nr_sects, gfp_t gfp_mask, int flags,
1310		struct bio **biop);
1311
1312#define BLKDEV_ZERO_NOUNMAP	(1 << 0)  /* do not free blocks */
1313#define BLKDEV_ZERO_NOFALLBACK	(1 << 1)  /* don't write explicit zeroes */
1314
1315extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1316		sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
1317		unsigned flags);
1318extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1319		sector_t nr_sects, gfp_t gfp_mask, unsigned flags);
1320
1321static inline int sb_issue_discard(struct super_block *sb, sector_t block,
1322		sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
1323{
1324	return blkdev_issue_discard(sb->s_bdev,
1325				    block << (sb->s_blocksize_bits -
1326					      SECTOR_SHIFT),
1327				    nr_blocks << (sb->s_blocksize_bits -
1328						  SECTOR_SHIFT),
1329				    gfp_mask, flags);
1330}
1331static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
1332		sector_t nr_blocks, gfp_t gfp_mask)
1333{
1334	return blkdev_issue_zeroout(sb->s_bdev,
1335				    block << (sb->s_blocksize_bits -
1336					      SECTOR_SHIFT),
1337				    nr_blocks << (sb->s_blocksize_bits -
1338						  SECTOR_SHIFT),
1339				    gfp_mask, 0);
1340}
1341
1342extern int blk_verify_command(unsigned char *cmd, fmode_t mode);
1343
1344enum blk_default_limits {
1345	BLK_MAX_SEGMENTS	= 128,
1346	BLK_SAFE_MAX_SECTORS	= 255,
1347	BLK_DEF_MAX_SECTORS	= 2560,
1348	BLK_MAX_SEGMENT_SIZE	= 65536,
1349	BLK_SEG_BOUNDARY_MASK	= 0xFFFFFFFFUL,
1350};
1351
1352static inline unsigned long queue_segment_boundary(const struct request_queue *q)
 
 
1353{
1354	return q->limits.seg_boundary_mask;
1355}
1356
1357static inline unsigned long queue_virt_boundary(const struct request_queue *q)
1358{
1359	return q->limits.virt_boundary_mask;
1360}
1361
1362static inline unsigned int queue_max_sectors(const struct request_queue *q)
1363{
1364	return q->limits.max_sectors;
1365}
1366
1367static inline unsigned int queue_max_hw_sectors(const struct request_queue *q)
1368{
1369	return q->limits.max_hw_sectors;
1370}
1371
1372static inline unsigned short queue_max_segments(const struct request_queue *q)
1373{
1374	return q->limits.max_segments;
1375}
1376
1377static inline unsigned short queue_max_discard_segments(const struct request_queue *q)
1378{
1379	return q->limits.max_discard_segments;
1380}
1381
1382static inline unsigned int queue_max_segment_size(const struct request_queue *q)
1383{
1384	return q->limits.max_segment_size;
1385}
1386
1387static inline unsigned int queue_max_zone_append_sectors(const struct request_queue *q)
1388{
1389	return q->limits.max_zone_append_sectors;
1390}
1391
1392static inline unsigned queue_logical_block_size(const struct request_queue *q)
1393{
1394	int retval = 512;
1395
1396	if (q && q->limits.logical_block_size)
1397		retval = q->limits.logical_block_size;
1398
1399	return retval;
1400}
1401
1402static inline unsigned int bdev_logical_block_size(struct block_device *bdev)
1403{
1404	return queue_logical_block_size(bdev_get_queue(bdev));
1405}
1406
1407static inline unsigned int queue_physical_block_size(const struct request_queue *q)
1408{
1409	return q->limits.physical_block_size;
1410}
1411
1412static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
1413{
1414	return queue_physical_block_size(bdev_get_queue(bdev));
1415}
1416
1417static inline unsigned int queue_io_min(const struct request_queue *q)
1418{
1419	return q->limits.io_min;
1420}
1421
1422static inline int bdev_io_min(struct block_device *bdev)
1423{
1424	return queue_io_min(bdev_get_queue(bdev));
1425}
1426
1427static inline unsigned int queue_io_opt(const struct request_queue *q)
1428{
1429	return q->limits.io_opt;
1430}
1431
1432static inline int bdev_io_opt(struct block_device *bdev)
1433{
1434	return queue_io_opt(bdev_get_queue(bdev));
1435}
1436
1437static inline int queue_alignment_offset(const struct request_queue *q)
1438{
1439	if (q->limits.misaligned)
1440		return -1;
1441
1442	return q->limits.alignment_offset;
1443}
1444
1445static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector)
1446{
1447	unsigned int granularity = max(lim->physical_block_size, lim->io_min);
1448	unsigned int alignment = sector_div(sector, granularity >> SECTOR_SHIFT)
1449		<< SECTOR_SHIFT;
1450
1451	return (granularity + lim->alignment_offset - alignment) % granularity;
 
1452}
1453
1454static inline int bdev_alignment_offset(struct block_device *bdev)
1455{
1456	struct request_queue *q = bdev_get_queue(bdev);
1457
1458	if (q->limits.misaligned)
1459		return -1;
1460
1461	if (bdev != bdev->bd_contains)
1462		return bdev->bd_part->alignment_offset;
1463
1464	return q->limits.alignment_offset;
1465}
1466
1467static inline int queue_discard_alignment(const struct request_queue *q)
1468{
1469	if (q->limits.discard_misaligned)
1470		return -1;
1471
1472	return q->limits.discard_alignment;
1473}
1474
1475static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector)
1476{
1477	unsigned int alignment, granularity, offset;
1478
1479	if (!lim->max_discard_sectors)
1480		return 0;
1481
1482	/* Why are these in bytes, not sectors? */
1483	alignment = lim->discard_alignment >> SECTOR_SHIFT;
1484	granularity = lim->discard_granularity >> SECTOR_SHIFT;
1485	if (!granularity)
1486		return 0;
1487
1488	/* Offset of the partition start in 'granularity' sectors */
1489	offset = sector_div(sector, granularity);
1490
1491	/* And why do we do this modulus *again* in blkdev_issue_discard()? */
1492	offset = (granularity + alignment - offset) % granularity;
1493
1494	/* Turn it back into bytes, gaah */
1495	return offset << SECTOR_SHIFT;
1496}
1497
1498static inline int bdev_discard_alignment(struct block_device *bdev)
1499{
1500	struct request_queue *q = bdev_get_queue(bdev);
1501
1502	if (bdev != bdev->bd_contains)
1503		return bdev->bd_part->discard_alignment;
1504
1505	return q->limits.discard_alignment;
1506}
1507
1508static inline unsigned int bdev_write_same(struct block_device *bdev)
1509{
1510	struct request_queue *q = bdev_get_queue(bdev);
1511
1512	if (q)
1513		return q->limits.max_write_same_sectors;
1514
1515	return 0;
1516}
1517
1518static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev)
1519{
1520	struct request_queue *q = bdev_get_queue(bdev);
1521
1522	if (q)
1523		return q->limits.max_write_zeroes_sectors;
1524
1525	return 0;
1526}
1527
1528static inline enum blk_zoned_model bdev_zoned_model(struct block_device *bdev)
 
1529{
1530	struct request_queue *q = bdev_get_queue(bdev);
 
 
1531
1532	if (q)
1533		return blk_queue_zoned_model(q);
 
 
 
 
 
 
 
 
1534
1535	return BLK_ZONED_NONE;
 
 
1536}
1537
1538static inline bool bdev_is_zoned(struct block_device *bdev)
1539{
1540	struct request_queue *q = bdev_get_queue(bdev);
 
1541
1542	if (q)
1543		return blk_queue_is_zoned(q);
1544
1545	return false;
1546}
1547
1548static inline sector_t bdev_zone_sectors(struct block_device *bdev)
1549{
1550	struct request_queue *q = bdev_get_queue(bdev);
1551
1552	if (q)
1553		return blk_queue_zone_sectors(q);
1554	return 0;
1555}
1556
1557static inline unsigned int bdev_max_open_zones(struct block_device *bdev)
1558{
1559	struct request_queue *q = bdev_get_queue(bdev);
1560
1561	if (q)
1562		return queue_max_open_zones(q);
1563	return 0;
 
 
 
 
 
 
 
 
1564}
1565
1566static inline unsigned int bdev_max_active_zones(struct block_device *bdev)
1567{
1568	struct request_queue *q = bdev_get_queue(bdev);
1569
1570	if (q)
1571		return queue_max_active_zones(q);
1572	return 0;
1573}
1574
1575static inline int queue_dma_alignment(const struct request_queue *q)
1576{
1577	return q ? q->dma_alignment : 511;
1578}
1579
1580static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr,
1581				 unsigned int len)
1582{
1583	unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask;
1584	return !(addr & alignment) && !(len & alignment);
1585}
1586
1587/* assumes size > 256 */
1588static inline unsigned int blksize_bits(unsigned int size)
 
1589{
1590	unsigned int bits = 8;
1591	do {
1592		bits++;
1593		size >>= 1;
1594	} while (size > 256);
1595	return bits;
1596}
1597
1598static inline unsigned int block_size(struct block_device *bdev)
1599{
1600	return 1 << bdev->bd_inode->i_blkbits;
1601}
1602
1603int kblockd_schedule_work(struct work_struct *work);
1604int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
1605
1606#define MODULE_ALIAS_BLOCKDEV(major,minor) \
1607	MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
1608#define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
1609	MODULE_ALIAS("block-major-" __stringify(major) "-*")
1610
1611#if defined(CONFIG_BLK_DEV_INTEGRITY)
1612
1613enum blk_integrity_flags {
1614	BLK_INTEGRITY_VERIFY		= 1 << 0,
1615	BLK_INTEGRITY_GENERATE		= 1 << 1,
1616	BLK_INTEGRITY_DEVICE_CAPABLE	= 1 << 2,
1617	BLK_INTEGRITY_IP_CHECKSUM	= 1 << 3,
1618};
1619
1620struct blk_integrity_iter {
1621	void			*prot_buf;
1622	void			*data_buf;
1623	sector_t		seed;
1624	unsigned int		data_size;
1625	unsigned short		interval;
1626	const char		*disk_name;
1627};
1628
1629typedef blk_status_t (integrity_processing_fn) (struct blk_integrity_iter *);
1630typedef void (integrity_prepare_fn) (struct request *);
1631typedef void (integrity_complete_fn) (struct request *, unsigned int);
1632
1633struct blk_integrity_profile {
1634	integrity_processing_fn		*generate_fn;
1635	integrity_processing_fn		*verify_fn;
1636	integrity_prepare_fn		*prepare_fn;
1637	integrity_complete_fn		*complete_fn;
1638	const char			*name;
 
 
 
 
 
 
 
 
 
1639};
1640
1641extern void blk_integrity_register(struct gendisk *, struct blk_integrity *);
 
1642extern void blk_integrity_unregister(struct gendisk *);
1643extern int blk_integrity_compare(struct gendisk *, struct gendisk *);
1644extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *,
1645				   struct scatterlist *);
1646extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
1647extern bool blk_integrity_merge_rq(struct request_queue *, struct request *,
1648				   struct request *);
1649extern bool blk_integrity_merge_bio(struct request_queue *, struct request *,
1650				    struct bio *);
1651
1652static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
1653{
1654	struct blk_integrity *bi = &disk->queue->integrity;
1655
1656	if (!bi->profile)
1657		return NULL;
1658
1659	return bi;
1660}
1661
1662static inline
1663struct blk_integrity *bdev_get_integrity(struct block_device *bdev)
1664{
1665	return blk_get_integrity(bdev->bd_disk);
1666}
1667
1668static inline bool
1669blk_integrity_queue_supports_integrity(struct request_queue *q)
1670{
1671	return q->integrity.profile;
1672}
1673
1674static inline bool blk_integrity_rq(struct request *rq)
1675{
1676	return rq->cmd_flags & REQ_INTEGRITY;
 
 
 
1677}
1678
1679static inline void blk_queue_max_integrity_segments(struct request_queue *q,
1680						    unsigned int segs)
1681{
1682	q->limits.max_integrity_segments = segs;
1683}
1684
1685static inline unsigned short
1686queue_max_integrity_segments(const struct request_queue *q)
1687{
1688	return q->limits.max_integrity_segments;
1689}
1690
1691/**
1692 * bio_integrity_intervals - Return number of integrity intervals for a bio
1693 * @bi:		blk_integrity profile for device
1694 * @sectors:	Size of the bio in 512-byte sectors
1695 *
1696 * Description: The block layer calculates everything in 512 byte
1697 * sectors but integrity metadata is done in terms of the data integrity
1698 * interval size of the storage device.  Convert the block layer sectors
1699 * to the appropriate number of integrity intervals.
1700 */
1701static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi,
1702						   unsigned int sectors)
1703{
1704	return sectors >> (bi->interval_exp - 9);
1705}
1706
1707static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi,
1708					       unsigned int sectors)
1709{
1710	return bio_integrity_intervals(bi, sectors) * bi->tuple_size;
1711}
1712
1713/*
1714 * Return the first bvec that contains integrity data.  Only drivers that are
1715 * limited to a single integrity segment should use this helper.
1716 */
1717static inline struct bio_vec *rq_integrity_vec(struct request *rq)
1718{
1719	if (WARN_ON_ONCE(queue_max_integrity_segments(rq->q) > 1))
1720		return NULL;
1721	return rq->bio->bi_integrity->bip_vec;
1722}
1723
1724#else /* CONFIG_BLK_DEV_INTEGRITY */
1725
1726struct bio;
1727struct block_device;
1728struct gendisk;
1729struct blk_integrity;
1730
1731static inline int blk_integrity_rq(struct request *rq)
1732{
1733	return 0;
1734}
1735static inline int blk_rq_count_integrity_sg(struct request_queue *q,
1736					    struct bio *b)
1737{
1738	return 0;
1739}
1740static inline int blk_rq_map_integrity_sg(struct request_queue *q,
1741					  struct bio *b,
1742					  struct scatterlist *s)
1743{
1744	return 0;
1745}
1746static inline struct blk_integrity *bdev_get_integrity(struct block_device *b)
1747{
1748	return NULL;
1749}
1750static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
1751{
1752	return NULL;
1753}
1754static inline bool
1755blk_integrity_queue_supports_integrity(struct request_queue *q)
1756{
1757	return false;
1758}
1759static inline int blk_integrity_compare(struct gendisk *a, struct gendisk *b)
1760{
1761	return 0;
1762}
1763static inline void blk_integrity_register(struct gendisk *d,
1764					 struct blk_integrity *b)
1765{
 
1766}
1767static inline void blk_integrity_unregister(struct gendisk *d)
1768{
1769}
1770static inline void blk_queue_max_integrity_segments(struct request_queue *q,
1771						    unsigned int segs)
1772{
1773}
1774static inline unsigned short queue_max_integrity_segments(const struct request_queue *q)
1775{
1776	return 0;
1777}
1778static inline bool blk_integrity_merge_rq(struct request_queue *rq,
1779					  struct request *r1,
1780					  struct request *r2)
1781{
1782	return true;
1783}
1784static inline bool blk_integrity_merge_bio(struct request_queue *rq,
1785					   struct request *r,
1786					   struct bio *b)
1787{
1788	return true;
1789}
1790
1791static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi,
1792						   unsigned int sectors)
1793{
1794	return 0;
1795}
1796
1797static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi,
1798					       unsigned int sectors)
1799{
1800	return 0;
1801}
1802
1803static inline struct bio_vec *rq_integrity_vec(struct request *rq)
1804{
1805	return NULL;
1806}
1807
1808#endif /* CONFIG_BLK_DEV_INTEGRITY */
1809
1810#ifdef CONFIG_BLK_INLINE_ENCRYPTION
1811
1812bool blk_ksm_register(struct blk_keyslot_manager *ksm, struct request_queue *q);
1813
1814void blk_ksm_unregister(struct request_queue *q);
1815
1816#else /* CONFIG_BLK_INLINE_ENCRYPTION */
1817
1818static inline bool blk_ksm_register(struct blk_keyslot_manager *ksm,
1819				    struct request_queue *q)
1820{
1821	return true;
1822}
1823
1824static inline void blk_ksm_unregister(struct request_queue *q) { }
1825
1826#endif /* CONFIG_BLK_INLINE_ENCRYPTION */
1827
1828
1829struct block_device_operations {
1830	blk_qc_t (*submit_bio) (struct bio *bio);
1831	int (*open) (struct block_device *, fmode_t);
1832	void (*release) (struct gendisk *, fmode_t);
1833	int (*rw_page)(struct block_device *, sector_t, struct page *, unsigned int);
1834	int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
1835	int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
 
 
1836	unsigned int (*check_events) (struct gendisk *disk,
1837				      unsigned int clearing);
 
 
1838	void (*unlock_native_capacity) (struct gendisk *);
1839	int (*revalidate_disk) (struct gendisk *);
1840	int (*getgeo)(struct block_device *, struct hd_geometry *);
1841	/* this callback is with swap_lock and sometimes page table lock held */
1842	void (*swap_slot_free_notify) (struct block_device *, unsigned long);
1843	int (*report_zones)(struct gendisk *, sector_t sector,
1844			unsigned int nr_zones, report_zones_cb cb, void *data);
1845	char *(*devnode)(struct gendisk *disk, umode_t *mode);
1846	struct module *owner;
1847	const struct pr_ops *pr_ops;
1848};
1849
1850#ifdef CONFIG_COMPAT
1851extern int blkdev_compat_ptr_ioctl(struct block_device *, fmode_t,
1852				      unsigned int, unsigned long);
1853#else
1854#define blkdev_compat_ptr_ioctl NULL
1855#endif
1856
1857extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int,
1858				 unsigned long);
1859extern int bdev_read_page(struct block_device *, sector_t, struct page *);
1860extern int bdev_write_page(struct block_device *, sector_t, struct page *,
1861						struct writeback_control *);
1862
1863#ifdef CONFIG_BLK_DEV_ZONED
1864bool blk_req_needs_zone_write_lock(struct request *rq);
1865bool blk_req_zone_write_trylock(struct request *rq);
1866void __blk_req_zone_write_lock(struct request *rq);
1867void __blk_req_zone_write_unlock(struct request *rq);
1868
1869static inline void blk_req_zone_write_lock(struct request *rq)
1870{
1871	if (blk_req_needs_zone_write_lock(rq))
1872		__blk_req_zone_write_lock(rq);
1873}
1874
1875static inline void blk_req_zone_write_unlock(struct request *rq)
1876{
1877	if (rq->rq_flags & RQF_ZONE_WRITE_LOCKED)
1878		__blk_req_zone_write_unlock(rq);
1879}
1880
1881static inline bool blk_req_zone_is_write_locked(struct request *rq)
1882{
1883	return rq->q->seq_zones_wlock &&
1884		test_bit(blk_rq_zone_no(rq), rq->q->seq_zones_wlock);
1885}
1886
1887static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
1888{
1889	if (!blk_req_needs_zone_write_lock(rq))
1890		return true;
1891	return !blk_req_zone_is_write_locked(rq);
1892}
1893#else
1894static inline bool blk_req_needs_zone_write_lock(struct request *rq)
1895{
1896	return false;
1897}
1898
1899static inline void blk_req_zone_write_lock(struct request *rq)
1900{
1901}
1902
1903static inline void blk_req_zone_write_unlock(struct request *rq)
1904{
1905}
1906static inline bool blk_req_zone_is_write_locked(struct request *rq)
1907{
1908	return false;
1909}
1910
1911static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
1912{
1913	return true;
1914}
1915#endif /* CONFIG_BLK_DEV_ZONED */
1916
1917static inline void blk_wake_io_task(struct task_struct *waiter)
1918{
1919	/*
1920	 * If we're polling, the task itself is doing the completions. For
1921	 * that case, we don't need to signal a wakeup, it's enough to just
1922	 * mark us as RUNNING.
1923	 */
1924	if (waiter == current)
1925		__set_current_state(TASK_RUNNING);
1926	else
1927		wake_up_process(waiter);
1928}
1929
1930unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,
1931		unsigned int op);
1932void disk_end_io_acct(struct gendisk *disk, unsigned int op,
1933		unsigned long start_time);
1934
1935/**
1936 * bio_start_io_acct - start I/O accounting for bio based drivers
1937 * @bio:	bio to start account for
1938 *
1939 * Returns the start time that should be passed back to bio_end_io_acct().
1940 */
1941static inline unsigned long bio_start_io_acct(struct bio *bio)
1942{
1943	return disk_start_io_acct(bio->bi_disk, bio_sectors(bio), bio_op(bio));
1944}
1945
1946/**
1947 * bio_end_io_acct - end I/O accounting for bio based drivers
1948 * @bio:	bio to end account for
1949 * @start:	start time returned by bio_start_io_acct()
1950 */
1951static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time)
1952{
1953	return disk_end_io_acct(bio->bi_disk, bio_op(bio), start_time);
1954}
1955
1956int bdev_read_only(struct block_device *bdev);
1957int set_blocksize(struct block_device *bdev, int size);
1958
1959const char *bdevname(struct block_device *bdev, char *buffer);
1960struct block_device *lookup_bdev(const char *);
1961
1962void blkdev_show(struct seq_file *seqf, off_t offset);
1963
1964#define BDEVNAME_SIZE	32	/* Largest string for a blockdev identifier */
1965#define BDEVT_SIZE	10	/* Largest string for MAJ:MIN for blkdev */
1966#ifdef CONFIG_BLOCK
1967#define BLKDEV_MAJOR_MAX	512
1968#else
1969#define BLKDEV_MAJOR_MAX	0
1970#endif
1971
1972int blkdev_get(struct block_device *bdev, fmode_t mode, void *holder);
1973struct block_device *blkdev_get_by_path(const char *path, fmode_t mode,
1974		void *holder);
1975struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder);
1976int bd_prepare_to_claim(struct block_device *bdev, struct block_device *whole,
1977		void *holder);
1978void bd_abort_claiming(struct block_device *bdev, struct block_device *whole,
1979		void *holder);
1980void blkdev_put(struct block_device *bdev, fmode_t mode);
1981
1982struct block_device *I_BDEV(struct inode *inode);
1983struct block_device *bdget(dev_t);
1984struct block_device *bdgrab(struct block_device *bdev);
1985void bdput(struct block_device *);
1986
1987#ifdef CONFIG_BLOCK
1988void invalidate_bdev(struct block_device *bdev);
1989int sync_blockdev(struct block_device *bdev);
1990#else
1991static inline void invalidate_bdev(struct block_device *bdev)
1992{
1993}
1994static inline int sync_blockdev(struct block_device *bdev)
1995{
1996	return 0;
1997}
1998#endif
1999int fsync_bdev(struct block_device *bdev);
2000
2001struct super_block *freeze_bdev(struct block_device *bdev);
2002int thaw_bdev(struct block_device *bdev, struct super_block *sb);
2003
2004#endif /* _LINUX_BLKDEV_H */