1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
   4 */
   5#include <linux/mm.h>
   6#include <linux/swap.h>
   7#include <linux/bio.h>
   8#include <linux/blkdev.h>
   9#include <linux/uio.h>
  10#include <linux/iocontext.h>
  11#include <linux/slab.h>
  12#include <linux/init.h>
  13#include <linux/kernel.h>
  14#include <linux/export.h>
  15#include <linux/mempool.h>
  16#include <linux/workqueue.h>
  17#include <linux/cgroup.h>
  18#include <linux/blk-cgroup.h>
  19#include <linux/highmem.h>
 
 
  20
  21#include <trace/events/block.h>
  22#include "blk.h"
  23#include "blk-rq-qos.h"
  24
  25/*
  26 * Test patch to inline a certain number of bi_io_vec's inside the bio
  27 * itself, to shrink a bio data allocation from two mempool calls to one
  28 */
  29#define BIO_INLINE_VECS		4
  30
  31/*
  32 * if you change this list, also change bvec_alloc or things will
  33 * break badly! cannot be bigger than what you can fit into an
  34 * unsigned short
  35 */
  36#define BV(x, n) { .nr_vecs = x, .name = "biovec-"#n }
  37static struct biovec_slab bvec_slabs[BVEC_POOL_NR] __read_mostly = {
  38	BV(1, 1), BV(4, 4), BV(16, 16), BV(64, 64), BV(128, 128), BV(BIO_MAX_PAGES, max),
  39};
  40#undef BV
  41
  42/*
  43 * fs_bio_set is the bio_set containing bio and iovec memory pools used by
  44 * IO code that does not need private memory pools.
  45 */
  46struct bio_set fs_bio_set;
  47EXPORT_SYMBOL(fs_bio_set);
  48
  49/*
  50 * Our slab pool management
  51 */
  52struct bio_slab {
  53	struct kmem_cache *slab;
  54	unsigned int slab_ref;
  55	unsigned int slab_size;
  56	char name[8];
  57};
  58static DEFINE_MUTEX(bio_slab_lock);
  59static struct bio_slab *bio_slabs;
  60static unsigned int bio_slab_nr, bio_slab_max;
  61
  62static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size)
  63{
  64	unsigned int sz = sizeof(struct bio) + extra_size;
  65	struct kmem_cache *slab = NULL;
  66	struct bio_slab *bslab, *new_bio_slabs;
  67	unsigned int new_bio_slab_max;
  68	unsigned int i, entry = -1;
  69
  70	mutex_lock(&bio_slab_lock);
  71
  72	i = 0;
  73	while (i < bio_slab_nr) {
  74		bslab = &bio_slabs[i];
  75
  76		if (!bslab->slab && entry == -1)
  77			entry = i;
  78		else if (bslab->slab_size == sz) {
  79			slab = bslab->slab;
  80			bslab->slab_ref++;
  81			break;
  82		}
  83		i++;
  84	}
  85
  86	if (slab)
  87		goto out_unlock;
  88
  89	if (bio_slab_nr == bio_slab_max && entry == -1) {
  90		new_bio_slab_max = bio_slab_max << 1;
  91		new_bio_slabs = krealloc(bio_slabs,
  92					 new_bio_slab_max * sizeof(struct bio_slab),
  93					 GFP_KERNEL);
  94		if (!new_bio_slabs)
  95			goto out_unlock;
  96		bio_slab_max = new_bio_slab_max;
  97		bio_slabs = new_bio_slabs;
  98	}
  99	if (entry == -1)
 100		entry = bio_slab_nr++;
 101
 102	bslab = &bio_slabs[entry];
 103
 104	snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry);
 105	slab = kmem_cache_create(bslab->name, sz, ARCH_KMALLOC_MINALIGN,
 106				 SLAB_HWCACHE_ALIGN, NULL);
 107	if (!slab)
 108		goto out_unlock;
 109
 110	bslab->slab = slab;
 111	bslab->slab_ref = 1;
 112	bslab->slab_size = sz;
 113out_unlock:
 114	mutex_unlock(&bio_slab_lock);
 115	return slab;
 116}
 117
 118static void bio_put_slab(struct bio_set *bs)
 119{
 120	struct bio_slab *bslab = NULL;
 121	unsigned int i;
 122
 123	mutex_lock(&bio_slab_lock);
 124
 125	for (i = 0; i < bio_slab_nr; i++) {
 126		if (bs->bio_slab == bio_slabs[i].slab) {
 127			bslab = &bio_slabs[i];
 128			break;
 129		}
 130	}
 131
 132	if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))
 133		goto out;
 134
 135	WARN_ON(!bslab->slab_ref);
 136
 137	if (--bslab->slab_ref)
 138		goto out;
 139
 140	kmem_cache_destroy(bslab->slab);
 141	bslab->slab = NULL;
 142
 143out:
 144	mutex_unlock(&bio_slab_lock);
 145}
 146
 147unsigned int bvec_nr_vecs(unsigned short idx)
 148{
 149	return bvec_slabs[--idx].nr_vecs;
 150}
 151
 152void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
 153{
 154	if (!idx)
 155		return;
 156	idx--;
 157
 158	BIO_BUG_ON(idx >= BVEC_POOL_NR);
 159
 160	if (idx == BVEC_POOL_MAX) {
 161		mempool_free(bv, pool);
 162	} else {
 163		struct biovec_slab *bvs = bvec_slabs + idx;
 164
 165		kmem_cache_free(bvs->slab, bv);
 166	}
 167}
 168
 169struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
 170			   mempool_t *pool)
 171{
 172	struct bio_vec *bvl;
 173
 174	/*
 175	 * see comment near bvec_array define!
 176	 */
 177	switch (nr) {
 178	case 1:
 179		*idx = 0;
 180		break;
 181	case 2 ... 4:
 182		*idx = 1;
 183		break;
 184	case 5 ... 16:
 185		*idx = 2;
 186		break;
 187	case 17 ... 64:
 188		*idx = 3;
 189		break;
 190	case 65 ... 128:
 191		*idx = 4;
 192		break;
 193	case 129 ... BIO_MAX_PAGES:
 194		*idx = 5;
 195		break;
 196	default:
 197		return NULL;
 198	}
 199
 200	/*
 201	 * idx now points to the pool we want to allocate from. only the
 202	 * 1-vec entry pool is mempool backed.
 203	 */
 204	if (*idx == BVEC_POOL_MAX) {
 205fallback:
 206		bvl = mempool_alloc(pool, gfp_mask);
 207	} else {
 208		struct biovec_slab *bvs = bvec_slabs + *idx;
 209		gfp_t __gfp_mask = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_IO);
 210
 211		/*
 212		 * Make this allocation restricted and don't dump info on
 213		 * allocation failures, since we'll fallback to the mempool
 214		 * in case of failure.
 215		 */
 216		__gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
 217
 218		/*
 219		 * Try a slab allocation. If this fails and __GFP_DIRECT_RECLAIM
 220		 * is set, retry with the 1-entry mempool
 221		 */
 222		bvl = kmem_cache_alloc(bvs->slab, __gfp_mask);
 223		if (unlikely(!bvl && (gfp_mask & __GFP_DIRECT_RECLAIM))) {
 224			*idx = BVEC_POOL_MAX;
 225			goto fallback;
 226		}
 227	}
 228
 229	(*idx)++;
 230	return bvl;
 231}
 232
 233void bio_uninit(struct bio *bio)
 234{
 235	bio_disassociate_blkg(bio);
 
 
 
 
 
 
 
 
 
 236}
 237EXPORT_SYMBOL(bio_uninit);
 238
 239static void bio_free(struct bio *bio)
 240{
 241	struct bio_set *bs = bio->bi_pool;
 242	void *p;
 243
 244	bio_uninit(bio);
 245
 246	if (bs) {
 247		bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));
 248
 249		/*
 250		 * If we have front padding, adjust the bio pointer before freeing
 251		 */
 252		p = bio;
 253		p -= bs->front_pad;
 254
 255		mempool_free(p, &bs->bio_pool);
 256	} else {
 257		/* Bio was allocated by bio_kmalloc() */
 258		kfree(bio);
 259	}
 260}
 261
 262/*
 263 * Users of this function have their own bio allocation. Subsequently,
 264 * they must remember to pair any call to bio_init() with bio_uninit()
 265 * when IO has completed, or when the bio is released.
 266 */
 267void bio_init(struct bio *bio, struct bio_vec *table,
 268	      unsigned short max_vecs)
 269{
 270	memset(bio, 0, sizeof(*bio));
 271	atomic_set(&bio->__bi_remaining, 1);
 272	atomic_set(&bio->__bi_cnt, 1);
 273
 274	bio->bi_io_vec = table;
 275	bio->bi_max_vecs = max_vecs;
 276}
 277EXPORT_SYMBOL(bio_init);
 278
 279/**
 280 * bio_reset - reinitialize a bio
 281 * @bio:	bio to reset
 282 *
 283 * Description:
 284 *   After calling bio_reset(), @bio will be in the same state as a freshly
 285 *   allocated bio returned bio bio_alloc_bioset() - the only fields that are
 286 *   preserved are the ones that are initialized by bio_alloc_bioset(). See
 287 *   comment in struct bio.
 288 */
 289void bio_reset(struct bio *bio)
 290{
 291	unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);
 292
 293	bio_uninit(bio);
 294
 295	memset(bio, 0, BIO_RESET_BYTES);
 296	bio->bi_flags = flags;
 297	atomic_set(&bio->__bi_remaining, 1);
 298}
 299EXPORT_SYMBOL(bio_reset);
 300
 301static struct bio *__bio_chain_endio(struct bio *bio)
 302{
 303	struct bio *parent = bio->bi_private;
 304
 305	if (!parent->bi_status)
 306		parent->bi_status = bio->bi_status;
 307	bio_put(bio);
 308	return parent;
 309}
 310
 311static void bio_chain_endio(struct bio *bio)
 312{
 313	bio_endio(__bio_chain_endio(bio));
 314}
 315
 316/**
 317 * bio_chain - chain bio completions
 318 * @bio: the target bio
 319 * @parent: the @bio's parent bio
 320 *
 321 * The caller won't have a bi_end_io called when @bio completes - instead,
 322 * @parent's bi_end_io won't be called until both @parent and @bio have
 323 * completed; the chained bio will also be freed when it completes.
 324 *
 325 * The caller must not set bi_private or bi_end_io in @bio.
 326 */
 327void bio_chain(struct bio *bio, struct bio *parent)
 328{
 329	BUG_ON(bio->bi_private || bio->bi_end_io);
 330
 331	bio->bi_private = parent;
 332	bio->bi_end_io	= bio_chain_endio;
 333	bio_inc_remaining(parent);
 334}
 335EXPORT_SYMBOL(bio_chain);
 336
 337static void bio_alloc_rescue(struct work_struct *work)
 338{
 339	struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
 340	struct bio *bio;
 341
 342	while (1) {
 343		spin_lock(&bs->rescue_lock);
 344		bio = bio_list_pop(&bs->rescue_list);
 345		spin_unlock(&bs->rescue_lock);
 346
 347		if (!bio)
 348			break;
 349
 350		generic_make_request(bio);
 351	}
 352}
 353
 354static void punt_bios_to_rescuer(struct bio_set *bs)
 355{
 356	struct bio_list punt, nopunt;
 357	struct bio *bio;
 358
 359	if (WARN_ON_ONCE(!bs->rescue_workqueue))
 360		return;
 361	/*
 362	 * In order to guarantee forward progress we must punt only bios that
 363	 * were allocated from this bio_set; otherwise, if there was a bio on
 364	 * there for a stacking driver higher up in the stack, processing it
 365	 * could require allocating bios from this bio_set, and doing that from
 366	 * our own rescuer would be bad.
 367	 *
 368	 * Since bio lists are singly linked, pop them all instead of trying to
 369	 * remove from the middle of the list:
 370	 */
 371
 372	bio_list_init(&punt);
 373	bio_list_init(&nopunt);
 374
 375	while ((bio = bio_list_pop(&current->bio_list[0])))
 376		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
 377	current->bio_list[0] = nopunt;
 378
 379	bio_list_init(&nopunt);
 380	while ((bio = bio_list_pop(&current->bio_list[1])))
 381		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
 382	current->bio_list[1] = nopunt;
 383
 384	spin_lock(&bs->rescue_lock);
 385	bio_list_merge(&bs->rescue_list, &punt);
 386	spin_unlock(&bs->rescue_lock);
 387
 388	queue_work(bs->rescue_workqueue, &bs->rescue_work);
 389}
 390
 391/**
 392 * bio_alloc_bioset - allocate a bio for I/O
 393 * @gfp_mask:   the GFP_* mask given to the slab allocator
 394 * @nr_iovecs:	number of iovecs to pre-allocate
 395 * @bs:		the bio_set to allocate from.
 396 *
 397 * Description:
 398 *   If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is
 399 *   backed by the @bs's mempool.
 400 *
 401 *   When @bs is not NULL, if %__GFP_DIRECT_RECLAIM is set then bio_alloc will
 402 *   always be able to allocate a bio. This is due to the mempool guarantees.
 403 *   To make this work, callers must never allocate more than 1 bio at a time
 404 *   from this pool. Callers that need to allocate more than 1 bio must always
 405 *   submit the previously allocated bio for IO before attempting to allocate
 406 *   a new one. Failure to do so can cause deadlocks under memory pressure.
 407 *
 408 *   Note that when running under generic_make_request() (i.e. any block
 409 *   driver), bios are not submitted until after you return - see the code in
 410 *   generic_make_request() that converts recursion into iteration, to prevent
 411 *   stack overflows.
 412 *
 413 *   This would normally mean allocating multiple bios under
 414 *   generic_make_request() would be susceptible to deadlocks, but we have
 415 *   deadlock avoidance code that resubmits any blocked bios from a rescuer
 416 *   thread.
 417 *
 418 *   However, we do not guarantee forward progress for allocations from other
 419 *   mempools. Doing multiple allocations from the same mempool under
 420 *   generic_make_request() should be avoided - instead, use bio_set's front_pad
 421 *   for per bio allocations.
 422 *
 423 *   RETURNS:
 424 *   Pointer to new bio on success, NULL on failure.
 425 */
 426struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
 427			     struct bio_set *bs)
 428{
 429	gfp_t saved_gfp = gfp_mask;
 430	unsigned front_pad;
 431	unsigned inline_vecs;
 432	struct bio_vec *bvl = NULL;
 433	struct bio *bio;
 434	void *p;
 435
 436	if (!bs) {
 437		if (nr_iovecs > UIO_MAXIOV)
 438			return NULL;
 439
 440		p = kmalloc(sizeof(struct bio) +
 441			    nr_iovecs * sizeof(struct bio_vec),
 442			    gfp_mask);
 443		front_pad = 0;
 444		inline_vecs = nr_iovecs;
 445	} else {
 446		/* should not use nobvec bioset for nr_iovecs > 0 */
 447		if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) &&
 448				 nr_iovecs > 0))
 449			return NULL;
 450		/*
 451		 * generic_make_request() converts recursion to iteration; this
 452		 * means if we're running beneath it, any bios we allocate and
 453		 * submit will not be submitted (and thus freed) until after we
 454		 * return.
 455		 *
 456		 * This exposes us to a potential deadlock if we allocate
 457		 * multiple bios from the same bio_set() while running
 458		 * underneath generic_make_request(). If we were to allocate
 459		 * multiple bios (say a stacking block driver that was splitting
 460		 * bios), we would deadlock if we exhausted the mempool's
 461		 * reserve.
 462		 *
 463		 * We solve this, and guarantee forward progress, with a rescuer
 464		 * workqueue per bio_set. If we go to allocate and there are
 465		 * bios on current->bio_list, we first try the allocation
 466		 * without __GFP_DIRECT_RECLAIM; if that fails, we punt those
 467		 * bios we would be blocking to the rescuer workqueue before
 468		 * we retry with the original gfp_flags.
 469		 */
 470
 471		if (current->bio_list &&
 472		    (!bio_list_empty(&current->bio_list[0]) ||
 473		     !bio_list_empty(&current->bio_list[1])) &&
 474		    bs->rescue_workqueue)
 475			gfp_mask &= ~__GFP_DIRECT_RECLAIM;
 476
 477		p = mempool_alloc(&bs->bio_pool, gfp_mask);
 478		if (!p && gfp_mask != saved_gfp) {
 479			punt_bios_to_rescuer(bs);
 480			gfp_mask = saved_gfp;
 481			p = mempool_alloc(&bs->bio_pool, gfp_mask);
 482		}
 483
 484		front_pad = bs->front_pad;
 485		inline_vecs = BIO_INLINE_VECS;
 486	}
 487
 488	if (unlikely(!p))
 489		return NULL;
 490
 491	bio = p + front_pad;
 492	bio_init(bio, NULL, 0);
 493
 494	if (nr_iovecs > inline_vecs) {
 495		unsigned long idx = 0;
 496
 497		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
 498		if (!bvl && gfp_mask != saved_gfp) {
 499			punt_bios_to_rescuer(bs);
 500			gfp_mask = saved_gfp;
 501			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
 502		}
 503
 504		if (unlikely(!bvl))
 505			goto err_free;
 506
 507		bio->bi_flags |= idx << BVEC_POOL_OFFSET;
 508	} else if (nr_iovecs) {
 509		bvl = bio->bi_inline_vecs;
 510	}
 511
 512	bio->bi_pool = bs;
 513	bio->bi_max_vecs = nr_iovecs;
 514	bio->bi_io_vec = bvl;
 515	return bio;
 516
 517err_free:
 518	mempool_free(p, &bs->bio_pool);
 519	return NULL;
 520}
 521EXPORT_SYMBOL(bio_alloc_bioset);
 522
 523void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start)
 524{
 525	unsigned long flags;
 526	struct bio_vec bv;
 527	struct bvec_iter iter;
 528
 529	__bio_for_each_segment(bv, bio, iter, start) {
 530		char *data = bvec_kmap_irq(&bv, &flags);
 531		memset(data, 0, bv.bv_len);
 532		flush_dcache_page(bv.bv_page);
 533		bvec_kunmap_irq(data, &flags);
 534	}
 535}
 536EXPORT_SYMBOL(zero_fill_bio_iter);
 537
 538/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 539 * bio_put - release a reference to a bio
 540 * @bio:   bio to release reference to
 541 *
 542 * Description:
 543 *   Put a reference to a &struct bio, either one you have gotten with
 544 *   bio_alloc, bio_get or bio_clone_*. The last put of a bio will free it.
 545 **/
 546void bio_put(struct bio *bio)
 547{
 548	if (!bio_flagged(bio, BIO_REFFED))
 549		bio_free(bio);
 550	else {
 551		BIO_BUG_ON(!atomic_read(&bio->__bi_cnt));
 552
 553		/*
 554		 * last put frees it
 555		 */
 556		if (atomic_dec_and_test(&bio->__bi_cnt))
 557			bio_free(bio);
 558	}
 559}
 560EXPORT_SYMBOL(bio_put);
 561
 562/**
 563 * 	__bio_clone_fast - clone a bio that shares the original bio's biovec
 564 * 	@bio: destination bio
 565 * 	@bio_src: bio to clone
 566 *
 567 *	Clone a &bio. Caller will own the returned bio, but not
 568 *	the actual data it points to. Reference count of returned
 569 * 	bio will be one.
 570 *
 571 * 	Caller must ensure that @bio_src is not freed before @bio.
 572 */
 573void __bio_clone_fast(struct bio *bio, struct bio *bio_src)
 574{
 575	BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio));
 576
 577	/*
 578	 * most users will be overriding ->bi_disk with a new target,
 579	 * so we don't set nor calculate new physical/hw segment counts here
 580	 */
 581	bio->bi_disk = bio_src->bi_disk;
 582	bio->bi_partno = bio_src->bi_partno;
 583	bio_set_flag(bio, BIO_CLONED);
 584	if (bio_flagged(bio_src, BIO_THROTTLED))
 585		bio_set_flag(bio, BIO_THROTTLED);
 586	bio->bi_opf = bio_src->bi_opf;
 587	bio->bi_ioprio = bio_src->bi_ioprio;
 588	bio->bi_write_hint = bio_src->bi_write_hint;
 589	bio->bi_iter = bio_src->bi_iter;
 590	bio->bi_io_vec = bio_src->bi_io_vec;
 591
 592	bio_clone_blkg_association(bio, bio_src);
 593	blkcg_bio_issue_init(bio);
 594}
 595EXPORT_SYMBOL(__bio_clone_fast);
 596
 597/**
 598 *	bio_clone_fast - clone a bio that shares the original bio's biovec
 599 *	@bio: bio to clone
 600 *	@gfp_mask: allocation priority
 601 *	@bs: bio_set to allocate from
 602 *
 603 * 	Like __bio_clone_fast, only also allocates the returned bio
 604 */
 605struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)
 606{
 607	struct bio *b;
 608
 609	b = bio_alloc_bioset(gfp_mask, 0, bs);
 610	if (!b)
 611		return NULL;
 612
 613	__bio_clone_fast(b, bio);
 614
 
 
 615	if (bio_integrity(bio)) {
 616		int ret;
 617
 618		ret = bio_integrity_clone(b, bio, gfp_mask);
 619
 620		if (ret < 0) {
 621			bio_put(b);
 622			return NULL;
 623		}
 624	}
 625
 626	return b;
 627}
 628EXPORT_SYMBOL(bio_clone_fast);
 629
 
 
 
 
 
 
 630static inline bool page_is_mergeable(const struct bio_vec *bv,
 631		struct page *page, unsigned int len, unsigned int off,
 632		bool *same_page)
 633{
 634	phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) +
 635		bv->bv_offset + bv->bv_len - 1;
 636	phys_addr_t page_addr = page_to_phys(page);
 637
 638	if (vec_end_addr + 1 != page_addr + off)
 639		return false;
 640	if (xen_domain() && !xen_biovec_phys_mergeable(bv, page))
 641		return false;
 642
 643	*same_page = ((vec_end_addr & PAGE_MASK) == page_addr);
 644	if (!*same_page && pfn_to_page(PFN_DOWN(vec_end_addr)) + 1 != page)
 645		return false;
 646	return true;
 647}
 648
 649static bool bio_try_merge_pc_page(struct request_queue *q, struct bio *bio,
 650		struct page *page, unsigned len, unsigned offset,
 651		bool *same_page)
 
 
 
 
 
 652{
 653	struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
 654	unsigned long mask = queue_segment_boundary(q);
 655	phys_addr_t addr1 = page_to_phys(bv->bv_page) + bv->bv_offset;
 656	phys_addr_t addr2 = page_to_phys(page) + offset + len - 1;
 657
 658	if ((addr1 | mask) != (addr2 | mask))
 659		return false;
 660	if (bv->bv_len + len > queue_max_segment_size(q))
 661		return false;
 662	return __bio_try_merge_page(bio, page, len, offset, same_page);
 663}
 664
 665/**
 666 *	__bio_add_pc_page	- attempt to add page to passthrough bio
 667 *	@q: the target queue
 668 *	@bio: destination bio
 669 *	@page: page to add
 670 *	@len: vec entry length
 671 *	@offset: vec entry offset
 672 *	@same_page: return if the merge happen inside the same page
 673 *
 674 *	Attempt to add a page to the bio_vec maplist. This can fail for a
 675 *	number of reasons, such as the bio being full or target block device
 676 *	limitations. The target block device must allow bio's up to PAGE_SIZE,
 677 *	so it is always possible to add a single page to an empty bio.
 678 *
 679 *	This should only be used by passthrough bios.
 
 680 */
 681static int __bio_add_pc_page(struct request_queue *q, struct bio *bio,
 682		struct page *page, unsigned int len, unsigned int offset,
 683		bool *same_page)
 684{
 685	struct bio_vec *bvec;
 686
 687	/*
 688	 * cloned bio must not modify vec list
 689	 */
 690	if (unlikely(bio_flagged(bio, BIO_CLONED)))
 691		return 0;
 692
 693	if (((bio->bi_iter.bi_size + len) >> 9) > queue_max_hw_sectors(q))
 694		return 0;
 695
 696	if (bio->bi_vcnt > 0) {
 697		if (bio_try_merge_pc_page(q, bio, page, len, offset, same_page))
 698			return len;
 699
 700		/*
 701		 * If the queue doesn't support SG gaps and adding this segment
 702		 * would create a gap, disallow it.
 703		 */
 704		bvec = &bio->bi_io_vec[bio->bi_vcnt - 1];
 705		if (bvec_gap_to_prev(q, bvec, offset))
 706			return 0;
 707	}
 708
 709	if (bio_full(bio, len))
 710		return 0;
 711
 712	if (bio->bi_vcnt >= queue_max_segments(q))
 713		return 0;
 714
 715	bvec = &bio->bi_io_vec[bio->bi_vcnt];
 716	bvec->bv_page = page;
 717	bvec->bv_len = len;
 718	bvec->bv_offset = offset;
 719	bio->bi_vcnt++;
 720	bio->bi_iter.bi_size += len;
 721	return len;
 722}
 723
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 724int bio_add_pc_page(struct request_queue *q, struct bio *bio,
 725		struct page *page, unsigned int len, unsigned int offset)
 726{
 727	bool same_page = false;
 728	return __bio_add_pc_page(q, bio, page, len, offset, &same_page);
 
 729}
 730EXPORT_SYMBOL(bio_add_pc_page);
 731
 732/**
 733 * __bio_try_merge_page - try appending data to an existing bvec.
 734 * @bio: destination bio
 735 * @page: start page to add
 736 * @len: length of the data to add
 737 * @off: offset of the data relative to @page
 738 * @same_page: return if the segment has been merged inside the same page
 739 *
 740 * Try to add the data at @page + @off to the last bvec of @bio.  This is a
 741 * a useful optimisation for file systems with a block size smaller than the
 742 * page size.
 743 *
 744 * Warn if (@len, @off) crosses pages in case that @same_page is true.
 745 *
 746 * Return %true on success or %false on failure.
 747 */
 748bool __bio_try_merge_page(struct bio *bio, struct page *page,
 749		unsigned int len, unsigned int off, bool *same_page)
 750{
 751	if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
 752		return false;
 753
 754	if (bio->bi_vcnt > 0 && !bio_full(bio, len)) {
 755		struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
 756
 757		if (page_is_mergeable(bv, page, len, off, same_page)) {
 
 
 
 
 758			bv->bv_len += len;
 759			bio->bi_iter.bi_size += len;
 760			return true;
 761		}
 762	}
 763	return false;
 764}
 765EXPORT_SYMBOL_GPL(__bio_try_merge_page);
 766
 767/**
 768 * __bio_add_page - add page(s) to a bio in a new segment
 769 * @bio: destination bio
 770 * @page: start page to add
 771 * @len: length of the data to add, may cross pages
 772 * @off: offset of the data relative to @page, may cross pages
 773 *
 774 * Add the data at @page + @off to @bio as a new bvec.  The caller must ensure
 775 * that @bio has space for another bvec.
 776 */
 777void __bio_add_page(struct bio *bio, struct page *page,
 778		unsigned int len, unsigned int off)
 779{
 780	struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
 781
 782	WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
 783	WARN_ON_ONCE(bio_full(bio, len));
 784
 785	bv->bv_page = page;
 786	bv->bv_offset = off;
 787	bv->bv_len = len;
 788
 789	bio->bi_iter.bi_size += len;
 790	bio->bi_vcnt++;
 791
 792	if (!bio_flagged(bio, BIO_WORKINGSET) && unlikely(PageWorkingset(page)))
 793		bio_set_flag(bio, BIO_WORKINGSET);
 794}
 795EXPORT_SYMBOL_GPL(__bio_add_page);
 796
 797/**
 798 *	bio_add_page	-	attempt to add page(s) to bio
 799 *	@bio: destination bio
 800 *	@page: start page to add
 801 *	@len: vec entry length, may cross pages
 802 *	@offset: vec entry offset relative to @page, may cross pages
 803 *
 804 *	Attempt to add page(s) to the bio_vec maplist. This will only fail
 805 *	if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio.
 806 */
 807int bio_add_page(struct bio *bio, struct page *page,
 808		 unsigned int len, unsigned int offset)
 809{
 810	bool same_page = false;
 811
 812	if (!__bio_try_merge_page(bio, page, len, offset, &same_page)) {
 813		if (bio_full(bio, len))
 814			return 0;
 815		__bio_add_page(bio, page, len, offset);
 816	}
 817	return len;
 818}
 819EXPORT_SYMBOL(bio_add_page);
 820
 821void bio_release_pages(struct bio *bio, bool mark_dirty)
 822{
 823	struct bvec_iter_all iter_all;
 824	struct bio_vec *bvec;
 825
 826	if (bio_flagged(bio, BIO_NO_PAGE_REF))
 827		return;
 828
 829	bio_for_each_segment_all(bvec, bio, iter_all) {
 830		if (mark_dirty && !PageCompound(bvec->bv_page))
 831			set_page_dirty_lock(bvec->bv_page);
 832		put_page(bvec->bv_page);
 833	}
 834}
 
 835
 836static int __bio_iov_bvec_add_pages(struct bio *bio, struct iov_iter *iter)
 837{
 838	const struct bio_vec *bv = iter->bvec;
 839	unsigned int len;
 840	size_t size;
 841
 842	if (WARN_ON_ONCE(iter->iov_offset > bv->bv_len))
 843		return -EINVAL;
 844
 845	len = min_t(size_t, bv->bv_len - iter->iov_offset, iter->count);
 846	size = bio_add_page(bio, bv->bv_page, len,
 847				bv->bv_offset + iter->iov_offset);
 848	if (unlikely(size != len))
 849		return -EINVAL;
 850	iov_iter_advance(iter, size);
 851	return 0;
 852}
 853
 854#define PAGE_PTRS_PER_BVEC     (sizeof(struct bio_vec) / sizeof(struct page *))
 855
 856/**
 857 * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio
 858 * @bio: bio to add pages to
 859 * @iter: iov iterator describing the region to be mapped
 860 *
 861 * Pins pages from *iter and appends them to @bio's bvec array. The
 862 * pages will have to be released using put_page() when done.
 863 * For multi-segment *iter, this function only adds pages from the
 864 * the next non-empty segment of the iov iterator.
 865 */
 866static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
 867{
 868	unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
 869	unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;
 870	struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;
 871	struct page **pages = (struct page **)bv;
 872	bool same_page = false;
 873	ssize_t size, left;
 874	unsigned len, i;
 875	size_t offset;
 876
 877	/*
 878	 * Move page array up in the allocated memory for the bio vecs as far as
 879	 * possible so that we can start filling biovecs from the beginning
 880	 * without overwriting the temporary page array.
 881	*/
 882	BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2);
 883	pages += entries_left * (PAGE_PTRS_PER_BVEC - 1);
 884
 885	size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset);
 886	if (unlikely(size <= 0))
 887		return size ? size : -EFAULT;
 888
 889	for (left = size, i = 0; left > 0; left -= len, i++) {
 890		struct page *page = pages[i];
 891
 892		len = min_t(size_t, PAGE_SIZE - offset, left);
 893
 894		if (__bio_try_merge_page(bio, page, len, offset, &same_page)) {
 895			if (same_page)
 896				put_page(page);
 897		} else {
 898			if (WARN_ON_ONCE(bio_full(bio, len)))
 899                                return -EINVAL;
 900			__bio_add_page(bio, page, len, offset);
 901		}
 902		offset = 0;
 903	}
 904
 905	iov_iter_advance(iter, size);
 906	return 0;
 907}
 908
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 909/**
 910 * bio_iov_iter_get_pages - add user or kernel pages to a bio
 911 * @bio: bio to add pages to
 912 * @iter: iov iterator describing the region to be added
 913 *
 914 * This takes either an iterator pointing to user memory, or one pointing to
 915 * kernel pages (BVEC iterator). If we're adding user pages, we pin them and
 916 * map them into the kernel. On IO completion, the caller should put those
 917 * pages. If we're adding kernel pages, and the caller told us it's safe to
 918 * do so, we just have to add the pages to the bio directly. We don't grab an
 919 * extra reference to those pages (the user should already have that), and we
 920 * don't put the page on IO completion. The caller needs to check if the bio is
 921 * flagged BIO_NO_PAGE_REF on IO completion. If it isn't, then pages should be
 922 * released.
 923 *
 924 * The function tries, but does not guarantee, to pin as many pages as
 925 * fit into the bio, or are requested in *iter, whatever is smaller. If
 926 * MM encounters an error pinning the requested pages, it stops. Error
 927 * is returned only if 0 pages could be pinned.
 928 */
 929int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
 930{
 931	const bool is_bvec = iov_iter_is_bvec(iter);
 932	int ret;
 933
 934	if (WARN_ON_ONCE(bio->bi_vcnt))
 935		return -EINVAL;
 936
 937	do {
 938		if (is_bvec)
 939			ret = __bio_iov_bvec_add_pages(bio, iter);
 940		else
 941			ret = __bio_iov_iter_get_pages(bio, iter);
 
 
 
 
 
 
 942	} while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
 943
 944	if (is_bvec)
 945		bio_set_flag(bio, BIO_NO_PAGE_REF);
 946	return bio->bi_vcnt ? 0 : ret;
 947}
 
 948
 949static void submit_bio_wait_endio(struct bio *bio)
 950{
 951	complete(bio->bi_private);
 952}
 953
 954/**
 955 * submit_bio_wait - submit a bio, and wait until it completes
 956 * @bio: The &struct bio which describes the I/O
 957 *
 958 * Simple wrapper around submit_bio(). Returns 0 on success, or the error from
 959 * bio_endio() on failure.
 960 *
 961 * WARNING: Unlike to how submit_bio() is usually used, this function does not
 962 * result in bio reference to be consumed. The caller must drop the reference
 963 * on his own.
 964 */
 965int submit_bio_wait(struct bio *bio)
 966{
 967	DECLARE_COMPLETION_ONSTACK_MAP(done, bio->bi_disk->lockdep_map);
 
 968
 969	bio->bi_private = &done;
 970	bio->bi_end_io = submit_bio_wait_endio;
 971	bio->bi_opf |= REQ_SYNC;
 972	submit_bio(bio);
 973	wait_for_completion_io(&done);
 
 
 
 
 
 
 
 
 974
 975	return blk_status_to_errno(bio->bi_status);
 976}
 977EXPORT_SYMBOL(submit_bio_wait);
 978
 979/**
 980 * bio_advance - increment/complete a bio by some number of bytes
 981 * @bio:	bio to advance
 982 * @bytes:	number of bytes to complete
 983 *
 984 * This updates bi_sector, bi_size and bi_idx; if the number of bytes to
 985 * complete doesn't align with a bvec boundary, then bv_len and bv_offset will
 986 * be updated on the last bvec as well.
 987 *
 988 * @bio will then represent the remaining, uncompleted portion of the io.
 989 */
 990void bio_advance(struct bio *bio, unsigned bytes)
 991{
 992	if (bio_integrity(bio))
 993		bio_integrity_advance(bio, bytes);
 994
 
 995	bio_advance_iter(bio, &bio->bi_iter, bytes);
 996}
 997EXPORT_SYMBOL(bio_advance);
 998
 999void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
1000			struct bio *src, struct bvec_iter *src_iter)
1001{
1002	struct bio_vec src_bv, dst_bv;
1003	void *src_p, *dst_p;
1004	unsigned bytes;
1005
1006	while (src_iter->bi_size && dst_iter->bi_size) {
1007		src_bv = bio_iter_iovec(src, *src_iter);
1008		dst_bv = bio_iter_iovec(dst, *dst_iter);
1009
1010		bytes = min(src_bv.bv_len, dst_bv.bv_len);
1011
1012		src_p = kmap_atomic(src_bv.bv_page);
1013		dst_p = kmap_atomic(dst_bv.bv_page);
1014
1015		memcpy(dst_p + dst_bv.bv_offset,
1016		       src_p + src_bv.bv_offset,
1017		       bytes);
1018
1019		kunmap_atomic(dst_p);
1020		kunmap_atomic(src_p);
1021
1022		flush_dcache_page(dst_bv.bv_page);
1023
1024		bio_advance_iter(src, src_iter, bytes);
1025		bio_advance_iter(dst, dst_iter, bytes);
1026	}
1027}
1028EXPORT_SYMBOL(bio_copy_data_iter);
1029
1030/**
1031 * bio_copy_data - copy contents of data buffers from one bio to another
1032 * @src: source bio
1033 * @dst: destination bio
1034 *
1035 * Stops when it reaches the end of either @src or @dst - that is, copies
1036 * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios).
1037 */
1038void bio_copy_data(struct bio *dst, struct bio *src)
1039{
1040	struct bvec_iter src_iter = src->bi_iter;
1041	struct bvec_iter dst_iter = dst->bi_iter;
1042
1043	bio_copy_data_iter(dst, &dst_iter, src, &src_iter);
1044}
1045EXPORT_SYMBOL(bio_copy_data);
1046
1047/**
1048 * bio_list_copy_data - copy contents of data buffers from one chain of bios to
1049 * another
1050 * @src: source bio list
1051 * @dst: destination bio list
1052 *
1053 * Stops when it reaches the end of either the @src list or @dst list - that is,
1054 * copies min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of
1055 * bios).
1056 */
1057void bio_list_copy_data(struct bio *dst, struct bio *src)
1058{
1059	struct bvec_iter src_iter = src->bi_iter;
1060	struct bvec_iter dst_iter = dst->bi_iter;
1061
1062	while (1) {
1063		if (!src_iter.bi_size) {
1064			src = src->bi_next;
1065			if (!src)
1066				break;
1067
1068			src_iter = src->bi_iter;
1069		}
1070
1071		if (!dst_iter.bi_size) {
1072			dst = dst->bi_next;
1073			if (!dst)
1074				break;
1075
1076			dst_iter = dst->bi_iter;
1077		}
1078
1079		bio_copy_data_iter(dst, &dst_iter, src, &src_iter);
1080	}
1081}
1082EXPORT_SYMBOL(bio_list_copy_data);
1083
1084struct bio_map_data {
1085	int is_our_pages;
1086	struct iov_iter iter;
1087	struct iovec iov[];
1088};
1089
1090static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data,
1091					       gfp_t gfp_mask)
1092{
1093	struct bio_map_data *bmd;
1094	if (data->nr_segs > UIO_MAXIOV)
1095		return NULL;
1096
1097	bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask);
1098	if (!bmd)
1099		return NULL;
1100	memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs);
1101	bmd->iter = *data;
1102	bmd->iter.iov = bmd->iov;
1103	return bmd;
1104}
1105
1106/**
1107 * bio_copy_from_iter - copy all pages from iov_iter to bio
1108 * @bio: The &struct bio which describes the I/O as destination
1109 * @iter: iov_iter as source
1110 *
1111 * Copy all pages from iov_iter to bio.
1112 * Returns 0 on success, or error on failure.
1113 */
1114static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)
1115{
1116	struct bio_vec *bvec;
1117	struct bvec_iter_all iter_all;
1118
1119	bio_for_each_segment_all(bvec, bio, iter_all) {
1120		ssize_t ret;
1121
1122		ret = copy_page_from_iter(bvec->bv_page,
1123					  bvec->bv_offset,
1124					  bvec->bv_len,
1125					  iter);
1126
1127		if (!iov_iter_count(iter))
1128			break;
1129
1130		if (ret < bvec->bv_len)
1131			return -EFAULT;
1132	}
1133
1134	return 0;
1135}
1136
1137/**
1138 * bio_copy_to_iter - copy all pages from bio to iov_iter
1139 * @bio: The &struct bio which describes the I/O as source
1140 * @iter: iov_iter as destination
1141 *
1142 * Copy all pages from bio to iov_iter.
1143 * Returns 0 on success, or error on failure.
1144 */
1145static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter)
1146{
1147	struct bio_vec *bvec;
1148	struct bvec_iter_all iter_all;
1149
1150	bio_for_each_segment_all(bvec, bio, iter_all) {
1151		ssize_t ret;
1152
1153		ret = copy_page_to_iter(bvec->bv_page,
1154					bvec->bv_offset,
1155					bvec->bv_len,
1156					&iter);
1157
1158		if (!iov_iter_count(&iter))
1159			break;
1160
1161		if (ret < bvec->bv_len)
1162			return -EFAULT;
1163	}
1164
1165	return 0;
1166}
1167
1168void bio_free_pages(struct bio *bio)
1169{
1170	struct bio_vec *bvec;
1171	struct bvec_iter_all iter_all;
1172
1173	bio_for_each_segment_all(bvec, bio, iter_all)
1174		__free_page(bvec->bv_page);
1175}
1176EXPORT_SYMBOL(bio_free_pages);
1177
1178/**
1179 *	bio_uncopy_user	-	finish previously mapped bio
1180 *	@bio: bio being terminated
1181 *
1182 *	Free pages allocated from bio_copy_user_iov() and write back data
1183 *	to user space in case of a read.
1184 */
1185int bio_uncopy_user(struct bio *bio)
1186{
1187	struct bio_map_data *bmd = bio->bi_private;
1188	int ret = 0;
1189
1190	if (!bio_flagged(bio, BIO_NULL_MAPPED)) {
1191		/*
1192		 * if we're in a workqueue, the request is orphaned, so
1193		 * don't copy into a random user address space, just free
1194		 * and return -EINTR so user space doesn't expect any data.
1195		 */
1196		if (!current->mm)
1197			ret = -EINTR;
1198		else if (bio_data_dir(bio) == READ)
1199			ret = bio_copy_to_iter(bio, bmd->iter);
1200		if (bmd->is_our_pages)
1201			bio_free_pages(bio);
1202	}
1203	kfree(bmd);
1204	bio_put(bio);
1205	return ret;
1206}
1207
1208/**
1209 *	bio_copy_user_iov	-	copy user data to bio
1210 *	@q:		destination block queue
1211 *	@map_data:	pointer to the rq_map_data holding pages (if necessary)
1212 *	@iter:		iovec iterator
1213 *	@gfp_mask:	memory allocation flags
1214 *
1215 *	Prepares and returns a bio for indirect user io, bouncing data
1216 *	to/from kernel pages as necessary. Must be paired with
1217 *	call bio_uncopy_user() on io completion.
1218 */
1219struct bio *bio_copy_user_iov(struct request_queue *q,
1220			      struct rq_map_data *map_data,
1221			      struct iov_iter *iter,
1222			      gfp_t gfp_mask)
1223{
1224	struct bio_map_data *bmd;
1225	struct page *page;
1226	struct bio *bio;
1227	int i = 0, ret;
1228	int nr_pages;
1229	unsigned int len = iter->count;
1230	unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0;
1231
1232	bmd = bio_alloc_map_data(iter, gfp_mask);
1233	if (!bmd)
1234		return ERR_PTR(-ENOMEM);
1235
1236	/*
1237	 * We need to do a deep copy of the iov_iter including the iovecs.
1238	 * The caller provided iov might point to an on-stack or otherwise
1239	 * shortlived one.
1240	 */
1241	bmd->is_our_pages = map_data ? 0 : 1;
1242
1243	nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE);
1244	if (nr_pages > BIO_MAX_PAGES)
1245		nr_pages = BIO_MAX_PAGES;
1246
1247	ret = -ENOMEM;
1248	bio = bio_kmalloc(gfp_mask, nr_pages);
1249	if (!bio)
1250		goto out_bmd;
1251
1252	ret = 0;
1253
1254	if (map_data) {
1255		nr_pages = 1 << map_data->page_order;
1256		i = map_data->offset / PAGE_SIZE;
1257	}
1258	while (len) {
1259		unsigned int bytes = PAGE_SIZE;
1260
1261		bytes -= offset;
1262
1263		if (bytes > len)
1264			bytes = len;
1265
1266		if (map_data) {
1267			if (i == map_data->nr_entries * nr_pages) {
1268				ret = -ENOMEM;
1269				break;
1270			}
1271
1272			page = map_data->pages[i / nr_pages];
1273			page += (i % nr_pages);
1274
1275			i++;
1276		} else {
1277			page = alloc_page(q->bounce_gfp | gfp_mask);
1278			if (!page) {
1279				ret = -ENOMEM;
1280				break;
1281			}
1282		}
1283
1284		if (bio_add_pc_page(q, bio, page, bytes, offset) < bytes) {
1285			if (!map_data)
1286				__free_page(page);
1287			break;
1288		}
1289
1290		len -= bytes;
1291		offset = 0;
1292	}
1293
1294	if (ret)
1295		goto cleanup;
1296
1297	if (map_data)
1298		map_data->offset += bio->bi_iter.bi_size;
1299
1300	/*
1301	 * success
1302	 */
1303	if ((iov_iter_rw(iter) == WRITE && (!map_data || !map_data->null_mapped)) ||
1304	    (map_data && map_data->from_user)) {
1305		ret = bio_copy_from_iter(bio, iter);
1306		if (ret)
1307			goto cleanup;
1308	} else {
1309		if (bmd->is_our_pages)
1310			zero_fill_bio(bio);
1311		iov_iter_advance(iter, bio->bi_iter.bi_size);
1312	}
1313
1314	bio->bi_private = bmd;
1315	if (map_data && map_data->null_mapped)
1316		bio_set_flag(bio, BIO_NULL_MAPPED);
1317	return bio;
1318cleanup:
1319	if (!map_data)
1320		bio_free_pages(bio);
1321	bio_put(bio);
1322out_bmd:
1323	kfree(bmd);
1324	return ERR_PTR(ret);
1325}
1326
1327/**
1328 *	bio_map_user_iov - map user iovec into bio
1329 *	@q:		the struct request_queue for the bio
1330 *	@iter:		iovec iterator
1331 *	@gfp_mask:	memory allocation flags
1332 *
1333 *	Map the user space address into a bio suitable for io to a block
1334 *	device. Returns an error pointer in case of error.
1335 */
1336struct bio *bio_map_user_iov(struct request_queue *q,
1337			     struct iov_iter *iter,
1338			     gfp_t gfp_mask)
1339{
1340	int j;
1341	struct bio *bio;
1342	int ret;
1343
1344	if (!iov_iter_count(iter))
1345		return ERR_PTR(-EINVAL);
1346
1347	bio = bio_kmalloc(gfp_mask, iov_iter_npages(iter, BIO_MAX_PAGES));
1348	if (!bio)
1349		return ERR_PTR(-ENOMEM);
1350
1351	while (iov_iter_count(iter)) {
1352		struct page **pages;
1353		ssize_t bytes;
1354		size_t offs, added = 0;
1355		int npages;
1356
1357		bytes = iov_iter_get_pages_alloc(iter, &pages, LONG_MAX, &offs);
1358		if (unlikely(bytes <= 0)) {
1359			ret = bytes ? bytes : -EFAULT;
1360			goto out_unmap;
1361		}
1362
1363		npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE);
1364
1365		if (unlikely(offs & queue_dma_alignment(q))) {
1366			ret = -EINVAL;
1367			j = 0;
1368		} else {
1369			for (j = 0; j < npages; j++) {
1370				struct page *page = pages[j];
1371				unsigned int n = PAGE_SIZE - offs;
1372				bool same_page = false;
1373
1374				if (n > bytes)
1375					n = bytes;
1376
1377				if (!__bio_add_pc_page(q, bio, page, n, offs,
1378						&same_page)) {
1379					if (same_page)
1380						put_page(page);
1381					break;
1382				}
1383
1384				added += n;
1385				bytes -= n;
1386				offs = 0;
1387			}
1388			iov_iter_advance(iter, added);
1389		}
1390		/*
1391		 * release the pages we didn't map into the bio, if any
1392		 */
1393		while (j < npages)
1394			put_page(pages[j++]);
1395		kvfree(pages);
1396		/* couldn't stuff something into bio? */
1397		if (bytes)
1398			break;
1399	}
1400
1401	bio_set_flag(bio, BIO_USER_MAPPED);
1402
1403	/*
1404	 * subtle -- if bio_map_user_iov() ended up bouncing a bio,
1405	 * it would normally disappear when its bi_end_io is run.
1406	 * however, we need it for the unmap, so grab an extra
1407	 * reference to it
1408	 */
1409	bio_get(bio);
1410	return bio;
1411
1412 out_unmap:
1413	bio_release_pages(bio, false);
1414	bio_put(bio);
1415	return ERR_PTR(ret);
1416}
1417
1418/**
1419 *	bio_unmap_user	-	unmap a bio
1420 *	@bio:		the bio being unmapped
1421 *
1422 *	Unmap a bio previously mapped by bio_map_user_iov(). Must be called from
1423 *	process context.
1424 *
1425 *	bio_unmap_user() may sleep.
1426 */
1427void bio_unmap_user(struct bio *bio)
1428{
1429	bio_release_pages(bio, bio_data_dir(bio) == READ);
1430	bio_put(bio);
1431	bio_put(bio);
1432}
1433
1434static void bio_invalidate_vmalloc_pages(struct bio *bio)
1435{
1436#ifdef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
1437	if (bio->bi_private && !op_is_write(bio_op(bio))) {
1438		unsigned long i, len = 0;
1439
1440		for (i = 0; i < bio->bi_vcnt; i++)
1441			len += bio->bi_io_vec[i].bv_len;
1442		invalidate_kernel_vmap_range(bio->bi_private, len);
1443	}
1444#endif
1445}
1446
1447static void bio_map_kern_endio(struct bio *bio)
1448{
1449	bio_invalidate_vmalloc_pages(bio);
1450	bio_put(bio);
1451}
1452
1453/**
1454 *	bio_map_kern	-	map kernel address into bio
1455 *	@q: the struct request_queue for the bio
1456 *	@data: pointer to buffer to map
1457 *	@len: length in bytes
1458 *	@gfp_mask: allocation flags for bio allocation
1459 *
1460 *	Map the kernel address into a bio suitable for io to a block
1461 *	device. Returns an error pointer in case of error.
1462 */
1463struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
1464			 gfp_t gfp_mask)
1465{
1466	unsigned long kaddr = (unsigned long)data;
1467	unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1468	unsigned long start = kaddr >> PAGE_SHIFT;
1469	const int nr_pages = end - start;
1470	bool is_vmalloc = is_vmalloc_addr(data);
1471	struct page *page;
1472	int offset, i;
1473	struct bio *bio;
1474
1475	bio = bio_kmalloc(gfp_mask, nr_pages);
1476	if (!bio)
1477		return ERR_PTR(-ENOMEM);
1478
1479	if (is_vmalloc) {
1480		flush_kernel_vmap_range(data, len);
1481		bio->bi_private = data;
1482	}
1483
1484	offset = offset_in_page(kaddr);
1485	for (i = 0; i < nr_pages; i++) {
1486		unsigned int bytes = PAGE_SIZE - offset;
1487
1488		if (len <= 0)
1489			break;
1490
1491		if (bytes > len)
1492			bytes = len;
1493
1494		if (!is_vmalloc)
1495			page = virt_to_page(data);
1496		else
1497			page = vmalloc_to_page(data);
1498		if (bio_add_pc_page(q, bio, page, bytes,
1499				    offset) < bytes) {
1500			/* we don't support partial mappings */
1501			bio_put(bio);
1502			return ERR_PTR(-EINVAL);
1503		}
1504
1505		data += bytes;
1506		len -= bytes;
1507		offset = 0;
1508	}
1509
1510	bio->bi_end_io = bio_map_kern_endio;
1511	return bio;
1512}
1513
1514static void bio_copy_kern_endio(struct bio *bio)
1515{
1516	bio_free_pages(bio);
1517	bio_put(bio);
1518}
1519
1520static void bio_copy_kern_endio_read(struct bio *bio)
1521{
1522	char *p = bio->bi_private;
1523	struct bio_vec *bvec;
1524	struct bvec_iter_all iter_all;
1525
1526	bio_for_each_segment_all(bvec, bio, iter_all) {
1527		memcpy(p, page_address(bvec->bv_page), bvec->bv_len);
1528		p += bvec->bv_len;
1529	}
1530
1531	bio_copy_kern_endio(bio);
1532}
1533
1534/**
1535 *	bio_copy_kern	-	copy kernel address into bio
1536 *	@q: the struct request_queue for the bio
1537 *	@data: pointer to buffer to copy
1538 *	@len: length in bytes
1539 *	@gfp_mask: allocation flags for bio and page allocation
1540 *	@reading: data direction is READ
1541 *
1542 *	copy the kernel address into a bio suitable for io to a block
1543 *	device. Returns an error pointer in case of error.
1544 */
1545struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len,
1546			  gfp_t gfp_mask, int reading)
1547{
1548	unsigned long kaddr = (unsigned long)data;
1549	unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1550	unsigned long start = kaddr >> PAGE_SHIFT;
1551	struct bio *bio;
1552	void *p = data;
1553	int nr_pages = 0;
1554
1555	/*
1556	 * Overflow, abort
1557	 */
1558	if (end < start)
1559		return ERR_PTR(-EINVAL);
1560
1561	nr_pages = end - start;
1562	bio = bio_kmalloc(gfp_mask, nr_pages);
1563	if (!bio)
1564		return ERR_PTR(-ENOMEM);
1565
1566	while (len) {
1567		struct page *page;
1568		unsigned int bytes = PAGE_SIZE;
1569
1570		if (bytes > len)
1571			bytes = len;
1572
1573		page = alloc_page(q->bounce_gfp | gfp_mask);
1574		if (!page)
1575			goto cleanup;
1576
1577		if (!reading)
1578			memcpy(page_address(page), p, bytes);
1579
1580		if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes)
1581			break;
1582
1583		len -= bytes;
1584		p += bytes;
1585	}
1586
1587	if (reading) {
1588		bio->bi_end_io = bio_copy_kern_endio_read;
1589		bio->bi_private = data;
1590	} else {
1591		bio->bi_end_io = bio_copy_kern_endio;
1592	}
1593
1594	return bio;
1595
1596cleanup:
1597	bio_free_pages(bio);
1598	bio_put(bio);
1599	return ERR_PTR(-ENOMEM);
1600}
1601
1602/*
1603 * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions
1604 * for performing direct-IO in BIOs.
1605 *
1606 * The problem is that we cannot run set_page_dirty() from interrupt context
1607 * because the required locks are not interrupt-safe.  So what we can do is to
1608 * mark the pages dirty _before_ performing IO.  And in interrupt context,
1609 * check that the pages are still dirty.   If so, fine.  If not, redirty them
1610 * in process context.
1611 *
1612 * We special-case compound pages here: normally this means reads into hugetlb
1613 * pages.  The logic in here doesn't really work right for compound pages
1614 * because the VM does not uniformly chase down the head page in all cases.
1615 * But dirtiness of compound pages is pretty meaningless anyway: the VM doesn't
1616 * handle them at all.  So we skip compound pages here at an early stage.
1617 *
1618 * Note that this code is very hard to test under normal circumstances because
1619 * direct-io pins the pages with get_user_pages().  This makes
1620 * is_page_cache_freeable return false, and the VM will not clean the pages.
1621 * But other code (eg, flusher threads) could clean the pages if they are mapped
1622 * pagecache.
1623 *
1624 * Simply disabling the call to bio_set_pages_dirty() is a good way to test the
1625 * deferred bio dirtying paths.
1626 */
1627
1628/*
1629 * bio_set_pages_dirty() will mark all the bio's pages as dirty.
1630 */
1631void bio_set_pages_dirty(struct bio *bio)
1632{
1633	struct bio_vec *bvec;
1634	struct bvec_iter_all iter_all;
1635
1636	bio_for_each_segment_all(bvec, bio, iter_all) {
1637		if (!PageCompound(bvec->bv_page))
1638			set_page_dirty_lock(bvec->bv_page);
1639	}
1640}
1641
1642/*
1643 * bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
1644 * If they are, then fine.  If, however, some pages are clean then they must
1645 * have been written out during the direct-IO read.  So we take another ref on
1646 * the BIO and re-dirty the pages in process context.
1647 *
1648 * It is expected that bio_check_pages_dirty() will wholly own the BIO from
1649 * here on.  It will run one put_page() against each page and will run one
1650 * bio_put() against the BIO.
1651 */
1652
1653static void bio_dirty_fn(struct work_struct *work);
1654
1655static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
1656static DEFINE_SPINLOCK(bio_dirty_lock);
1657static struct bio *bio_dirty_list;
1658
1659/*
1660 * This runs in process context
1661 */
1662static void bio_dirty_fn(struct work_struct *work)
1663{
1664	struct bio *bio, *next;
1665
1666	spin_lock_irq(&bio_dirty_lock);
1667	next = bio_dirty_list;
1668	bio_dirty_list = NULL;
1669	spin_unlock_irq(&bio_dirty_lock);
1670
1671	while ((bio = next) != NULL) {
1672		next = bio->bi_private;
1673
1674		bio_release_pages(bio, true);
1675		bio_put(bio);
1676	}
1677}
1678
1679void bio_check_pages_dirty(struct bio *bio)
1680{
1681	struct bio_vec *bvec;
1682	unsigned long flags;
1683	struct bvec_iter_all iter_all;
1684
1685	bio_for_each_segment_all(bvec, bio, iter_all) {
1686		if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page))
1687			goto defer;
1688	}
1689
1690	bio_release_pages(bio, false);
1691	bio_put(bio);
1692	return;
1693defer:
1694	spin_lock_irqsave(&bio_dirty_lock, flags);
1695	bio->bi_private = bio_dirty_list;
1696	bio_dirty_list = bio;
1697	spin_unlock_irqrestore(&bio_dirty_lock, flags);
1698	schedule_work(&bio_dirty_work);
1699}
1700
1701void update_io_ticks(struct hd_struct *part, unsigned long now)
1702{
1703	unsigned long stamp;
1704again:
1705	stamp = READ_ONCE(part->stamp);
1706	if (unlikely(stamp != now)) {
1707		if (likely(cmpxchg(&part->stamp, stamp, now) == stamp)) {
1708			__part_stat_add(part, io_ticks, 1);
1709		}
1710	}
1711	if (part->partno) {
1712		part = &part_to_disk(part)->part0;
1713		goto again;
1714	}
1715}
1716
1717void generic_start_io_acct(struct request_queue *q, int op,
1718			   unsigned long sectors, struct hd_struct *part)
1719{
1720	const int sgrp = op_stat_group(op);
1721
1722	part_stat_lock();
1723
1724	update_io_ticks(part, jiffies);
1725	part_stat_inc(part, ios[sgrp]);
1726	part_stat_add(part, sectors[sgrp], sectors);
1727	part_inc_in_flight(q, part, op_is_write(op));
1728
1729	part_stat_unlock();
1730}
1731EXPORT_SYMBOL(generic_start_io_acct);
1732
1733void generic_end_io_acct(struct request_queue *q, int req_op,
1734			 struct hd_struct *part, unsigned long start_time)
1735{
1736	unsigned long now = jiffies;
1737	unsigned long duration = now - start_time;
1738	const int sgrp = op_stat_group(req_op);
1739
1740	part_stat_lock();
1741
1742	update_io_ticks(part, now);
1743	part_stat_add(part, nsecs[sgrp], jiffies_to_nsecs(duration));
1744	part_stat_add(part, time_in_queue, duration);
1745	part_dec_in_flight(q, part, op_is_write(req_op));
1746
1747	part_stat_unlock();
1748}
1749EXPORT_SYMBOL(generic_end_io_acct);
1750
1751static inline bool bio_remaining_done(struct bio *bio)
1752{
1753	/*
1754	 * If we're not chaining, then ->__bi_remaining is always 1 and
1755	 * we always end io on the first invocation.
1756	 */
1757	if (!bio_flagged(bio, BIO_CHAIN))
1758		return true;
1759
1760	BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
1761
1762	if (atomic_dec_and_test(&bio->__bi_remaining)) {
1763		bio_clear_flag(bio, BIO_CHAIN);
1764		return true;
1765	}
1766
1767	return false;
1768}
1769
1770/**
1771 * bio_endio - end I/O on a bio
1772 * @bio:	bio
1773 *
1774 * Description:
1775 *   bio_endio() will end I/O on the whole bio. bio_endio() is the preferred
1776 *   way to end I/O on a bio. No one should call bi_end_io() directly on a
1777 *   bio unless they own it and thus know that it has an end_io function.
1778 *
1779 *   bio_endio() can be called several times on a bio that has been chained
1780 *   using bio_chain().  The ->bi_end_io() function will only be called the
1781 *   last time.  At this point the BLK_TA_COMPLETE tracing event will be
1782 *   generated if BIO_TRACE_COMPLETION is set.
1783 **/
1784void bio_endio(struct bio *bio)
1785{
1786again:
1787	if (!bio_remaining_done(bio))
1788		return;
1789	if (!bio_integrity_endio(bio))
1790		return;
1791
1792	if (bio->bi_disk)
1793		rq_qos_done_bio(bio->bi_disk->queue, bio);
1794
1795	/*
1796	 * Need to have a real endio function for chained bios, otherwise
1797	 * various corner cases will break (like stacking block devices that
1798	 * save/restore bi_end_io) - however, we want to avoid unbounded
1799	 * recursion and blowing the stack. Tail call optimization would
1800	 * handle this, but compiling with frame pointers also disables
1801	 * gcc's sibling call optimization.
1802	 */
1803	if (bio->bi_end_io == bio_chain_endio) {
1804		bio = __bio_chain_endio(bio);
1805		goto again;
1806	}
1807
1808	if (bio->bi_disk && bio_flagged(bio, BIO_TRACE_COMPLETION)) {
1809		trace_block_bio_complete(bio->bi_disk->queue, bio,
1810					 blk_status_to_errno(bio->bi_status));
1811		bio_clear_flag(bio, BIO_TRACE_COMPLETION);
1812	}
1813
1814	blk_throtl_bio_endio(bio);
1815	/* release cgroup info */
1816	bio_uninit(bio);
1817	if (bio->bi_end_io)
1818		bio->bi_end_io(bio);
1819}
1820EXPORT_SYMBOL(bio_endio);
1821
1822/**
1823 * bio_split - split a bio
1824 * @bio:	bio to split
1825 * @sectors:	number of sectors to split from the front of @bio
1826 * @gfp:	gfp mask
1827 * @bs:		bio set to allocate from
1828 *
1829 * Allocates and returns a new bio which represents @sectors from the start of
1830 * @bio, and updates @bio to represent the remaining sectors.
1831 *
1832 * Unless this is a discard request the newly allocated bio will point
1833 * to @bio's bi_io_vec. It is the caller's responsibility to ensure that
1834 * neither @bio nor @bs are freed before the split bio.
1835 */
1836struct bio *bio_split(struct bio *bio, int sectors,
1837		      gfp_t gfp, struct bio_set *bs)
1838{
1839	struct bio *split;
1840
1841	BUG_ON(sectors <= 0);
1842	BUG_ON(sectors >= bio_sectors(bio));
1843
 
 
 
 
1844	split = bio_clone_fast(bio, gfp, bs);
1845	if (!split)
1846		return NULL;
1847
1848	split->bi_iter.bi_size = sectors << 9;
1849
1850	if (bio_integrity(split))
1851		bio_integrity_trim(split);
1852
1853	bio_advance(bio, split->bi_iter.bi_size);
1854
1855	if (bio_flagged(bio, BIO_TRACE_COMPLETION))
1856		bio_set_flag(split, BIO_TRACE_COMPLETION);
1857
1858	return split;
1859}
1860EXPORT_SYMBOL(bio_split);
1861
1862/**
1863 * bio_trim - trim a bio
1864 * @bio:	bio to trim
1865 * @offset:	number of sectors to trim from the front of @bio
1866 * @size:	size we want to trim @bio to, in sectors
1867 */
1868void bio_trim(struct bio *bio, int offset, int size)
1869{
1870	/* 'bio' is a cloned bio which we need to trim to match
1871	 * the given offset and size.
1872	 */
1873
1874	size <<= 9;
1875	if (offset == 0 && size == bio->bi_iter.bi_size)
1876		return;
1877
1878	bio_advance(bio, offset << 9);
1879	bio->bi_iter.bi_size = size;
1880
1881	if (bio_integrity(bio))
1882		bio_integrity_trim(bio);
1883
1884}
1885EXPORT_SYMBOL_GPL(bio_trim);
1886
1887/*
1888 * create memory pools for biovec's in a bio_set.
1889 * use the global biovec slabs created for general use.
1890 */
1891int biovec_init_pool(mempool_t *pool, int pool_entries)
1892{
1893	struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX;
1894
1895	return mempool_init_slab_pool(pool, pool_entries, bp->slab);
1896}
1897
1898/*
1899 * bioset_exit - exit a bioset initialized with bioset_init()
1900 *
1901 * May be called on a zeroed but uninitialized bioset (i.e. allocated with
1902 * kzalloc()).
1903 */
1904void bioset_exit(struct bio_set *bs)
1905{
1906	if (bs->rescue_workqueue)
1907		destroy_workqueue(bs->rescue_workqueue);
1908	bs->rescue_workqueue = NULL;
1909
1910	mempool_exit(&bs->bio_pool);
1911	mempool_exit(&bs->bvec_pool);
1912
1913	bioset_integrity_free(bs);
1914	if (bs->bio_slab)
1915		bio_put_slab(bs);
1916	bs->bio_slab = NULL;
1917}
1918EXPORT_SYMBOL(bioset_exit);
1919
1920/**
1921 * bioset_init - Initialize a bio_set
1922 * @bs:		pool to initialize
1923 * @pool_size:	Number of bio and bio_vecs to cache in the mempool
1924 * @front_pad:	Number of bytes to allocate in front of the returned bio
1925 * @flags:	Flags to modify behavior, currently %BIOSET_NEED_BVECS
1926 *              and %BIOSET_NEED_RESCUER
1927 *
1928 * Description:
1929 *    Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller
1930 *    to ask for a number of bytes to be allocated in front of the bio.
1931 *    Front pad allocation is useful for embedding the bio inside
1932 *    another structure, to avoid allocating extra data to go with the bio.
1933 *    Note that the bio must be embedded at the END of that structure always,
1934 *    or things will break badly.
1935 *    If %BIOSET_NEED_BVECS is set in @flags, a separate pool will be allocated
1936 *    for allocating iovecs.  This pool is not needed e.g. for bio_clone_fast().
1937 *    If %BIOSET_NEED_RESCUER is set, a workqueue is created which can be used to
1938 *    dispatch queued requests when the mempool runs out of space.
1939 *
1940 */
1941int bioset_init(struct bio_set *bs,
1942		unsigned int pool_size,
1943		unsigned int front_pad,
1944		int flags)
1945{
1946	unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
1947
1948	bs->front_pad = front_pad;
1949
1950	spin_lock_init(&bs->rescue_lock);
1951	bio_list_init(&bs->rescue_list);
1952	INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
1953
1954	bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);
1955	if (!bs->bio_slab)
1956		return -ENOMEM;
1957
1958	if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab))
1959		goto bad;
1960
1961	if ((flags & BIOSET_NEED_BVECS) &&
1962	    biovec_init_pool(&bs->bvec_pool, pool_size))
1963		goto bad;
1964
1965	if (!(flags & BIOSET_NEED_RESCUER))
1966		return 0;
1967
1968	bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
1969	if (!bs->rescue_workqueue)
1970		goto bad;
1971
1972	return 0;
1973bad:
1974	bioset_exit(bs);
1975	return -ENOMEM;
1976}
1977EXPORT_SYMBOL(bioset_init);
1978
1979/*
1980 * Initialize and setup a new bio_set, based on the settings from
1981 * another bio_set.
1982 */
1983int bioset_init_from_src(struct bio_set *bs, struct bio_set *src)
1984{
1985	int flags;
1986
1987	flags = 0;
1988	if (src->bvec_pool.min_nr)
1989		flags |= BIOSET_NEED_BVECS;
1990	if (src->rescue_workqueue)
1991		flags |= BIOSET_NEED_RESCUER;
1992
1993	return bioset_init(bs, src->bio_pool.min_nr, src->front_pad, flags);
1994}
1995EXPORT_SYMBOL(bioset_init_from_src);
1996
1997#ifdef CONFIG_BLK_CGROUP
1998
1999/**
2000 * bio_disassociate_blkg - puts back the blkg reference if associated
2001 * @bio: target bio
2002 *
2003 * Helper to disassociate the blkg from @bio if a blkg is associated.
2004 */
2005void bio_disassociate_blkg(struct bio *bio)
2006{
2007	if (bio->bi_blkg) {
2008		blkg_put(bio->bi_blkg);
2009		bio->bi_blkg = NULL;
2010	}
2011}
2012EXPORT_SYMBOL_GPL(bio_disassociate_blkg);
2013
2014/**
2015 * __bio_associate_blkg - associate a bio with the a blkg
2016 * @bio: target bio
2017 * @blkg: the blkg to associate
2018 *
2019 * This tries to associate @bio with the specified @blkg.  Association failure
2020 * is handled by walking up the blkg tree.  Therefore, the blkg associated can
2021 * be anything between @blkg and the root_blkg.  This situation only happens
2022 * when a cgroup is dying and then the remaining bios will spill to the closest
2023 * alive blkg.
2024 *
2025 * A reference will be taken on the @blkg and will be released when @bio is
2026 * freed.
2027 */
2028static void __bio_associate_blkg(struct bio *bio, struct blkcg_gq *blkg)
2029{
2030	bio_disassociate_blkg(bio);
2031
2032	bio->bi_blkg = blkg_tryget_closest(blkg);
2033}
2034
2035/**
2036 * bio_associate_blkg_from_css - associate a bio with a specified css
2037 * @bio: target bio
2038 * @css: target css
2039 *
2040 * Associate @bio with the blkg found by combining the css's blkg and the
2041 * request_queue of the @bio.  This falls back to the queue's root_blkg if
2042 * the association fails with the css.
2043 */
2044void bio_associate_blkg_from_css(struct bio *bio,
2045				 struct cgroup_subsys_state *css)
2046{
2047	struct request_queue *q = bio->bi_disk->queue;
2048	struct blkcg_gq *blkg;
2049
2050	rcu_read_lock();
2051
2052	if (!css || !css->parent)
2053		blkg = q->root_blkg;
2054	else
2055		blkg = blkg_lookup_create(css_to_blkcg(css), q);
2056
2057	__bio_associate_blkg(bio, blkg);
2058
2059	rcu_read_unlock();
2060}
2061EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css);
2062
2063#ifdef CONFIG_MEMCG
2064/**
2065 * bio_associate_blkg_from_page - associate a bio with the page's blkg
2066 * @bio: target bio
2067 * @page: the page to lookup the blkcg from
2068 *
2069 * Associate @bio with the blkg from @page's owning memcg and the respective
2070 * request_queue.  If cgroup_e_css returns %NULL, fall back to the queue's
2071 * root_blkg.
2072 */
2073void bio_associate_blkg_from_page(struct bio *bio, struct page *page)
2074{
2075	struct cgroup_subsys_state *css;
2076
2077	if (!page->mem_cgroup)
2078		return;
2079
2080	rcu_read_lock();
2081
2082	css = cgroup_e_css(page->mem_cgroup->css.cgroup, &io_cgrp_subsys);
2083	bio_associate_blkg_from_css(bio, css);
2084
2085	rcu_read_unlock();
2086}
2087#endif /* CONFIG_MEMCG */
2088
2089/**
2090 * bio_associate_blkg - associate a bio with a blkg
2091 * @bio: target bio
2092 *
2093 * Associate @bio with the blkg found from the bio's css and request_queue.
2094 * If one is not found, bio_lookup_blkg() creates the blkg.  If a blkg is
2095 * already associated, the css is reused and association redone as the
2096 * request_queue may have changed.
2097 */
2098void bio_associate_blkg(struct bio *bio)
2099{
2100	struct cgroup_subsys_state *css;
2101
2102	rcu_read_lock();
2103
2104	if (bio->bi_blkg)
2105		css = &bio_blkcg(bio)->css;
2106	else
2107		css = blkcg_css();
2108
2109	bio_associate_blkg_from_css(bio, css);
2110
2111	rcu_read_unlock();
2112}
2113EXPORT_SYMBOL_GPL(bio_associate_blkg);
2114
2115/**
2116 * bio_clone_blkg_association - clone blkg association from src to dst bio
2117 * @dst: destination bio
2118 * @src: source bio
2119 */
2120void bio_clone_blkg_association(struct bio *dst, struct bio *src)
2121{
2122	rcu_read_lock();
2123
2124	if (src->bi_blkg)
2125		__bio_associate_blkg(dst, src->bi_blkg);
2126
2127	rcu_read_unlock();
2128}
2129EXPORT_SYMBOL_GPL(bio_clone_blkg_association);
2130#endif /* CONFIG_BLK_CGROUP */
2131
2132static void __init biovec_init_slabs(void)
2133{
2134	int i;
2135
2136	for (i = 0; i < BVEC_POOL_NR; i++) {
2137		int size;
2138		struct biovec_slab *bvs = bvec_slabs + i;
2139
2140		if (bvs->nr_vecs <= BIO_INLINE_VECS) {
2141			bvs->slab = NULL;
2142			continue;
2143		}
2144
2145		size = bvs->nr_vecs * sizeof(struct bio_vec);
2146		bvs->slab = kmem_cache_create(bvs->name, size, 0,
2147                                SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
2148	}
2149}
2150
2151static int __init init_bio(void)
2152{
2153	bio_slab_max = 2;
2154	bio_slab_nr = 0;
2155	bio_slabs = kcalloc(bio_slab_max, sizeof(struct bio_slab),
2156			    GFP_KERNEL);
2157
2158	BUILD_BUG_ON(BIO_FLAG_LAST > BVEC_POOL_OFFSET);
2159
2160	if (!bio_slabs)
2161		panic("bio: can't allocate bios\n");
2162
2163	bio_integrity_init();
2164	biovec_init_slabs();
2165
2166	if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS))
2167		panic("bio: can't allocate bios\n");
2168
2169	if (bioset_integrity_create(&fs_bio_set, BIO_POOL_SIZE))
2170		panic("bio: can't create integrity pool\n");
2171
2172	return 0;
2173}
2174subsys_initcall(init_bio);