Linux Audio

Check our new training course

Loading...
v6.13.7
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * raid1.c : Multiple Devices driver for Linux
   4 *
   5 * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat
   6 *
   7 * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman
   8 *
   9 * RAID-1 management functions.
  10 *
  11 * Better read-balancing code written by Mika Kuoppala <miku@iki.fi>, 2000
  12 *
  13 * Fixes to reconstruction by Jakob Østergaard" <jakob@ostenfeld.dk>
  14 * Various fixes by Neil Brown <neilb@cse.unsw.edu.au>
  15 *
  16 * Changes by Peter T. Breuer <ptb@it.uc3m.es> 31/1/2003 to support
  17 * bitmapped intelligence in resync:
  18 *
  19 *      - bitmap marked during normal i/o
  20 *      - bitmap used to skip nondirty blocks during sync
  21 *
  22 * Additions to bitmap code, (C) 2003-2004 Paul Clements, SteelEye Technology:
  23 * - persistent bitmap code
  24 */
  25
  26#include <linux/slab.h>
  27#include <linux/delay.h>
  28#include <linux/blkdev.h>
  29#include <linux/module.h>
  30#include <linux/seq_file.h>
  31#include <linux/ratelimit.h>
  32#include <linux/interval_tree_generic.h>
  33
  34#include <trace/events/block.h>
  35
  36#include "md.h"
  37#include "raid1.h"
  38#include "md-bitmap.h"
  39
  40#define UNSUPPORTED_MDDEV_FLAGS		\
  41	((1L << MD_HAS_JOURNAL) |	\
  42	 (1L << MD_JOURNAL_CLEAN) |	\
  43	 (1L << MD_HAS_PPL) |		\
  44	 (1L << MD_HAS_MULTIPLE_PPLS))
  45
  46static void allow_barrier(struct r1conf *conf, sector_t sector_nr);
  47static void lower_barrier(struct r1conf *conf, sector_t sector_nr);
  48
  49#define RAID_1_10_NAME "raid1"
 
 
  50#include "raid1-10.c"
  51
  52#define START(node) ((node)->start)
  53#define LAST(node) ((node)->last)
  54INTERVAL_TREE_DEFINE(struct serial_info, node, sector_t, _subtree_last,
  55		     START, LAST, static inline, raid1_rb);
  56
  57static int check_and_add_serial(struct md_rdev *rdev, struct r1bio *r1_bio,
  58				struct serial_info *si, int idx)
  59{
  60	unsigned long flags;
  61	int ret = 0;
  62	sector_t lo = r1_bio->sector;
  63	sector_t hi = lo + r1_bio->sectors;
  64	struct serial_in_rdev *serial = &rdev->serial[idx];
  65
  66	spin_lock_irqsave(&serial->serial_lock, flags);
  67	/* collision happened */
  68	if (raid1_rb_iter_first(&serial->serial_rb, lo, hi))
  69		ret = -EBUSY;
  70	else {
  71		si->start = lo;
  72		si->last = hi;
  73		raid1_rb_insert(si, &serial->serial_rb);
  74	}
  75	spin_unlock_irqrestore(&serial->serial_lock, flags);
  76
  77	return ret;
  78}
  79
  80static void wait_for_serialization(struct md_rdev *rdev, struct r1bio *r1_bio)
  81{
  82	struct mddev *mddev = rdev->mddev;
  83	struct serial_info *si;
  84	int idx = sector_to_idx(r1_bio->sector);
  85	struct serial_in_rdev *serial = &rdev->serial[idx];
  86
  87	if (WARN_ON(!mddev->serial_info_pool))
  88		return;
  89	si = mempool_alloc(mddev->serial_info_pool, GFP_NOIO);
  90	wait_event(serial->serial_io_wait,
  91		   check_and_add_serial(rdev, r1_bio, si, idx) == 0);
  92}
  93
  94static void remove_serial(struct md_rdev *rdev, sector_t lo, sector_t hi)
  95{
  96	struct serial_info *si;
  97	unsigned long flags;
  98	int found = 0;
  99	struct mddev *mddev = rdev->mddev;
 100	int idx = sector_to_idx(lo);
 101	struct serial_in_rdev *serial = &rdev->serial[idx];
 102
 103	spin_lock_irqsave(&serial->serial_lock, flags);
 104	for (si = raid1_rb_iter_first(&serial->serial_rb, lo, hi);
 105	     si; si = raid1_rb_iter_next(si, lo, hi)) {
 106		if (si->start == lo && si->last == hi) {
 107			raid1_rb_remove(si, &serial->serial_rb);
 108			mempool_free(si, mddev->serial_info_pool);
 109			found = 1;
 110			break;
 111		}
 112	}
 113	if (!found)
 114		WARN(1, "The write IO is not recorded for serialization\n");
 115	spin_unlock_irqrestore(&serial->serial_lock, flags);
 116	wake_up(&serial->serial_io_wait);
 117}
 118
 119/*
 120 * for resync bio, r1bio pointer can be retrieved from the per-bio
 121 * 'struct resync_pages'.
 122 */
 123static inline struct r1bio *get_resync_r1bio(struct bio *bio)
 124{
 125	return get_resync_pages(bio)->raid_bio;
 126}
 127
 128static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data)
 129{
 130	struct pool_info *pi = data;
 131	int size = offsetof(struct r1bio, bios[pi->raid_disks]);
 132
 133	/* allocate a r1bio with room for raid_disks entries in the bios array */
 134	return kzalloc(size, gfp_flags);
 135}
 136
 137#define RESYNC_DEPTH 32
 138#define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
 139#define RESYNC_WINDOW (RESYNC_BLOCK_SIZE * RESYNC_DEPTH)
 140#define RESYNC_WINDOW_SECTORS (RESYNC_WINDOW >> 9)
 141#define CLUSTER_RESYNC_WINDOW (16 * RESYNC_WINDOW)
 142#define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9)
 143
 144static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data)
 145{
 146	struct pool_info *pi = data;
 147	struct r1bio *r1_bio;
 148	struct bio *bio;
 149	int need_pages;
 150	int j;
 151	struct resync_pages *rps;
 152
 153	r1_bio = r1bio_pool_alloc(gfp_flags, pi);
 154	if (!r1_bio)
 155		return NULL;
 156
 157	rps = kmalloc_array(pi->raid_disks, sizeof(struct resync_pages),
 158			    gfp_flags);
 159	if (!rps)
 160		goto out_free_r1bio;
 161
 162	/*
 163	 * Allocate bios : 1 for reading, n-1 for writing
 164	 */
 165	for (j = pi->raid_disks ; j-- ; ) {
 166		bio = bio_kmalloc(RESYNC_PAGES, gfp_flags);
 167		if (!bio)
 168			goto out_free_bio;
 169		bio_init(bio, NULL, bio->bi_inline_vecs, RESYNC_PAGES, 0);
 170		r1_bio->bios[j] = bio;
 171	}
 172	/*
 173	 * Allocate RESYNC_PAGES data pages and attach them to
 174	 * the first bio.
 175	 * If this is a user-requested check/repair, allocate
 176	 * RESYNC_PAGES for each bio.
 177	 */
 178	if (test_bit(MD_RECOVERY_REQUESTED, &pi->mddev->recovery))
 179		need_pages = pi->raid_disks;
 180	else
 181		need_pages = 1;
 182	for (j = 0; j < pi->raid_disks; j++) {
 183		struct resync_pages *rp = &rps[j];
 184
 185		bio = r1_bio->bios[j];
 186
 187		if (j < need_pages) {
 188			if (resync_alloc_pages(rp, gfp_flags))
 189				goto out_free_pages;
 190		} else {
 191			memcpy(rp, &rps[0], sizeof(*rp));
 192			resync_get_all_pages(rp);
 193		}
 194
 195		rp->raid_bio = r1_bio;
 196		bio->bi_private = rp;
 197	}
 198
 199	r1_bio->master_bio = NULL;
 200
 201	return r1_bio;
 202
 203out_free_pages:
 204	while (--j >= 0)
 205		resync_free_pages(&rps[j]);
 206
 207out_free_bio:
 208	while (++j < pi->raid_disks) {
 209		bio_uninit(r1_bio->bios[j]);
 210		kfree(r1_bio->bios[j]);
 211	}
 212	kfree(rps);
 213
 214out_free_r1bio:
 215	rbio_pool_free(r1_bio, data);
 216	return NULL;
 217}
 218
 219static void r1buf_pool_free(void *__r1_bio, void *data)
 220{
 221	struct pool_info *pi = data;
 222	int i;
 223	struct r1bio *r1bio = __r1_bio;
 224	struct resync_pages *rp = NULL;
 225
 226	for (i = pi->raid_disks; i--; ) {
 227		rp = get_resync_pages(r1bio->bios[i]);
 228		resync_free_pages(rp);
 229		bio_uninit(r1bio->bios[i]);
 230		kfree(r1bio->bios[i]);
 231	}
 232
 233	/* resync pages array stored in the 1st bio's .bi_private */
 234	kfree(rp);
 235
 236	rbio_pool_free(r1bio, data);
 237}
 238
 239static void put_all_bios(struct r1conf *conf, struct r1bio *r1_bio)
 240{
 241	int i;
 242
 243	for (i = 0; i < conf->raid_disks * 2; i++) {
 244		struct bio **bio = r1_bio->bios + i;
 245		if (!BIO_SPECIAL(*bio))
 246			bio_put(*bio);
 247		*bio = NULL;
 248	}
 249}
 250
 251static void free_r1bio(struct r1bio *r1_bio)
 252{
 253	struct r1conf *conf = r1_bio->mddev->private;
 254
 255	put_all_bios(conf, r1_bio);
 256	mempool_free(r1_bio, &conf->r1bio_pool);
 257}
 258
 259static void put_buf(struct r1bio *r1_bio)
 260{
 261	struct r1conf *conf = r1_bio->mddev->private;
 262	sector_t sect = r1_bio->sector;
 263	int i;
 264
 265	for (i = 0; i < conf->raid_disks * 2; i++) {
 266		struct bio *bio = r1_bio->bios[i];
 267		if (bio->bi_end_io)
 268			rdev_dec_pending(conf->mirrors[i].rdev, r1_bio->mddev);
 269	}
 270
 271	mempool_free(r1_bio, &conf->r1buf_pool);
 272
 273	lower_barrier(conf, sect);
 274}
 275
 276static void reschedule_retry(struct r1bio *r1_bio)
 277{
 278	unsigned long flags;
 279	struct mddev *mddev = r1_bio->mddev;
 280	struct r1conf *conf = mddev->private;
 281	int idx;
 282
 283	idx = sector_to_idx(r1_bio->sector);
 284	spin_lock_irqsave(&conf->device_lock, flags);
 285	list_add(&r1_bio->retry_list, &conf->retry_list);
 286	atomic_inc(&conf->nr_queued[idx]);
 287	spin_unlock_irqrestore(&conf->device_lock, flags);
 288
 289	wake_up(&conf->wait_barrier);
 290	md_wakeup_thread(mddev->thread);
 291}
 292
 293/*
 294 * raid_end_bio_io() is called when we have finished servicing a mirrored
 295 * operation and are ready to return a success/failure code to the buffer
 296 * cache layer.
 297 */
 298static void call_bio_endio(struct r1bio *r1_bio)
 299{
 300	struct bio *bio = r1_bio->master_bio;
 301
 302	if (!test_bit(R1BIO_Uptodate, &r1_bio->state))
 303		bio->bi_status = BLK_STS_IOERR;
 304
 305	bio_endio(bio);
 306}
 307
 308static void raid_end_bio_io(struct r1bio *r1_bio)
 309{
 310	struct bio *bio = r1_bio->master_bio;
 311	struct r1conf *conf = r1_bio->mddev->private;
 312	sector_t sector = r1_bio->sector;
 313
 314	/* if nobody has done the final endio yet, do it now */
 315	if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) {
 316		pr_debug("raid1: sync end %s on sectors %llu-%llu\n",
 317			 (bio_data_dir(bio) == WRITE) ? "write" : "read",
 318			 (unsigned long long) bio->bi_iter.bi_sector,
 319			 (unsigned long long) bio_end_sector(bio) - 1);
 320
 321		call_bio_endio(r1_bio);
 322	}
 323
 324	free_r1bio(r1_bio);
 325	/*
 326	 * Wake up any possible resync thread that waits for the device
 327	 * to go idle.  All I/Os, even write-behind writes, are done.
 328	 */
 329	allow_barrier(conf, sector);
 
 
 330}
 331
 332/*
 333 * Update disk head position estimator based on IRQ completion info.
 334 */
 335static inline void update_head_pos(int disk, struct r1bio *r1_bio)
 336{
 337	struct r1conf *conf = r1_bio->mddev->private;
 338
 339	conf->mirrors[disk].head_position =
 340		r1_bio->sector + (r1_bio->sectors);
 341}
 342
 343/*
 344 * Find the disk number which triggered given bio
 345 */
 346static int find_bio_disk(struct r1bio *r1_bio, struct bio *bio)
 347{
 348	int mirror;
 349	struct r1conf *conf = r1_bio->mddev->private;
 350	int raid_disks = conf->raid_disks;
 351
 352	for (mirror = 0; mirror < raid_disks * 2; mirror++)
 353		if (r1_bio->bios[mirror] == bio)
 354			break;
 355
 356	BUG_ON(mirror == raid_disks * 2);
 357	update_head_pos(mirror, r1_bio);
 358
 359	return mirror;
 360}
 361
 362static void raid1_end_read_request(struct bio *bio)
 363{
 364	int uptodate = !bio->bi_status;
 365	struct r1bio *r1_bio = bio->bi_private;
 366	struct r1conf *conf = r1_bio->mddev->private;
 367	struct md_rdev *rdev = conf->mirrors[r1_bio->read_disk].rdev;
 368
 369	/*
 370	 * this branch is our 'one mirror IO has finished' event handler:
 371	 */
 372	update_head_pos(r1_bio->read_disk, r1_bio);
 373
 374	if (uptodate)
 375		set_bit(R1BIO_Uptodate, &r1_bio->state);
 376	else if (test_bit(FailFast, &rdev->flags) &&
 377		 test_bit(R1BIO_FailFast, &r1_bio->state))
 378		/* This was a fail-fast read so we definitely
 379		 * want to retry */
 380		;
 381	else {
 382		/* If all other devices have failed, we want to return
 383		 * the error upwards rather than fail the last device.
 384		 * Here we redefine "uptodate" to mean "Don't want to retry"
 385		 */
 386		unsigned long flags;
 387		spin_lock_irqsave(&conf->device_lock, flags);
 388		if (r1_bio->mddev->degraded == conf->raid_disks ||
 389		    (r1_bio->mddev->degraded == conf->raid_disks-1 &&
 390		     test_bit(In_sync, &rdev->flags)))
 391			uptodate = 1;
 392		spin_unlock_irqrestore(&conf->device_lock, flags);
 393	}
 394
 395	if (uptodate) {
 396		raid_end_bio_io(r1_bio);
 397		rdev_dec_pending(rdev, conf->mddev);
 398	} else {
 399		/*
 400		 * oops, read error:
 401		 */
 402		pr_err_ratelimited("md/raid1:%s: %pg: rescheduling sector %llu\n",
 
 403				   mdname(conf->mddev),
 404				   rdev->bdev,
 405				   (unsigned long long)r1_bio->sector);
 406		set_bit(R1BIO_ReadError, &r1_bio->state);
 407		reschedule_retry(r1_bio);
 408		/* don't drop the reference on read_disk yet */
 409	}
 410}
 411
 412static void close_write(struct r1bio *r1_bio)
 413{
 414	struct mddev *mddev = r1_bio->mddev;
 415
 416	/* it really is the end of this request */
 417	if (test_bit(R1BIO_BehindIO, &r1_bio->state)) {
 418		bio_free_pages(r1_bio->behind_master_bio);
 419		bio_put(r1_bio->behind_master_bio);
 420		r1_bio->behind_master_bio = NULL;
 421	}
 422
 423	if (test_bit(R1BIO_BehindIO, &r1_bio->state))
 424		mddev->bitmap_ops->end_behind_write(mddev);
 425	md_write_end(mddev);
 
 
 426}
 427
 428static void r1_bio_write_done(struct r1bio *r1_bio)
 429{
 430	if (!atomic_dec_and_test(&r1_bio->remaining))
 431		return;
 432
 433	if (test_bit(R1BIO_WriteError, &r1_bio->state))
 434		reschedule_retry(r1_bio);
 435	else {
 436		close_write(r1_bio);
 437		if (test_bit(R1BIO_MadeGood, &r1_bio->state))
 438			reschedule_retry(r1_bio);
 439		else
 440			raid_end_bio_io(r1_bio);
 441	}
 442}
 443
 444static void raid1_end_write_request(struct bio *bio)
 445{
 446	struct r1bio *r1_bio = bio->bi_private;
 447	int behind = test_bit(R1BIO_BehindIO, &r1_bio->state);
 448	struct r1conf *conf = r1_bio->mddev->private;
 449	struct bio *to_put = NULL;
 450	int mirror = find_bio_disk(r1_bio, bio);
 451	struct md_rdev *rdev = conf->mirrors[mirror].rdev;
 452	bool discard_error;
 453	sector_t lo = r1_bio->sector;
 454	sector_t hi = r1_bio->sector + r1_bio->sectors;
 455
 456	discard_error = bio->bi_status && bio_op(bio) == REQ_OP_DISCARD;
 457
 458	/*
 459	 * 'one mirror IO has finished' event handler:
 460	 */
 461	if (bio->bi_status && !discard_error) {
 462		set_bit(WriteErrorSeen,	&rdev->flags);
 463		if (!test_and_set_bit(WantReplacement, &rdev->flags))
 464			set_bit(MD_RECOVERY_NEEDED, &
 465				conf->mddev->recovery);
 466
 467		if (test_bit(FailFast, &rdev->flags) &&
 468		    (bio->bi_opf & MD_FAILFAST) &&
 469		    /* We never try FailFast to WriteMostly devices */
 470		    !test_bit(WriteMostly, &rdev->flags)) {
 471			md_error(r1_bio->mddev, rdev);
 472		}
 473
 474		/*
 475		 * When the device is faulty, it is not necessary to
 476		 * handle write error.
 
 
 477		 */
 478		if (!test_bit(Faulty, &rdev->flags))
 479			set_bit(R1BIO_WriteError, &r1_bio->state);
 480		else {
 481			/* Finished with this branch */
 482			r1_bio->bios[mirror] = NULL;
 483			to_put = bio;
 484		}
 485	} else {
 486		/*
 487		 * Set R1BIO_Uptodate in our master bio, so that we
 488		 * will return a good error code for to the higher
 489		 * levels even if IO on some other mirrored buffer
 490		 * fails.
 491		 *
 492		 * The 'master' represents the composite IO operation
 493		 * to user-side. So if something waits for IO, then it
 494		 * will wait for the 'master' bio.
 495		 */
 
 
 
 496		r1_bio->bios[mirror] = NULL;
 497		to_put = bio;
 498		/*
 499		 * Do not set R1BIO_Uptodate if the current device is
 500		 * rebuilding or Faulty. This is because we cannot use
 501		 * such device for properly reading the data back (we could
 502		 * potentially use it, if the current write would have felt
 503		 * before rdev->recovery_offset, but for simplicity we don't
 504		 * check this here.
 505		 */
 506		if (test_bit(In_sync, &rdev->flags) &&
 507		    !test_bit(Faulty, &rdev->flags))
 508			set_bit(R1BIO_Uptodate, &r1_bio->state);
 509
 510		/* Maybe we can clear some bad blocks. */
 511		if (rdev_has_badblock(rdev, r1_bio->sector, r1_bio->sectors) &&
 512		    !discard_error) {
 513			r1_bio->bios[mirror] = IO_MADE_GOOD;
 514			set_bit(R1BIO_MadeGood, &r1_bio->state);
 515		}
 516	}
 517
 518	if (behind) {
 519		if (test_bit(CollisionCheck, &rdev->flags))
 520			remove_serial(rdev, lo, hi);
 521		if (test_bit(WriteMostly, &rdev->flags))
 522			atomic_dec(&r1_bio->behind_remaining);
 523
 524		/*
 525		 * In behind mode, we ACK the master bio once the I/O
 526		 * has safely reached all non-writemostly
 527		 * disks. Setting the Returned bit ensures that this
 528		 * gets done only once -- we don't ever want to return
 529		 * -EIO here, instead we'll wait
 530		 */
 531		if (atomic_read(&r1_bio->behind_remaining) >= (atomic_read(&r1_bio->remaining)-1) &&
 532		    test_bit(R1BIO_Uptodate, &r1_bio->state)) {
 533			/* Maybe we can return now */
 534			if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) {
 535				struct bio *mbio = r1_bio->master_bio;
 536				pr_debug("raid1: behind end write sectors"
 537					 " %llu-%llu\n",
 538					 (unsigned long long) mbio->bi_iter.bi_sector,
 539					 (unsigned long long) bio_end_sector(mbio) - 1);
 540				call_bio_endio(r1_bio);
 541			}
 542		}
 543	} else if (rdev->mddev->serialize_policy)
 544		remove_serial(rdev, lo, hi);
 545	if (r1_bio->bios[mirror] == NULL)
 546		rdev_dec_pending(rdev, conf->mddev);
 547
 548	/*
 549	 * Let's see if all mirrored write operations have finished
 550	 * already.
 551	 */
 552	r1_bio_write_done(r1_bio);
 553
 554	if (to_put)
 555		bio_put(to_put);
 556}
 557
 558static sector_t align_to_barrier_unit_end(sector_t start_sector,
 559					  sector_t sectors)
 560{
 561	sector_t len;
 562
 563	WARN_ON(sectors == 0);
 564	/*
 565	 * len is the number of sectors from start_sector to end of the
 566	 * barrier unit which start_sector belongs to.
 567	 */
 568	len = round_up(start_sector + 1, BARRIER_UNIT_SECTOR_SIZE) -
 569	      start_sector;
 570
 571	if (len > sectors)
 572		len = sectors;
 573
 574	return len;
 575}
 576
 577static void update_read_sectors(struct r1conf *conf, int disk,
 578				sector_t this_sector, int len)
 579{
 580	struct raid1_info *info = &conf->mirrors[disk];
 581
 582	atomic_inc(&info->rdev->nr_pending);
 583	if (info->next_seq_sect != this_sector)
 584		info->seq_start = this_sector;
 585	info->next_seq_sect = this_sector + len;
 586}
 587
 588static int choose_first_rdev(struct r1conf *conf, struct r1bio *r1_bio,
 589			     int *max_sectors)
 
 
 590{
 591	sector_t this_sector = r1_bio->sector;
 592	int len = r1_bio->sectors;
 
 
 
 593	int disk;
 
 
 
 
 
 594
 595	for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) {
 596		struct md_rdev *rdev;
 597		int read_len;
 598
 599		if (r1_bio->bios[disk] == IO_BLOCKED)
 600			continue;
 601
 602		rdev = conf->mirrors[disk].rdev;
 603		if (!rdev || test_bit(Faulty, &rdev->flags))
 604			continue;
 605
 606		/* choose the first disk even if it has some bad blocks. */
 607		read_len = raid1_check_read_range(rdev, this_sector, &len);
 608		if (read_len > 0) {
 609			update_read_sectors(conf, disk, this_sector, read_len);
 610			*max_sectors = read_len;
 611			return disk;
 612		}
 613	}
 614
 615	return -1;
 616}
 617
 618static bool rdev_in_recovery(struct md_rdev *rdev, struct r1bio *r1_bio)
 619{
 620	return !test_bit(In_sync, &rdev->flags) &&
 621	       rdev->recovery_offset < r1_bio->sector + r1_bio->sectors;
 622}
 623
 624static int choose_bb_rdev(struct r1conf *conf, struct r1bio *r1_bio,
 625			  int *max_sectors)
 626{
 627	sector_t this_sector = r1_bio->sector;
 628	int best_disk = -1;
 629	int best_len = 0;
 630	int disk;
 631
 632	for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) {
 633		struct md_rdev *rdev;
 634		int len;
 635		int read_len;
 636
 637		if (r1_bio->bios[disk] == IO_BLOCKED)
 638			continue;
 639
 640		rdev = conf->mirrors[disk].rdev;
 641		if (!rdev || test_bit(Faulty, &rdev->flags) ||
 642		    rdev_in_recovery(rdev, r1_bio) ||
 643		    test_bit(WriteMostly, &rdev->flags))
 644			continue;
 645
 646		/* keep track of the disk with the most readable sectors. */
 647		len = r1_bio->sectors;
 648		read_len = raid1_check_read_range(rdev, this_sector, &len);
 649		if (read_len > best_len) {
 650			best_disk = disk;
 651			best_len = read_len;
 652		}
 653	}
 654
 655	if (best_disk != -1) {
 656		*max_sectors = best_len;
 657		update_read_sectors(conf, best_disk, this_sector, best_len);
 658	}
 659
 660	return best_disk;
 661}
 662
 663static int choose_slow_rdev(struct r1conf *conf, struct r1bio *r1_bio,
 664			    int *max_sectors)
 665{
 666	sector_t this_sector = r1_bio->sector;
 667	int bb_disk = -1;
 668	int bb_read_len = 0;
 669	int disk;
 670
 671	for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) {
 672		struct md_rdev *rdev;
 673		int len;
 674		int read_len;
 675
 676		if (r1_bio->bios[disk] == IO_BLOCKED)
 677			continue;
 678
 679		rdev = conf->mirrors[disk].rdev;
 680		if (!rdev || test_bit(Faulty, &rdev->flags) ||
 681		    !test_bit(WriteMostly, &rdev->flags) ||
 682		    rdev_in_recovery(rdev, r1_bio))
 
 
 
 
 
 
 
 
 
 
 683			continue;
 684
 685		/* there are no bad blocks, we can use this disk */
 686		len = r1_bio->sectors;
 687		read_len = raid1_check_read_range(rdev, this_sector, &len);
 688		if (read_len == r1_bio->sectors) {
 689			*max_sectors = read_len;
 690			update_read_sectors(conf, disk, this_sector, read_len);
 691			return disk;
 692		}
 693
 694		/*
 695		 * there are partial bad blocks, choose the rdev with largest
 696		 * read length.
 697		 */
 698		if (read_len > bb_read_len) {
 699			bb_disk = disk;
 700			bb_read_len = read_len;
 701		}
 702	}
 703
 704	if (bb_disk != -1) {
 705		*max_sectors = bb_read_len;
 706		update_read_sectors(conf, bb_disk, this_sector, bb_read_len);
 707	}
 708
 709	return bb_disk;
 710}
 711
 712static bool is_sequential(struct r1conf *conf, int disk, struct r1bio *r1_bio)
 713{
 714	/* TODO: address issues with this check and concurrency. */
 715	return conf->mirrors[disk].next_seq_sect == r1_bio->sector ||
 716	       conf->mirrors[disk].head_position == r1_bio->sector;
 717}
 718
 719/*
 720 * If buffered sequential IO size exceeds optimal iosize, check if there is idle
 721 * disk. If yes, choose the idle disk.
 722 */
 723static bool should_choose_next(struct r1conf *conf, int disk)
 724{
 725	struct raid1_info *mirror = &conf->mirrors[disk];
 726	int opt_iosize;
 727
 728	if (!test_bit(Nonrot, &mirror->rdev->flags))
 729		return false;
 730
 731	opt_iosize = bdev_io_opt(mirror->rdev->bdev) >> 9;
 732	return opt_iosize > 0 && mirror->seq_start != MaxSector &&
 733	       mirror->next_seq_sect > opt_iosize &&
 734	       mirror->next_seq_sect - opt_iosize >= mirror->seq_start;
 735}
 736
 737static bool rdev_readable(struct md_rdev *rdev, struct r1bio *r1_bio)
 738{
 739	if (!rdev || test_bit(Faulty, &rdev->flags))
 740		return false;
 741
 742	if (rdev_in_recovery(rdev, r1_bio))
 743		return false;
 744
 745	/* don't read from slow disk unless have to */
 746	if (test_bit(WriteMostly, &rdev->flags))
 747		return false;
 748
 749	/* don't split IO for bad blocks unless have to */
 750	if (rdev_has_badblock(rdev, r1_bio->sector, r1_bio->sectors))
 751		return false;
 752
 753	return true;
 754}
 755
 756struct read_balance_ctl {
 757	sector_t closest_dist;
 758	int closest_dist_disk;
 759	int min_pending;
 760	int min_pending_disk;
 761	int sequential_disk;
 762	int readable_disks;
 763};
 764
 765static int choose_best_rdev(struct r1conf *conf, struct r1bio *r1_bio)
 766{
 767	int disk;
 768	struct read_balance_ctl ctl = {
 769		.closest_dist_disk      = -1,
 770		.closest_dist           = MaxSector,
 771		.min_pending_disk       = -1,
 772		.min_pending            = UINT_MAX,
 773		.sequential_disk	= -1,
 774	};
 775
 776	for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) {
 777		struct md_rdev *rdev;
 778		sector_t dist;
 779		unsigned int pending;
 780
 781		if (r1_bio->bios[disk] == IO_BLOCKED)
 782			continue;
 783
 784		rdev = conf->mirrors[disk].rdev;
 785		if (!rdev_readable(rdev, r1_bio))
 786			continue;
 
 
 
 
 
 787
 788		/* At least two disks to choose from so failfast is OK */
 789		if (ctl.readable_disks++ == 1)
 790			set_bit(R1BIO_FailFast, &r1_bio->state);
 791
 
 
 792		pending = atomic_read(&rdev->nr_pending);
 793		dist = abs(r1_bio->sector - conf->mirrors[disk].head_position);
 794
 
 
 
 795		/* Don't change to another disk for sequential reads */
 796		if (is_sequential(conf, disk, r1_bio)) {
 797			if (!should_choose_next(conf, disk))
 798				return disk;
 
 799
 
 800			/*
 801			 * Add 'pending' to avoid choosing this disk if
 802			 * there is other idle disk.
 803			 */
 804			pending++;
 805			/*
 806			 * If there is no other idle disk, this disk
 807			 * will be chosen.
 
 
 
 
 808			 */
 809			ctl.sequential_disk = disk;
 
 
 
 
 
 
 
 
 810		}
 811
 812		if (ctl.min_pending > pending) {
 813			ctl.min_pending = pending;
 814			ctl.min_pending_disk = disk;
 
 
 
 815		}
 816
 817		if (ctl.closest_dist > dist) {
 818			ctl.closest_dist = dist;
 819			ctl.closest_dist_disk = disk;
 820		}
 821	}
 822
 823	/*
 824	 * sequential IO size exceeds optimal iosize, however, there is no other
 825	 * idle disk, so choose the sequential disk.
 826	 */
 827	if (ctl.sequential_disk != -1 && ctl.min_pending != 0)
 828		return ctl.sequential_disk;
 829
 830	/*
 831	 * If all disks are rotational, choose the closest disk. If any disk is
 832	 * non-rotational, choose the disk with less pending request even the
 833	 * disk is rotational, which might/might not be optimal for raids with
 834	 * mixed ratation/non-rotational disks depending on workload.
 835	 */
 836	if (ctl.min_pending_disk != -1 &&
 837	    (READ_ONCE(conf->nonrot_disks) || ctl.min_pending == 0))
 838		return ctl.min_pending_disk;
 839	else
 840		return ctl.closest_dist_disk;
 841}
 842
 843/*
 844 * This routine returns the disk from which the requested read should be done.
 845 *
 846 * 1) If resync is in progress, find the first usable disk and use it even if it
 847 * has some bad blocks.
 848 *
 849 * 2) Now that there is no resync, loop through all disks and skipping slow
 850 * disks and disks with bad blocks for now. Only pay attention to key disk
 851 * choice.
 852 *
 853 * 3) If we've made it this far, now look for disks with bad blocks and choose
 854 * the one with most number of sectors.
 855 *
 856 * 4) If we are all the way at the end, we have no choice but to use a disk even
 857 * if it is write mostly.
 858 *
 859 * The rdev for the device selected will have nr_pending incremented.
 860 */
 861static int read_balance(struct r1conf *conf, struct r1bio *r1_bio,
 862			int *max_sectors)
 863{
 864	int disk;
 865
 866	clear_bit(R1BIO_FailFast, &r1_bio->state);
 
 
 
 
 
 867
 868	if (raid1_should_read_first(conf->mddev, r1_bio->sector,
 869				    r1_bio->sectors))
 870		return choose_first_rdev(conf, r1_bio, max_sectors);
 871
 872	disk = choose_best_rdev(conf, r1_bio);
 873	if (disk >= 0) {
 874		*max_sectors = r1_bio->sectors;
 875		update_read_sectors(conf, disk, r1_bio->sector,
 876				    r1_bio->sectors);
 877		return disk;
 878	}
 
 
 879
 880	/*
 881	 * If we are here it means we didn't find a perfectly good disk so
 882	 * now spend a bit more time trying to find one with the most good
 883	 * sectors.
 884	 */
 885	disk = choose_bb_rdev(conf, r1_bio, max_sectors);
 886	if (disk >= 0)
 887		return disk;
 888
 889	return choose_slow_rdev(conf, r1_bio, max_sectors);
 890}
 891
 892static void wake_up_barrier(struct r1conf *conf)
 893{
 894	if (wq_has_sleeper(&conf->wait_barrier))
 895		wake_up(&conf->wait_barrier);
 896}
 897
 898static void flush_bio_list(struct r1conf *conf, struct bio *bio)
 899{
 900	/* flush any pending bitmap writes to disk before proceeding w/ I/O */
 901	raid1_prepare_flush_writes(conf->mddev);
 902	wake_up_barrier(conf);
 903
 904	while (bio) { /* submit pending writes */
 905		struct bio *next = bio->bi_next;
 906
 907		raid1_submit_write(bio);
 
 
 
 
 
 
 
 
 
 908		bio = next;
 909		cond_resched();
 910	}
 911}
 912
 913static void flush_pending_writes(struct r1conf *conf)
 914{
 915	/* Any writes that have been queued but are awaiting
 916	 * bitmap updates get flushed here.
 917	 */
 918	spin_lock_irq(&conf->device_lock);
 919
 920	if (conf->pending_bio_list.head) {
 921		struct blk_plug plug;
 922		struct bio *bio;
 923
 924		bio = bio_list_get(&conf->pending_bio_list);
 
 925		spin_unlock_irq(&conf->device_lock);
 926
 927		/*
 928		 * As this is called in a wait_event() loop (see freeze_array),
 929		 * current->state might be TASK_UNINTERRUPTIBLE which will
 930		 * cause a warning when we prepare to wait again.  As it is
 931		 * rare that this path is taken, it is perfectly safe to force
 932		 * us to go around the wait_event() loop again, so the warning
 933		 * is a false-positive.  Silence the warning by resetting
 934		 * thread state
 935		 */
 936		__set_current_state(TASK_RUNNING);
 937		blk_start_plug(&plug);
 938		flush_bio_list(conf, bio);
 939		blk_finish_plug(&plug);
 940	} else
 941		spin_unlock_irq(&conf->device_lock);
 942}
 943
 944/* Barriers....
 945 * Sometimes we need to suspend IO while we do something else,
 946 * either some resync/recovery, or reconfigure the array.
 947 * To do this we raise a 'barrier'.
 948 * The 'barrier' is a counter that can be raised multiple times
 949 * to count how many activities are happening which preclude
 950 * normal IO.
 951 * We can only raise the barrier if there is no pending IO.
 952 * i.e. if nr_pending == 0.
 953 * We choose only to raise the barrier if no-one is waiting for the
 954 * barrier to go down.  This means that as soon as an IO request
 955 * is ready, no other operations which require a barrier will start
 956 * until the IO request has had a chance.
 957 *
 958 * So: regular IO calls 'wait_barrier'.  When that returns there
 959 *    is no backgroup IO happening,  It must arrange to call
 960 *    allow_barrier when it has finished its IO.
 961 * backgroup IO calls must call raise_barrier.  Once that returns
 962 *    there is no normal IO happeing.  It must arrange to call
 963 *    lower_barrier when the particular background IO completes.
 964 *
 965 * If resync/recovery is interrupted, returns -EINTR;
 966 * Otherwise, returns 0.
 967 */
 968static int raise_barrier(struct r1conf *conf, sector_t sector_nr)
 969{
 970	int idx = sector_to_idx(sector_nr);
 971
 972	spin_lock_irq(&conf->resync_lock);
 973
 974	/* Wait until no block IO is waiting */
 975	wait_event_lock_irq(conf->wait_barrier,
 976			    !atomic_read(&conf->nr_waiting[idx]),
 977			    conf->resync_lock);
 978
 979	/* block any new IO from starting */
 980	atomic_inc(&conf->barrier[idx]);
 981	/*
 982	 * In raise_barrier() we firstly increase conf->barrier[idx] then
 983	 * check conf->nr_pending[idx]. In _wait_barrier() we firstly
 984	 * increase conf->nr_pending[idx] then check conf->barrier[idx].
 985	 * A memory barrier here to make sure conf->nr_pending[idx] won't
 986	 * be fetched before conf->barrier[idx] is increased. Otherwise
 987	 * there will be a race between raise_barrier() and _wait_barrier().
 988	 */
 989	smp_mb__after_atomic();
 990
 991	/* For these conditions we must wait:
 992	 * A: while the array is in frozen state
 993	 * B: while conf->nr_pending[idx] is not 0, meaning regular I/O
 994	 *    existing in corresponding I/O barrier bucket.
 995	 * C: while conf->barrier[idx] >= RESYNC_DEPTH, meaning reaches
 996	 *    max resync count which allowed on current I/O barrier bucket.
 997	 */
 998	wait_event_lock_irq(conf->wait_barrier,
 999			    (!conf->array_frozen &&
1000			     !atomic_read(&conf->nr_pending[idx]) &&
1001			     atomic_read(&conf->barrier[idx]) < RESYNC_DEPTH) ||
1002				test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery),
1003			    conf->resync_lock);
1004
1005	if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
1006		atomic_dec(&conf->barrier[idx]);
1007		spin_unlock_irq(&conf->resync_lock);
1008		wake_up(&conf->wait_barrier);
1009		return -EINTR;
1010	}
1011
1012	atomic_inc(&conf->nr_sync_pending);
1013	spin_unlock_irq(&conf->resync_lock);
1014
1015	return 0;
1016}
1017
1018static void lower_barrier(struct r1conf *conf, sector_t sector_nr)
1019{
1020	int idx = sector_to_idx(sector_nr);
1021
1022	BUG_ON(atomic_read(&conf->barrier[idx]) <= 0);
1023
1024	atomic_dec(&conf->barrier[idx]);
1025	atomic_dec(&conf->nr_sync_pending);
1026	wake_up(&conf->wait_barrier);
1027}
1028
1029static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait)
1030{
1031	bool ret = true;
1032
1033	/*
1034	 * We need to increase conf->nr_pending[idx] very early here,
1035	 * then raise_barrier() can be blocked when it waits for
1036	 * conf->nr_pending[idx] to be 0. Then we can avoid holding
1037	 * conf->resync_lock when there is no barrier raised in same
1038	 * barrier unit bucket. Also if the array is frozen, I/O
1039	 * should be blocked until array is unfrozen.
1040	 */
1041	atomic_inc(&conf->nr_pending[idx]);
1042	/*
1043	 * In _wait_barrier() we firstly increase conf->nr_pending[idx], then
1044	 * check conf->barrier[idx]. In raise_barrier() we firstly increase
1045	 * conf->barrier[idx], then check conf->nr_pending[idx]. A memory
1046	 * barrier is necessary here to make sure conf->barrier[idx] won't be
1047	 * fetched before conf->nr_pending[idx] is increased. Otherwise there
1048	 * will be a race between _wait_barrier() and raise_barrier().
1049	 */
1050	smp_mb__after_atomic();
1051
1052	/*
1053	 * Don't worry about checking two atomic_t variables at same time
1054	 * here. If during we check conf->barrier[idx], the array is
1055	 * frozen (conf->array_frozen is 1), and chonf->barrier[idx] is
1056	 * 0, it is safe to return and make the I/O continue. Because the
1057	 * array is frozen, all I/O returned here will eventually complete
1058	 * or be queued, no race will happen. See code comment in
1059	 * frozen_array().
1060	 */
1061	if (!READ_ONCE(conf->array_frozen) &&
1062	    !atomic_read(&conf->barrier[idx]))
1063		return ret;
1064
1065	/*
1066	 * After holding conf->resync_lock, conf->nr_pending[idx]
1067	 * should be decreased before waiting for barrier to drop.
1068	 * Otherwise, we may encounter a race condition because
1069	 * raise_barrer() might be waiting for conf->nr_pending[idx]
1070	 * to be 0 at same time.
1071	 */
1072	spin_lock_irq(&conf->resync_lock);
1073	atomic_inc(&conf->nr_waiting[idx]);
1074	atomic_dec(&conf->nr_pending[idx]);
1075	/*
1076	 * In case freeze_array() is waiting for
1077	 * get_unqueued_pending() == extra
1078	 */
1079	wake_up_barrier(conf);
1080	/* Wait for the barrier in same barrier unit bucket to drop. */
1081
1082	/* Return false when nowait flag is set */
1083	if (nowait) {
1084		ret = false;
1085	} else {
1086		wait_event_lock_irq(conf->wait_barrier,
1087				!conf->array_frozen &&
1088				!atomic_read(&conf->barrier[idx]),
1089				conf->resync_lock);
1090		atomic_inc(&conf->nr_pending[idx]);
1091	}
1092
1093	atomic_dec(&conf->nr_waiting[idx]);
1094	spin_unlock_irq(&conf->resync_lock);
1095	return ret;
1096}
1097
1098static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait)
1099{
1100	int idx = sector_to_idx(sector_nr);
1101	bool ret = true;
1102
1103	/*
1104	 * Very similar to _wait_barrier(). The difference is, for read
1105	 * I/O we don't need wait for sync I/O, but if the whole array
1106	 * is frozen, the read I/O still has to wait until the array is
1107	 * unfrozen. Since there is no ordering requirement with
1108	 * conf->barrier[idx] here, memory barrier is unnecessary as well.
1109	 */
1110	atomic_inc(&conf->nr_pending[idx]);
1111
1112	if (!READ_ONCE(conf->array_frozen))
1113		return ret;
1114
1115	spin_lock_irq(&conf->resync_lock);
1116	atomic_inc(&conf->nr_waiting[idx]);
1117	atomic_dec(&conf->nr_pending[idx]);
1118	/*
1119	 * In case freeze_array() is waiting for
1120	 * get_unqueued_pending() == extra
1121	 */
1122	wake_up_barrier(conf);
1123	/* Wait for array to be unfrozen */
1124
1125	/* Return false when nowait flag is set */
1126	if (nowait) {
1127		/* Return false when nowait flag is set */
1128		ret = false;
1129	} else {
1130		wait_event_lock_irq(conf->wait_barrier,
1131				!conf->array_frozen,
1132				conf->resync_lock);
1133		atomic_inc(&conf->nr_pending[idx]);
1134	}
1135
1136	atomic_dec(&conf->nr_waiting[idx]);
1137	spin_unlock_irq(&conf->resync_lock);
1138	return ret;
1139}
1140
1141static bool wait_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait)
1142{
1143	int idx = sector_to_idx(sector_nr);
1144
1145	return _wait_barrier(conf, idx, nowait);
1146}
1147
1148static void _allow_barrier(struct r1conf *conf, int idx)
1149{
1150	atomic_dec(&conf->nr_pending[idx]);
1151	wake_up_barrier(conf);
1152}
1153
1154static void allow_barrier(struct r1conf *conf, sector_t sector_nr)
1155{
1156	int idx = sector_to_idx(sector_nr);
1157
1158	_allow_barrier(conf, idx);
1159}
1160
1161/* conf->resync_lock should be held */
1162static int get_unqueued_pending(struct r1conf *conf)
1163{
1164	int idx, ret;
1165
1166	ret = atomic_read(&conf->nr_sync_pending);
1167	for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++)
1168		ret += atomic_read(&conf->nr_pending[idx]) -
1169			atomic_read(&conf->nr_queued[idx]);
1170
1171	return ret;
1172}
1173
1174static void freeze_array(struct r1conf *conf, int extra)
1175{
1176	/* Stop sync I/O and normal I/O and wait for everything to
1177	 * go quiet.
1178	 * This is called in two situations:
1179	 * 1) management command handlers (reshape, remove disk, quiesce).
1180	 * 2) one normal I/O request failed.
1181
1182	 * After array_frozen is set to 1, new sync IO will be blocked at
1183	 * raise_barrier(), and new normal I/O will blocked at _wait_barrier()
1184	 * or wait_read_barrier(). The flying I/Os will either complete or be
1185	 * queued. When everything goes quite, there are only queued I/Os left.
1186
1187	 * Every flying I/O contributes to a conf->nr_pending[idx], idx is the
1188	 * barrier bucket index which this I/O request hits. When all sync and
1189	 * normal I/O are queued, sum of all conf->nr_pending[] will match sum
1190	 * of all conf->nr_queued[]. But normal I/O failure is an exception,
1191	 * in handle_read_error(), we may call freeze_array() before trying to
1192	 * fix the read error. In this case, the error read I/O is not queued,
1193	 * so get_unqueued_pending() == 1.
1194	 *
1195	 * Therefore before this function returns, we need to wait until
1196	 * get_unqueued_pendings(conf) gets equal to extra. For
1197	 * normal I/O context, extra is 1, in rested situations extra is 0.
1198	 */
1199	spin_lock_irq(&conf->resync_lock);
1200	conf->array_frozen = 1;
1201	mddev_add_trace_msg(conf->mddev, "raid1 wait freeze");
1202	wait_event_lock_irq_cmd(
1203		conf->wait_barrier,
1204		get_unqueued_pending(conf) == extra,
1205		conf->resync_lock,
1206		flush_pending_writes(conf));
1207	spin_unlock_irq(&conf->resync_lock);
1208}
1209static void unfreeze_array(struct r1conf *conf)
1210{
1211	/* reverse the effect of the freeze */
1212	spin_lock_irq(&conf->resync_lock);
1213	conf->array_frozen = 0;
1214	spin_unlock_irq(&conf->resync_lock);
1215	wake_up(&conf->wait_barrier);
1216}
1217
1218static void alloc_behind_master_bio(struct r1bio *r1_bio,
1219					   struct bio *bio)
1220{
1221	int size = bio->bi_iter.bi_size;
1222	unsigned vcnt = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1223	int i = 0;
1224	struct bio *behind_bio = NULL;
1225
1226	behind_bio = bio_alloc_bioset(NULL, vcnt, 0, GFP_NOIO,
1227				      &r1_bio->mddev->bio_set);
 
1228
1229	/* discard op, we don't support writezero/writesame yet */
1230	if (!bio_has_data(bio)) {
1231		behind_bio->bi_iter.bi_size = size;
1232		goto skip_copy;
1233	}
1234
 
 
1235	while (i < vcnt && size) {
1236		struct page *page;
1237		int len = min_t(int, PAGE_SIZE, size);
1238
1239		page = alloc_page(GFP_NOIO);
1240		if (unlikely(!page))
1241			goto free_pages;
1242
1243		if (!bio_add_page(behind_bio, page, len, 0)) {
1244			put_page(page);
1245			goto free_pages;
1246		}
1247
1248		size -= len;
1249		i++;
1250	}
1251
1252	bio_copy_data(behind_bio, bio);
1253skip_copy:
1254	r1_bio->behind_master_bio = behind_bio;
1255	set_bit(R1BIO_BehindIO, &r1_bio->state);
1256
1257	return;
1258
1259free_pages:
1260	pr_debug("%dB behind alloc failed, doing sync I/O\n",
1261		 bio->bi_iter.bi_size);
1262	bio_free_pages(behind_bio);
1263	bio_put(behind_bio);
1264}
1265
 
 
 
 
 
 
1266static void raid1_unplug(struct blk_plug_cb *cb, bool from_schedule)
1267{
1268	struct raid1_plug_cb *plug = container_of(cb, struct raid1_plug_cb,
1269						  cb);
1270	struct mddev *mddev = plug->cb.data;
1271	struct r1conf *conf = mddev->private;
1272	struct bio *bio;
1273
1274	if (from_schedule) {
1275		spin_lock_irq(&conf->device_lock);
1276		bio_list_merge(&conf->pending_bio_list, &plug->pending);
 
1277		spin_unlock_irq(&conf->device_lock);
1278		wake_up_barrier(conf);
1279		md_wakeup_thread(mddev->thread);
1280		kfree(plug);
1281		return;
1282	}
1283
1284	/* we aren't scheduling, so we can do the write-out directly. */
1285	bio = bio_list_get(&plug->pending);
1286	flush_bio_list(conf, bio);
1287	kfree(plug);
1288}
1289
1290static void init_r1bio(struct r1bio *r1_bio, struct mddev *mddev, struct bio *bio)
1291{
1292	r1_bio->master_bio = bio;
1293	r1_bio->sectors = bio_sectors(bio);
1294	r1_bio->state = 0;
1295	r1_bio->mddev = mddev;
1296	r1_bio->sector = bio->bi_iter.bi_sector;
1297}
1298
1299static inline struct r1bio *
1300alloc_r1bio(struct mddev *mddev, struct bio *bio)
1301{
1302	struct r1conf *conf = mddev->private;
1303	struct r1bio *r1_bio;
1304
1305	r1_bio = mempool_alloc(&conf->r1bio_pool, GFP_NOIO);
1306	/* Ensure no bio records IO_BLOCKED */
1307	memset(r1_bio->bios, 0, conf->raid_disks * sizeof(r1_bio->bios[0]));
1308	init_r1bio(r1_bio, mddev, bio);
1309	return r1_bio;
1310}
1311
1312static void raid1_read_request(struct mddev *mddev, struct bio *bio,
1313			       int max_read_sectors, struct r1bio *r1_bio)
1314{
1315	struct r1conf *conf = mddev->private;
1316	struct raid1_info *mirror;
1317	struct bio *read_bio;
1318	const enum req_op op = bio_op(bio);
1319	const blk_opf_t do_sync = bio->bi_opf & REQ_SYNC;
 
1320	int max_sectors;
1321	int rdisk, error;
1322	bool r1bio_existed = !!r1_bio;
 
1323
1324	/*
1325	 * If r1_bio is set, we are blocking the raid1d thread
1326	 * so there is a tiny risk of deadlock.  So ask for
1327	 * emergency memory if needed.
1328	 */
1329	gfp_t gfp = r1_bio ? (GFP_NOIO | __GFP_HIGH) : GFP_NOIO;
1330
 
 
 
 
 
 
 
 
 
 
 
 
1331	/*
1332	 * Still need barrier for READ in case that whole
1333	 * array is frozen.
1334	 */
1335	if (!wait_read_barrier(conf, bio->bi_iter.bi_sector,
1336				bio->bi_opf & REQ_NOWAIT)) {
1337		bio_wouldblock_error(bio);
1338		return;
1339	}
1340
1341	if (!r1_bio)
1342		r1_bio = alloc_r1bio(mddev, bio);
1343	else
1344		init_r1bio(r1_bio, mddev, bio);
1345	r1_bio->sectors = max_read_sectors;
1346
1347	/*
1348	 * make_request() can abort the operation when read-ahead is being
1349	 * used and no empty request is available.
1350	 */
1351	rdisk = read_balance(conf, r1_bio, &max_sectors);
 
1352	if (rdisk < 0) {
1353		/* couldn't find anywhere to read from */
1354		if (r1bio_existed)
1355			pr_crit_ratelimited("md/raid1:%s: %pg: unrecoverable I/O read error for block %llu\n",
1356					    mdname(mddev),
1357					    conf->mirrors[r1_bio->read_disk].rdev->bdev,
1358					    r1_bio->sector);
 
1359		raid_end_bio_io(r1_bio);
1360		return;
1361	}
1362	mirror = conf->mirrors + rdisk;
1363
1364	if (r1bio_existed)
1365		pr_info_ratelimited("md/raid1:%s: redirecting sector %llu to other mirror: %pg\n",
1366				    mdname(mddev),
1367				    (unsigned long long)r1_bio->sector,
1368				    mirror->rdev->bdev);
1369
1370	if (test_bit(WriteMostly, &mirror->rdev->flags)) {
 
1371		/*
1372		 * Reading from a write-mostly device must take care not to
1373		 * over-take any writes that are 'behind'
1374		 */
1375		mddev_add_trace_msg(mddev, "raid1 wait behind writes");
1376		mddev->bitmap_ops->wait_behind_writes(mddev);
 
1377	}
1378
1379	if (max_sectors < bio_sectors(bio)) {
1380		struct bio *split = bio_split(bio, max_sectors,
1381					      gfp, &conf->bio_split);
1382
1383		if (IS_ERR(split)) {
1384			error = PTR_ERR(split);
1385			goto err_handle;
1386		}
1387		bio_chain(split, bio);
1388		submit_bio_noacct(bio);
1389		bio = split;
1390		r1_bio->master_bio = bio;
1391		r1_bio->sectors = max_sectors;
1392	}
1393
1394	r1_bio->read_disk = rdisk;
1395	if (!r1bio_existed) {
1396		md_account_bio(mddev, &bio);
1397		r1_bio->master_bio = bio;
1398	}
1399	read_bio = bio_alloc_clone(mirror->rdev->bdev, bio, gfp,
1400				   &mddev->bio_set);
1401
1402	r1_bio->bios[rdisk] = read_bio;
1403
1404	read_bio->bi_iter.bi_sector = r1_bio->sector +
1405		mirror->rdev->data_offset;
 
1406	read_bio->bi_end_io = raid1_end_read_request;
1407	read_bio->bi_opf = op | do_sync;
1408	if (test_bit(FailFast, &mirror->rdev->flags) &&
1409	    test_bit(R1BIO_FailFast, &r1_bio->state))
1410	        read_bio->bi_opf |= MD_FAILFAST;
1411	read_bio->bi_private = r1_bio;
1412	mddev_trace_remap(mddev, read_bio, r1_bio->sector);
1413	submit_bio_noacct(read_bio);
1414	return;
1415
1416err_handle:
1417	atomic_dec(&mirror->rdev->nr_pending);
1418	bio->bi_status = errno_to_blk_status(error);
1419	set_bit(R1BIO_Uptodate, &r1_bio->state);
1420	raid_end_bio_io(r1_bio);
1421}
1422
1423static bool wait_blocked_rdev(struct mddev *mddev, struct bio *bio)
1424{
1425	struct r1conf *conf = mddev->private;
1426	int disks = conf->raid_disks * 2;
1427	int i;
1428
1429retry:
1430	for (i = 0; i < disks; i++) {
1431		struct md_rdev *rdev = conf->mirrors[i].rdev;
1432
1433		if (!rdev)
1434			continue;
1435
1436		/* don't write here until the bad block is acknowledged */
1437		if (test_bit(WriteErrorSeen, &rdev->flags) &&
1438		    rdev_has_badblock(rdev, bio->bi_iter.bi_sector,
1439				      bio_sectors(bio)) < 0)
1440			set_bit(BlockedBadBlocks, &rdev->flags);
1441
1442		if (rdev_blocked(rdev)) {
1443			if (bio->bi_opf & REQ_NOWAIT)
1444				return false;
1445
1446			mddev_add_trace_msg(rdev->mddev, "raid1 wait rdev %d blocked",
1447					    rdev->raid_disk);
1448			atomic_inc(&rdev->nr_pending);
1449			md_wait_for_blocked_rdev(rdev, rdev->mddev);
1450			goto retry;
1451		}
1452	}
1453
1454	return true;
1455}
1456
1457static void raid1_write_request(struct mddev *mddev, struct bio *bio,
1458				int max_write_sectors)
1459{
1460	struct r1conf *conf = mddev->private;
1461	struct r1bio *r1_bio;
1462	int i, disks, k, error;
 
1463	unsigned long flags;
 
 
 
1464	int first_clone;
1465	int max_sectors;
1466	bool write_behind = false;
1467	bool is_discard = (bio_op(bio) == REQ_OP_DISCARD);
1468
1469	if (mddev_is_clustered(mddev) &&
1470	     md_cluster_ops->area_resyncing(mddev, WRITE,
1471		     bio->bi_iter.bi_sector, bio_end_sector(bio))) {
1472
1473		DEFINE_WAIT(w);
1474		if (bio->bi_opf & REQ_NOWAIT) {
1475			bio_wouldblock_error(bio);
1476			return;
1477		}
1478		for (;;) {
1479			prepare_to_wait(&conf->wait_barrier,
1480					&w, TASK_IDLE);
1481			if (!md_cluster_ops->area_resyncing(mddev, WRITE,
1482							bio->bi_iter.bi_sector,
1483							bio_end_sector(bio)))
1484				break;
1485			schedule();
1486		}
1487		finish_wait(&conf->wait_barrier, &w);
1488	}
1489
1490	/*
1491	 * Register the new request and wait if the reconstruction
1492	 * thread has put up a bar for new requests.
1493	 * Continue immediately if no resync is active currently.
1494	 */
1495	if (!wait_barrier(conf, bio->bi_iter.bi_sector,
1496				bio->bi_opf & REQ_NOWAIT)) {
1497		bio_wouldblock_error(bio);
1498		return;
1499	}
1500
1501	if (!wait_blocked_rdev(mddev, bio)) {
1502		bio_wouldblock_error(bio);
1503		return;
1504	}
1505
1506	r1_bio = alloc_r1bio(mddev, bio);
1507	r1_bio->sectors = max_write_sectors;
1508
 
 
 
 
 
 
1509	/* first select target devices under rcu_lock and
1510	 * inc refcount on their rdev.  Record them by setting
1511	 * bios[x] to bio
1512	 * If there are known/acknowledged bad blocks on any device on
1513	 * which we have seen a write error, we want to avoid writing those
1514	 * blocks.
1515	 * This potentially requires several writes to write around
1516	 * the bad blocks.  Each set of writes gets it's own r1bio
1517	 * with a set of bios attached.
1518	 */
1519
1520	disks = conf->raid_disks * 2;
 
 
 
1521	max_sectors = r1_bio->sectors;
1522	for (i = 0;  i < disks; i++) {
1523		struct md_rdev *rdev = conf->mirrors[i].rdev;
1524
1525		/*
1526		 * The write-behind io is only attempted on drives marked as
1527		 * write-mostly, which means we could allocate write behind
1528		 * bio later.
1529		 */
1530		if (!is_discard && rdev && test_bit(WriteMostly, &rdev->flags))
1531			write_behind = true;
1532
1533		r1_bio->bios[i] = NULL;
1534		if (!rdev || test_bit(Faulty, &rdev->flags))
 
 
1535			continue;
 
1536
1537		atomic_inc(&rdev->nr_pending);
1538		if (test_bit(WriteErrorSeen, &rdev->flags)) {
1539			sector_t first_bad;
1540			int bad_sectors;
1541			int is_bad;
1542
1543			is_bad = is_badblock(rdev, r1_bio->sector, max_sectors,
1544					     &first_bad, &bad_sectors);
 
 
 
 
 
 
 
1545			if (is_bad && first_bad <= r1_bio->sector) {
1546				/* Cannot write here at all */
1547				bad_sectors -= (r1_bio->sector - first_bad);
1548				if (bad_sectors < max_sectors)
1549					/* mustn't write more than bad_sectors
1550					 * to other devices yet
1551					 */
1552					max_sectors = bad_sectors;
1553				rdev_dec_pending(rdev, mddev);
 
 
 
 
 
 
 
 
 
 
1554				continue;
1555			}
1556			if (is_bad) {
1557				int good_sectors;
1558
1559				/*
1560				 * We cannot atomically write this, so just
1561				 * error in that case. It could be possible to
1562				 * atomically write other mirrors, but the
1563				 * complexity of supporting that is not worth
1564				 * the benefit.
1565				 */
1566				if (bio->bi_opf & REQ_ATOMIC) {
1567					error = -EIO;
1568					goto err_handle;
1569				}
1570
1571				good_sectors = first_bad - r1_bio->sector;
1572				if (good_sectors < max_sectors)
1573					max_sectors = good_sectors;
1574			}
1575		}
1576		r1_bio->bios[i] = bio;
1577	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1578
1579	/*
1580	 * When using a bitmap, we may call alloc_behind_master_bio below.
1581	 * alloc_behind_master_bio allocates a copy of the data payload a page
1582	 * at a time and thus needs a new bio that can fit the whole payload
1583	 * this bio in page sized chunks.
1584	 */
1585	if (write_behind && mddev->bitmap)
1586		max_sectors = min_t(int, max_sectors,
1587				    BIO_MAX_VECS * (PAGE_SIZE >> 9));
1588	if (max_sectors < bio_sectors(bio)) {
1589		struct bio *split = bio_split(bio, max_sectors,
1590					      GFP_NOIO, &conf->bio_split);
1591
1592		if (IS_ERR(split)) {
1593			error = PTR_ERR(split);
1594			goto err_handle;
1595		}
1596		bio_chain(split, bio);
1597		submit_bio_noacct(bio);
1598		bio = split;
1599		r1_bio->master_bio = bio;
1600		r1_bio->sectors = max_sectors;
1601	}
1602
1603	md_account_bio(mddev, &bio);
1604	r1_bio->master_bio = bio;
1605	atomic_set(&r1_bio->remaining, 1);
1606	atomic_set(&r1_bio->behind_remaining, 0);
1607
1608	first_clone = 1;
1609
1610	for (i = 0; i < disks; i++) {
1611		struct bio *mbio = NULL;
1612		struct md_rdev *rdev = conf->mirrors[i].rdev;
1613		if (!r1_bio->bios[i])
1614			continue;
1615
1616		if (first_clone) {
1617			unsigned long max_write_behind =
1618				mddev->bitmap_info.max_write_behind;
1619			struct md_bitmap_stats stats;
1620			int err;
1621
1622			/* do behind I/O ?
1623			 * Not if there are too many, or cannot
1624			 * allocate memory, or a reader on WriteMostly
1625			 * is waiting for behind writes to flush */
1626			err = mddev->bitmap_ops->get_stats(mddev->bitmap, &stats);
1627			if (!err && write_behind && !stats.behind_wait &&
1628			    stats.behind_writes < max_write_behind)
 
1629				alloc_behind_master_bio(r1_bio, bio);
 
1630
1631			if (test_bit(R1BIO_BehindIO, &r1_bio->state))
1632				mddev->bitmap_ops->start_behind_write(mddev);
1633			first_clone = 0;
1634		}
1635
 
 
 
 
 
 
1636		if (r1_bio->behind_master_bio) {
1637			mbio = bio_alloc_clone(rdev->bdev,
1638					       r1_bio->behind_master_bio,
1639					       GFP_NOIO, &mddev->bio_set);
1640			if (test_bit(CollisionCheck, &rdev->flags))
1641				wait_for_serialization(rdev, r1_bio);
1642			if (test_bit(WriteMostly, &rdev->flags))
1643				atomic_inc(&r1_bio->behind_remaining);
1644		} else {
1645			mbio = bio_alloc_clone(rdev->bdev, bio, GFP_NOIO,
1646					       &mddev->bio_set);
1647
1648			if (mddev->serialize_policy)
1649				wait_for_serialization(rdev, r1_bio);
1650		}
1651
1652		r1_bio->bios[i] = mbio;
1653
1654		mbio->bi_iter.bi_sector	= (r1_bio->sector + rdev->data_offset);
 
 
1655		mbio->bi_end_io	= raid1_end_write_request;
1656		mbio->bi_opf = bio_op(bio) |
1657			(bio->bi_opf & (REQ_SYNC | REQ_FUA | REQ_ATOMIC));
1658		if (test_bit(FailFast, &rdev->flags) &&
1659		    !test_bit(WriteMostly, &rdev->flags) &&
1660		    conf->raid_disks - mddev->degraded > 1)
1661			mbio->bi_opf |= MD_FAILFAST;
1662		mbio->bi_private = r1_bio;
1663
1664		atomic_inc(&r1_bio->remaining);
1665		mddev_trace_remap(mddev, mbio, r1_bio->sector);
 
 
 
 
1666		/* flush_pending_writes() needs access to the rdev so...*/
1667		mbio->bi_bdev = (void *)rdev;
1668		if (!raid1_add_bio_to_plug(mddev, mbio, raid1_unplug, disks)) {
 
 
 
 
 
 
 
 
 
1669			spin_lock_irqsave(&conf->device_lock, flags);
1670			bio_list_add(&conf->pending_bio_list, mbio);
 
1671			spin_unlock_irqrestore(&conf->device_lock, flags);
1672			md_wakeup_thread(mddev->thread);
1673		}
1674	}
1675
1676	r1_bio_write_done(r1_bio);
1677
1678	/* In case raid1d snuck in to freeze_array */
1679	wake_up_barrier(conf);
1680	return;
1681err_handle:
1682	for (k = 0; k < i; k++) {
1683		if (r1_bio->bios[k]) {
1684			rdev_dec_pending(conf->mirrors[k].rdev, mddev);
1685			r1_bio->bios[k] = NULL;
1686		}
1687	}
1688
1689	bio->bi_status = errno_to_blk_status(error);
1690	set_bit(R1BIO_Uptodate, &r1_bio->state);
1691	raid_end_bio_io(r1_bio);
1692}
1693
1694static bool raid1_make_request(struct mddev *mddev, struct bio *bio)
1695{
1696	sector_t sectors;
1697
1698	if (unlikely(bio->bi_opf & REQ_PREFLUSH)
1699	    && md_flush_request(mddev, bio))
1700		return true;
1701
1702	/*
1703	 * There is a limit to the maximum size, but
1704	 * the read/write handler might find a lower limit
1705	 * due to bad blocks.  To avoid multiple splits,
1706	 * we pass the maximum number of sectors down
1707	 * and let the lower level perform the split.
1708	 */
1709	sectors = align_to_barrier_unit_end(
1710		bio->bi_iter.bi_sector, bio_sectors(bio));
1711
1712	if (bio_data_dir(bio) == READ)
1713		raid1_read_request(mddev, bio, sectors, NULL);
1714	else {
1715		md_write_start(mddev,bio);
 
1716		raid1_write_request(mddev, bio, sectors);
1717	}
1718	return true;
1719}
1720
1721static void raid1_status(struct seq_file *seq, struct mddev *mddev)
1722{
1723	struct r1conf *conf = mddev->private;
1724	int i;
1725
1726	lockdep_assert_held(&mddev->lock);
1727
1728	seq_printf(seq, " [%d/%d] [", conf->raid_disks,
1729		   conf->raid_disks - mddev->degraded);
 
1730	for (i = 0; i < conf->raid_disks; i++) {
1731		struct md_rdev *rdev = READ_ONCE(conf->mirrors[i].rdev);
1732
1733		seq_printf(seq, "%s",
1734			   rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
1735	}
 
1736	seq_printf(seq, "]");
1737}
1738
1739/**
1740 * raid1_error() - RAID1 error handler.
1741 * @mddev: affected md device.
1742 * @rdev: member device to fail.
1743 *
1744 * The routine acknowledges &rdev failure and determines new @mddev state.
1745 * If it failed, then:
1746 *	- &MD_BROKEN flag is set in &mddev->flags.
1747 *	- recovery is disabled.
1748 * Otherwise, it must be degraded:
1749 *	- recovery is interrupted.
1750 *	- &mddev->degraded is bumped.
1751 *
1752 * @rdev is marked as &Faulty excluding case when array is failed and
1753 * &mddev->fail_last_dev is off.
1754 */
1755static void raid1_error(struct mddev *mddev, struct md_rdev *rdev)
1756{
 
1757	struct r1conf *conf = mddev->private;
1758	unsigned long flags;
1759
 
 
 
 
 
 
1760	spin_lock_irqsave(&conf->device_lock, flags);
1761
1762	if (test_bit(In_sync, &rdev->flags) &&
1763	    (conf->raid_disks - mddev->degraded) == 1) {
1764		set_bit(MD_BROKEN, &mddev->flags);
1765
1766		if (!mddev->fail_last_dev) {
1767			conf->recovery_disabled = mddev->recovery_disabled;
1768			spin_unlock_irqrestore(&conf->device_lock, flags);
1769			return;
1770		}
 
1771	}
1772	set_bit(Blocked, &rdev->flags);
1773	if (test_and_clear_bit(In_sync, &rdev->flags))
1774		mddev->degraded++;
1775	set_bit(Faulty, &rdev->flags);
1776	spin_unlock_irqrestore(&conf->device_lock, flags);
1777	/*
1778	 * if recovery is running, make sure it aborts.
1779	 */
1780	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1781	set_mask_bits(&mddev->sb_flags, 0,
1782		      BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
1783	pr_crit("md/raid1:%s: Disk failure on %pg, disabling device.\n"
1784		"md/raid1:%s: Operation continuing on %d devices.\n",
1785		mdname(mddev), rdev->bdev,
1786		mdname(mddev), conf->raid_disks - mddev->degraded);
1787}
1788
1789static void print_conf(struct r1conf *conf)
1790{
1791	int i;
1792
1793	pr_debug("RAID1 conf printout:\n");
1794	if (!conf) {
1795		pr_debug("(!conf)\n");
1796		return;
1797	}
1798	pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
1799		 conf->raid_disks);
1800
1801	lockdep_assert_held(&conf->mddev->reconfig_mutex);
1802	for (i = 0; i < conf->raid_disks; i++) {
1803		struct md_rdev *rdev = conf->mirrors[i].rdev;
 
1804		if (rdev)
1805			pr_debug(" disk %d, wo:%d, o:%d, dev:%pg\n",
1806				 i, !test_bit(In_sync, &rdev->flags),
1807				 !test_bit(Faulty, &rdev->flags),
1808				 rdev->bdev);
1809	}
 
1810}
1811
1812static void close_sync(struct r1conf *conf)
1813{
1814	int idx;
1815
1816	for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) {
1817		_wait_barrier(conf, idx, false);
1818		_allow_barrier(conf, idx);
1819	}
1820
1821	mempool_exit(&conf->r1buf_pool);
1822}
1823
1824static int raid1_spare_active(struct mddev *mddev)
1825{
1826	int i;
1827	struct r1conf *conf = mddev->private;
1828	int count = 0;
1829	unsigned long flags;
1830
1831	/*
1832	 * Find all failed disks within the RAID1 configuration
1833	 * and mark them readable.
1834	 * Called under mddev lock, so rcu protection not needed.
1835	 * device_lock used to avoid races with raid1_end_read_request
1836	 * which expects 'In_sync' flags and ->degraded to be consistent.
1837	 */
1838	spin_lock_irqsave(&conf->device_lock, flags);
1839	for (i = 0; i < conf->raid_disks; i++) {
1840		struct md_rdev *rdev = conf->mirrors[i].rdev;
1841		struct md_rdev *repl = conf->mirrors[conf->raid_disks + i].rdev;
1842		if (repl
1843		    && !test_bit(Candidate, &repl->flags)
1844		    && repl->recovery_offset == MaxSector
1845		    && !test_bit(Faulty, &repl->flags)
1846		    && !test_and_set_bit(In_sync, &repl->flags)) {
1847			/* replacement has just become active */
1848			if (!rdev ||
1849			    !test_and_clear_bit(In_sync, &rdev->flags))
1850				count++;
1851			if (rdev) {
1852				/* Replaced device not technically
1853				 * faulty, but we need to be sure
1854				 * it gets removed and never re-added
1855				 */
1856				set_bit(Faulty, &rdev->flags);
1857				sysfs_notify_dirent_safe(
1858					rdev->sysfs_state);
1859			}
1860		}
1861		if (rdev
1862		    && rdev->recovery_offset == MaxSector
1863		    && !test_bit(Faulty, &rdev->flags)
1864		    && !test_and_set_bit(In_sync, &rdev->flags)) {
1865			count++;
1866			sysfs_notify_dirent_safe(rdev->sysfs_state);
1867		}
1868	}
1869	mddev->degraded -= count;
1870	spin_unlock_irqrestore(&conf->device_lock, flags);
1871
1872	print_conf(conf);
1873	return count;
1874}
1875
1876static bool raid1_add_conf(struct r1conf *conf, struct md_rdev *rdev, int disk,
1877			   bool replacement)
1878{
1879	struct raid1_info *info = conf->mirrors + disk;
1880
1881	if (replacement)
1882		info += conf->raid_disks;
1883
1884	if (info->rdev)
1885		return false;
1886
1887	if (bdev_nonrot(rdev->bdev)) {
1888		set_bit(Nonrot, &rdev->flags);
1889		WRITE_ONCE(conf->nonrot_disks, conf->nonrot_disks + 1);
1890	}
1891
1892	rdev->raid_disk = disk;
1893	info->head_position = 0;
1894	info->seq_start = MaxSector;
1895	WRITE_ONCE(info->rdev, rdev);
1896
1897	return true;
1898}
1899
1900static bool raid1_remove_conf(struct r1conf *conf, int disk)
1901{
1902	struct raid1_info *info = conf->mirrors + disk;
1903	struct md_rdev *rdev = info->rdev;
1904
1905	if (!rdev || test_bit(In_sync, &rdev->flags) ||
1906	    atomic_read(&rdev->nr_pending))
1907		return false;
1908
1909	/* Only remove non-faulty devices if recovery is not possible. */
1910	if (!test_bit(Faulty, &rdev->flags) &&
1911	    rdev->mddev->recovery_disabled != conf->recovery_disabled &&
1912	    rdev->mddev->degraded < conf->raid_disks)
1913		return false;
1914
1915	if (test_and_clear_bit(Nonrot, &rdev->flags))
1916		WRITE_ONCE(conf->nonrot_disks, conf->nonrot_disks - 1);
1917
1918	WRITE_ONCE(info->rdev, NULL);
1919	return true;
1920}
1921
1922static int raid1_add_disk(struct mddev *mddev, struct md_rdev *rdev)
1923{
1924	struct r1conf *conf = mddev->private;
1925	int err = -EEXIST;
1926	int mirror = 0, repl_slot = -1;
1927	struct raid1_info *p;
1928	int first = 0;
1929	int last = conf->raid_disks - 1;
1930
1931	if (mddev->recovery_disabled == conf->recovery_disabled)
1932		return -EBUSY;
1933
 
 
 
1934	if (rdev->raid_disk >= 0)
1935		first = last = rdev->raid_disk;
1936
1937	/*
1938	 * find the disk ... but prefer rdev->saved_raid_disk
1939	 * if possible.
1940	 */
1941	if (rdev->saved_raid_disk >= 0 &&
1942	    rdev->saved_raid_disk >= first &&
1943	    rdev->saved_raid_disk < conf->raid_disks &&
1944	    conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
1945		first = last = rdev->saved_raid_disk;
1946
1947	for (mirror = first; mirror <= last; mirror++) {
1948		p = conf->mirrors + mirror;
1949		if (!p->rdev) {
1950			err = mddev_stack_new_rdev(mddev, rdev);
1951			if (err)
1952				return err;
1953
1954			raid1_add_conf(conf, rdev, mirror, false);
 
 
1955			/* As all devices are equivalent, we don't need a full recovery
1956			 * if this was recently any drive of the array
1957			 */
1958			if (rdev->saved_raid_disk < 0)
1959				conf->fullsync = 1;
 
1960			break;
1961		}
1962		if (test_bit(WantReplacement, &p->rdev->flags) &&
1963		    p[conf->raid_disks].rdev == NULL && repl_slot < 0)
1964			repl_slot = mirror;
1965	}
1966
1967	if (err && repl_slot >= 0) {
1968		/* Add this device as a replacement */
1969		clear_bit(In_sync, &rdev->flags);
1970		set_bit(Replacement, &rdev->flags);
1971		raid1_add_conf(conf, rdev, repl_slot, true);
1972		err = 0;
1973		conf->fullsync = 1;
1974	}
1975
 
1976	print_conf(conf);
1977	return err;
1978}
1979
1980static int raid1_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
1981{
1982	struct r1conf *conf = mddev->private;
1983	int err = 0;
1984	int number = rdev->raid_disk;
1985	struct raid1_info *p = conf->mirrors + number;
1986
1987	if (unlikely(number >= conf->raid_disks))
1988		goto abort;
1989
1990	if (rdev != p->rdev) {
1991		number += conf->raid_disks;
1992		p = conf->mirrors + number;
1993	}
1994
1995	print_conf(conf);
1996	if (rdev == p->rdev) {
1997		if (!raid1_remove_conf(conf, number)) {
 
1998			err = -EBUSY;
1999			goto abort;
2000		}
2001
2002		if (number < conf->raid_disks &&
2003		    conf->mirrors[conf->raid_disks + number].rdev) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2004			/* We just removed a device that is being replaced.
2005			 * Move down the replacement.  We drain all IO before
2006			 * doing this to avoid confusion.
2007			 */
2008			struct md_rdev *repl =
2009				conf->mirrors[conf->raid_disks + number].rdev;
2010			freeze_array(conf, 0);
2011			if (atomic_read(&repl->nr_pending)) {
2012				/* It means that some queued IO of retry_list
2013				 * hold repl. Thus, we cannot set replacement
2014				 * as NULL, avoiding rdev NULL pointer
2015				 * dereference in sync_request_write and
2016				 * handle_write_finished.
2017				 */
2018				err = -EBUSY;
2019				unfreeze_array(conf);
2020				goto abort;
2021			}
2022			clear_bit(Replacement, &repl->flags);
2023			WRITE_ONCE(p->rdev, repl);
2024			conf->mirrors[conf->raid_disks + number].rdev = NULL;
2025			unfreeze_array(conf);
2026		}
2027
2028		clear_bit(WantReplacement, &rdev->flags);
2029		err = md_integrity_register(mddev);
2030	}
2031abort:
2032
2033	print_conf(conf);
2034	return err;
2035}
2036
2037static void end_sync_read(struct bio *bio)
2038{
2039	struct r1bio *r1_bio = get_resync_r1bio(bio);
2040
2041	update_head_pos(r1_bio->read_disk, r1_bio);
2042
2043	/*
2044	 * we have read a block, now it needs to be re-written,
2045	 * or re-read if the read failed.
2046	 * We don't do much here, just schedule handling by raid1d
2047	 */
2048	if (!bio->bi_status)
2049		set_bit(R1BIO_Uptodate, &r1_bio->state);
2050
2051	if (atomic_dec_and_test(&r1_bio->remaining))
2052		reschedule_retry(r1_bio);
2053}
2054
2055static void abort_sync_write(struct mddev *mddev, struct r1bio *r1_bio)
2056{
2057	sector_t sync_blocks = 0;
2058	sector_t s = r1_bio->sector;
2059	long sectors_to_go = r1_bio->sectors;
2060
2061	/* make sure these bits don't get cleared. */
2062	do {
2063		mddev->bitmap_ops->end_sync(mddev, s, &sync_blocks);
2064		s += sync_blocks;
2065		sectors_to_go -= sync_blocks;
2066	} while (sectors_to_go > 0);
2067}
2068
2069static void put_sync_write_buf(struct r1bio *r1_bio, int uptodate)
2070{
2071	if (atomic_dec_and_test(&r1_bio->remaining)) {
2072		struct mddev *mddev = r1_bio->mddev;
2073		int s = r1_bio->sectors;
2074
2075		if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||
2076		    test_bit(R1BIO_WriteError, &r1_bio->state))
2077			reschedule_retry(r1_bio);
2078		else {
2079			put_buf(r1_bio);
2080			md_done_sync(mddev, s, uptodate);
2081		}
2082	}
2083}
2084
2085static void end_sync_write(struct bio *bio)
2086{
2087	int uptodate = !bio->bi_status;
2088	struct r1bio *r1_bio = get_resync_r1bio(bio);
2089	struct mddev *mddev = r1_bio->mddev;
2090	struct r1conf *conf = mddev->private;
 
 
2091	struct md_rdev *rdev = conf->mirrors[find_bio_disk(r1_bio, bio)].rdev;
2092
2093	if (!uptodate) {
2094		abort_sync_write(mddev, r1_bio);
2095		set_bit(WriteErrorSeen, &rdev->flags);
2096		if (!test_and_set_bit(WantReplacement, &rdev->flags))
2097			set_bit(MD_RECOVERY_NEEDED, &
2098				mddev->recovery);
2099		set_bit(R1BIO_WriteError, &r1_bio->state);
2100	} else if (rdev_has_badblock(rdev, r1_bio->sector, r1_bio->sectors) &&
2101		   !rdev_has_badblock(conf->mirrors[r1_bio->read_disk].rdev,
2102				      r1_bio->sector, r1_bio->sectors)) {
 
 
 
 
2103		set_bit(R1BIO_MadeGood, &r1_bio->state);
2104	}
2105
2106	put_sync_write_buf(r1_bio, uptodate);
2107}
2108
2109static int r1_sync_page_io(struct md_rdev *rdev, sector_t sector,
2110			   int sectors, struct page *page, blk_opf_t rw)
2111{
2112	if (sync_page_io(rdev, sector, sectors << 9, page, rw, false))
2113		/* success */
2114		return 1;
2115	if (rw == REQ_OP_WRITE) {
2116		set_bit(WriteErrorSeen, &rdev->flags);
2117		if (!test_and_set_bit(WantReplacement,
2118				      &rdev->flags))
2119			set_bit(MD_RECOVERY_NEEDED, &
2120				rdev->mddev->recovery);
2121	}
2122	/* need to record an error - either for the block or the device */
2123	if (!rdev_set_badblocks(rdev, sector, sectors, 0))
2124		md_error(rdev->mddev, rdev);
2125	return 0;
2126}
2127
2128static int fix_sync_read_error(struct r1bio *r1_bio)
2129{
2130	/* Try some synchronous reads of other devices to get
2131	 * good data, much like with normal read errors.  Only
2132	 * read into the pages we already have so we don't
2133	 * need to re-issue the read request.
2134	 * We don't need to freeze the array, because being in an
2135	 * active sync request, there is no normal IO, and
2136	 * no overlapping syncs.
2137	 * We don't need to check is_badblock() again as we
2138	 * made sure that anything with a bad block in range
2139	 * will have bi_end_io clear.
2140	 */
2141	struct mddev *mddev = r1_bio->mddev;
2142	struct r1conf *conf = mddev->private;
2143	struct bio *bio = r1_bio->bios[r1_bio->read_disk];
2144	struct page **pages = get_resync_pages(bio)->pages;
2145	sector_t sect = r1_bio->sector;
2146	int sectors = r1_bio->sectors;
2147	int idx = 0;
2148	struct md_rdev *rdev;
2149
2150	rdev = conf->mirrors[r1_bio->read_disk].rdev;
2151	if (test_bit(FailFast, &rdev->flags)) {
2152		/* Don't try recovering from here - just fail it
2153		 * ... unless it is the last working device of course */
2154		md_error(mddev, rdev);
2155		if (test_bit(Faulty, &rdev->flags))
2156			/* Don't try to read from here, but make sure
2157			 * put_buf does it's thing
2158			 */
2159			bio->bi_end_io = end_sync_write;
2160	}
2161
2162	while(sectors) {
2163		int s = sectors;
2164		int d = r1_bio->read_disk;
2165		int success = 0;
2166		int start;
2167
2168		if (s > (PAGE_SIZE>>9))
2169			s = PAGE_SIZE >> 9;
2170		do {
2171			if (r1_bio->bios[d]->bi_end_io == end_sync_read) {
2172				/* No rcu protection needed here devices
2173				 * can only be removed when no resync is
2174				 * active, and resync is currently active
2175				 */
2176				rdev = conf->mirrors[d].rdev;
2177				if (sync_page_io(rdev, sect, s<<9,
2178						 pages[idx],
2179						 REQ_OP_READ, false)) {
2180					success = 1;
2181					break;
2182				}
2183			}
2184			d++;
2185			if (d == conf->raid_disks * 2)
2186				d = 0;
2187		} while (!success && d != r1_bio->read_disk);
2188
2189		if (!success) {
 
2190			int abort = 0;
2191			/* Cannot read from anywhere, this block is lost.
2192			 * Record a bad block on each device.  If that doesn't
2193			 * work just disable and interrupt the recovery.
2194			 * Don't fail devices as that won't really help.
2195			 */
2196			pr_crit_ratelimited("md/raid1:%s: %pg: unrecoverable I/O read error for block %llu\n",
2197					    mdname(mddev), bio->bi_bdev,
2198					    (unsigned long long)r1_bio->sector);
2199			for (d = 0; d < conf->raid_disks * 2; d++) {
2200				rdev = conf->mirrors[d].rdev;
2201				if (!rdev || test_bit(Faulty, &rdev->flags))
2202					continue;
2203				if (!rdev_set_badblocks(rdev, sect, s, 0))
2204					abort = 1;
2205			}
2206			if (abort) {
2207				conf->recovery_disabled =
2208					mddev->recovery_disabled;
2209				set_bit(MD_RECOVERY_INTR, &mddev->recovery);
2210				md_done_sync(mddev, r1_bio->sectors, 0);
2211				put_buf(r1_bio);
2212				return 0;
2213			}
2214			/* Try next page */
2215			sectors -= s;
2216			sect += s;
2217			idx++;
2218			continue;
2219		}
2220
2221		start = d;
2222		/* write it back and re-read */
2223		while (d != r1_bio->read_disk) {
2224			if (d == 0)
2225				d = conf->raid_disks * 2;
2226			d--;
2227			if (r1_bio->bios[d]->bi_end_io != end_sync_read)
2228				continue;
2229			rdev = conf->mirrors[d].rdev;
2230			if (r1_sync_page_io(rdev, sect, s,
2231					    pages[idx],
2232					    REQ_OP_WRITE) == 0) {
2233				r1_bio->bios[d]->bi_end_io = NULL;
2234				rdev_dec_pending(rdev, mddev);
2235			}
2236		}
2237		d = start;
2238		while (d != r1_bio->read_disk) {
2239			if (d == 0)
2240				d = conf->raid_disks * 2;
2241			d--;
2242			if (r1_bio->bios[d]->bi_end_io != end_sync_read)
2243				continue;
2244			rdev = conf->mirrors[d].rdev;
2245			if (r1_sync_page_io(rdev, sect, s,
2246					    pages[idx],
2247					    REQ_OP_READ) != 0)
2248				atomic_add(s, &rdev->corrected_errors);
2249		}
2250		sectors -= s;
2251		sect += s;
2252		idx ++;
2253	}
2254	set_bit(R1BIO_Uptodate, &r1_bio->state);
2255	bio->bi_status = 0;
2256	return 1;
2257}
2258
2259static void process_checks(struct r1bio *r1_bio)
2260{
2261	/* We have read all readable devices.  If we haven't
2262	 * got the block, then there is no hope left.
2263	 * If we have, then we want to do a comparison
2264	 * and skip the write if everything is the same.
2265	 * If any blocks failed to read, then we need to
2266	 * attempt an over-write
2267	 */
2268	struct mddev *mddev = r1_bio->mddev;
2269	struct r1conf *conf = mddev->private;
2270	int primary;
2271	int i;
2272	int vcnt;
2273
2274	/* Fix variable parts of all bios */
2275	vcnt = (r1_bio->sectors + PAGE_SIZE / 512 - 1) >> (PAGE_SHIFT - 9);
2276	for (i = 0; i < conf->raid_disks * 2; i++) {
2277		blk_status_t status;
2278		struct bio *b = r1_bio->bios[i];
2279		struct resync_pages *rp = get_resync_pages(b);
2280		if (b->bi_end_io != end_sync_read)
2281			continue;
2282		/* fixup the bio for reuse, but preserve errno */
2283		status = b->bi_status;
2284		bio_reset(b, conf->mirrors[i].rdev->bdev, REQ_OP_READ);
2285		b->bi_status = status;
2286		b->bi_iter.bi_sector = r1_bio->sector +
2287			conf->mirrors[i].rdev->data_offset;
 
2288		b->bi_end_io = end_sync_read;
2289		rp->raid_bio = r1_bio;
2290		b->bi_private = rp;
2291
2292		/* initialize bvec table again */
2293		md_bio_reset_resync_pages(b, rp, r1_bio->sectors << 9);
2294	}
2295	for (primary = 0; primary < conf->raid_disks * 2; primary++)
2296		if (r1_bio->bios[primary]->bi_end_io == end_sync_read &&
2297		    !r1_bio->bios[primary]->bi_status) {
2298			r1_bio->bios[primary]->bi_end_io = NULL;
2299			rdev_dec_pending(conf->mirrors[primary].rdev, mddev);
2300			break;
2301		}
2302	r1_bio->read_disk = primary;
2303	for (i = 0; i < conf->raid_disks * 2; i++) {
2304		int j = 0;
2305		struct bio *pbio = r1_bio->bios[primary];
2306		struct bio *sbio = r1_bio->bios[i];
2307		blk_status_t status = sbio->bi_status;
2308		struct page **ppages = get_resync_pages(pbio)->pages;
2309		struct page **spages = get_resync_pages(sbio)->pages;
2310		struct bio_vec *bi;
2311		int page_len[RESYNC_PAGES] = { 0 };
2312		struct bvec_iter_all iter_all;
2313
2314		if (sbio->bi_end_io != end_sync_read)
2315			continue;
2316		/* Now we can 'fixup' the error value */
2317		sbio->bi_status = 0;
2318
2319		bio_for_each_segment_all(bi, sbio, iter_all)
2320			page_len[j++] = bi->bv_len;
2321
2322		if (!status) {
2323			for (j = vcnt; j-- ; ) {
2324				if (memcmp(page_address(ppages[j]),
2325					   page_address(spages[j]),
2326					   page_len[j]))
2327					break;
2328			}
2329		} else
2330			j = 0;
2331		if (j >= 0)
2332			atomic64_add(r1_bio->sectors, &mddev->resync_mismatches);
2333		if (j < 0 || (test_bit(MD_RECOVERY_CHECK, &mddev->recovery)
2334			      && !status)) {
2335			/* No need to write to this device. */
2336			sbio->bi_end_io = NULL;
2337			rdev_dec_pending(conf->mirrors[i].rdev, mddev);
2338			continue;
2339		}
2340
2341		bio_copy_data(sbio, pbio);
2342	}
2343}
2344
2345static void sync_request_write(struct mddev *mddev, struct r1bio *r1_bio)
2346{
2347	struct r1conf *conf = mddev->private;
2348	int i;
2349	int disks = conf->raid_disks * 2;
2350	struct bio *wbio;
2351
2352	if (!test_bit(R1BIO_Uptodate, &r1_bio->state))
2353		/* ouch - failed to read all of that. */
2354		if (!fix_sync_read_error(r1_bio))
2355			return;
2356
2357	if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))
2358		process_checks(r1_bio);
2359
2360	/*
2361	 * schedule writes
2362	 */
2363	atomic_set(&r1_bio->remaining, 1);
2364	for (i = 0; i < disks ; i++) {
2365		wbio = r1_bio->bios[i];
2366		if (wbio->bi_end_io == NULL ||
2367		    (wbio->bi_end_io == end_sync_read &&
2368		     (i == r1_bio->read_disk ||
2369		      !test_bit(MD_RECOVERY_SYNC, &mddev->recovery))))
2370			continue;
2371		if (test_bit(Faulty, &conf->mirrors[i].rdev->flags)) {
2372			abort_sync_write(mddev, r1_bio);
2373			continue;
2374		}
2375
2376		wbio->bi_opf = REQ_OP_WRITE;
2377		if (test_bit(FailFast, &conf->mirrors[i].rdev->flags))
2378			wbio->bi_opf |= MD_FAILFAST;
2379
2380		wbio->bi_end_io = end_sync_write;
2381		atomic_inc(&r1_bio->remaining);
2382		md_sync_acct(conf->mirrors[i].rdev->bdev, bio_sectors(wbio));
2383
2384		submit_bio_noacct(wbio);
2385	}
2386
2387	put_sync_write_buf(r1_bio, 1);
2388}
2389
2390/*
2391 * This is a kernel thread which:
2392 *
2393 *	1.	Retries failed read operations on working mirrors.
2394 *	2.	Updates the raid superblock when problems encounter.
2395 *	3.	Performs writes following reads for array synchronising.
2396 */
2397
2398static void fix_read_error(struct r1conf *conf, struct r1bio *r1_bio)
 
2399{
2400	sector_t sect = r1_bio->sector;
2401	int sectors = r1_bio->sectors;
2402	int read_disk = r1_bio->read_disk;
2403	struct mddev *mddev = conf->mddev;
2404	struct md_rdev *rdev = conf->mirrors[read_disk].rdev;
2405
2406	if (exceed_read_errors(mddev, rdev)) {
2407		r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED;
2408		return;
2409	}
2410
2411	while(sectors) {
2412		int s = sectors;
2413		int d = read_disk;
2414		int success = 0;
2415		int start;
 
2416
2417		if (s > (PAGE_SIZE>>9))
2418			s = PAGE_SIZE >> 9;
2419
2420		do {
2421			rdev = conf->mirrors[d].rdev;
 
 
 
 
2422			if (rdev &&
2423			    (test_bit(In_sync, &rdev->flags) ||
2424			     (!test_bit(Faulty, &rdev->flags) &&
2425			      rdev->recovery_offset >= sect + s)) &&
2426			    rdev_has_badblock(rdev, sect, s) == 0) {
 
2427				atomic_inc(&rdev->nr_pending);
 
2428				if (sync_page_io(rdev, sect, s<<9,
2429					 conf->tmppage, REQ_OP_READ, false))
2430					success = 1;
2431				rdev_dec_pending(rdev, mddev);
2432				if (success)
2433					break;
2434			}
2435
2436			d++;
2437			if (d == conf->raid_disks * 2)
2438				d = 0;
2439		} while (d != read_disk);
2440
2441		if (!success) {
2442			/* Cannot read from anywhere - mark it bad */
2443			struct md_rdev *rdev = conf->mirrors[read_disk].rdev;
2444			if (!rdev_set_badblocks(rdev, sect, s, 0))
2445				md_error(mddev, rdev);
2446			break;
2447		}
2448		/* write it back and re-read */
2449		start = d;
2450		while (d != read_disk) {
2451			if (d==0)
2452				d = conf->raid_disks * 2;
2453			d--;
2454			rdev = conf->mirrors[d].rdev;
 
2455			if (rdev &&
2456			    !test_bit(Faulty, &rdev->flags)) {
2457				atomic_inc(&rdev->nr_pending);
 
2458				r1_sync_page_io(rdev, sect, s,
2459						conf->tmppage, REQ_OP_WRITE);
2460				rdev_dec_pending(rdev, mddev);
2461			}
 
2462		}
2463		d = start;
2464		while (d != read_disk) {
 
2465			if (d==0)
2466				d = conf->raid_disks * 2;
2467			d--;
2468			rdev = conf->mirrors[d].rdev;
 
2469			if (rdev &&
2470			    !test_bit(Faulty, &rdev->flags)) {
2471				atomic_inc(&rdev->nr_pending);
 
2472				if (r1_sync_page_io(rdev, sect, s,
2473						conf->tmppage, REQ_OP_READ)) {
2474					atomic_add(s, &rdev->corrected_errors);
2475					pr_info("md/raid1:%s: read error corrected (%d sectors at %llu on %pg)\n",
2476						mdname(mddev), s,
2477						(unsigned long long)(sect +
2478								     rdev->data_offset),
2479						rdev->bdev);
2480				}
2481				rdev_dec_pending(rdev, mddev);
2482			}
 
2483		}
2484		sectors -= s;
2485		sect += s;
2486	}
2487}
2488
2489static int narrow_write_error(struct r1bio *r1_bio, int i)
2490{
2491	struct mddev *mddev = r1_bio->mddev;
2492	struct r1conf *conf = mddev->private;
2493	struct md_rdev *rdev = conf->mirrors[i].rdev;
2494
2495	/* bio has the data to be written to device 'i' where
2496	 * we just recently had a write error.
2497	 * We repeatedly clone the bio and trim down to one block,
2498	 * then try the write.  Where the write fails we record
2499	 * a bad block.
2500	 * It is conceivable that the bio doesn't exactly align with
2501	 * blocks.  We must handle this somehow.
2502	 *
2503	 * We currently own a reference on the rdev.
2504	 */
2505
2506	int block_sectors;
2507	sector_t sector;
2508	int sectors;
2509	int sect_to_write = r1_bio->sectors;
2510	int ok = 1;
2511
2512	if (rdev->badblocks.shift < 0)
2513		return 0;
2514
2515	block_sectors = roundup(1 << rdev->badblocks.shift,
2516				bdev_logical_block_size(rdev->bdev) >> 9);
2517	sector = r1_bio->sector;
2518	sectors = ((sector + block_sectors)
2519		   & ~(sector_t)(block_sectors - 1))
2520		- sector;
2521
2522	while (sect_to_write) {
2523		struct bio *wbio;
2524		if (sectors > sect_to_write)
2525			sectors = sect_to_write;
2526		/* Write at 'sector' for 'sectors'*/
2527
2528		if (test_bit(R1BIO_BehindIO, &r1_bio->state)) {
2529			wbio = bio_alloc_clone(rdev->bdev,
2530					       r1_bio->behind_master_bio,
2531					       GFP_NOIO, &mddev->bio_set);
2532		} else {
2533			wbio = bio_alloc_clone(rdev->bdev, r1_bio->master_bio,
2534					       GFP_NOIO, &mddev->bio_set);
2535		}
2536
2537		wbio->bi_opf = REQ_OP_WRITE;
2538		wbio->bi_iter.bi_sector = r1_bio->sector;
2539		wbio->bi_iter.bi_size = r1_bio->sectors << 9;
2540
2541		bio_trim(wbio, sector - r1_bio->sector, sectors);
2542		wbio->bi_iter.bi_sector += rdev->data_offset;
 
2543
2544		if (submit_bio_wait(wbio) < 0)
2545			/* failure! */
2546			ok = rdev_set_badblocks(rdev, sector,
2547						sectors, 0)
2548				&& ok;
2549
2550		bio_put(wbio);
2551		sect_to_write -= sectors;
2552		sector += sectors;
2553		sectors = block_sectors;
2554	}
2555	return ok;
2556}
2557
2558static void handle_sync_write_finished(struct r1conf *conf, struct r1bio *r1_bio)
2559{
2560	int m;
2561	int s = r1_bio->sectors;
2562	for (m = 0; m < conf->raid_disks * 2 ; m++) {
2563		struct md_rdev *rdev = conf->mirrors[m].rdev;
2564		struct bio *bio = r1_bio->bios[m];
2565		if (bio->bi_end_io == NULL)
2566			continue;
2567		if (!bio->bi_status &&
2568		    test_bit(R1BIO_MadeGood, &r1_bio->state)) {
2569			rdev_clear_badblocks(rdev, r1_bio->sector, s, 0);
2570		}
2571		if (bio->bi_status &&
2572		    test_bit(R1BIO_WriteError, &r1_bio->state)) {
2573			if (!rdev_set_badblocks(rdev, r1_bio->sector, s, 0))
2574				md_error(conf->mddev, rdev);
2575		}
2576	}
2577	put_buf(r1_bio);
2578	md_done_sync(conf->mddev, s, 1);
2579}
2580
2581static void handle_write_finished(struct r1conf *conf, struct r1bio *r1_bio)
2582{
2583	int m, idx;
2584	bool fail = false;
2585
2586	for (m = 0; m < conf->raid_disks * 2 ; m++)
2587		if (r1_bio->bios[m] == IO_MADE_GOOD) {
2588			struct md_rdev *rdev = conf->mirrors[m].rdev;
2589			rdev_clear_badblocks(rdev,
2590					     r1_bio->sector,
2591					     r1_bio->sectors, 0);
2592			rdev_dec_pending(rdev, conf->mddev);
2593		} else if (r1_bio->bios[m] != NULL) {
2594			/* This drive got a write error.  We need to
2595			 * narrow down and record precise write
2596			 * errors.
2597			 */
2598			fail = true;
2599			if (!narrow_write_error(r1_bio, m))
2600				md_error(conf->mddev,
2601					 conf->mirrors[m].rdev);
2602				/* an I/O failed, we can't clear the bitmap */
 
 
2603			rdev_dec_pending(conf->mirrors[m].rdev,
2604					 conf->mddev);
2605		}
2606	if (fail) {
2607		spin_lock_irq(&conf->device_lock);
2608		list_add(&r1_bio->retry_list, &conf->bio_end_io_list);
2609		idx = sector_to_idx(r1_bio->sector);
2610		atomic_inc(&conf->nr_queued[idx]);
2611		spin_unlock_irq(&conf->device_lock);
2612		/*
2613		 * In case freeze_array() is waiting for condition
2614		 * get_unqueued_pending() == extra to be true.
2615		 */
2616		wake_up(&conf->wait_barrier);
2617		md_wakeup_thread(conf->mddev->thread);
2618	} else {
2619		if (test_bit(R1BIO_WriteError, &r1_bio->state))
2620			close_write(r1_bio);
2621		raid_end_bio_io(r1_bio);
2622	}
2623}
2624
2625static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio)
2626{
2627	struct mddev *mddev = conf->mddev;
2628	struct bio *bio;
2629	struct md_rdev *rdev;
2630	sector_t sector;
2631
2632	clear_bit(R1BIO_ReadError, &r1_bio->state);
2633	/* we got a read error. Maybe the drive is bad.  Maybe just
2634	 * the block and we can fix it.
2635	 * We freeze all other IO, and try reading the block from
2636	 * other devices.  When we find one, we re-write
2637	 * and check it that fixes the read error.
2638	 * This is all done synchronously while the array is
2639	 * frozen
2640	 */
2641
2642	bio = r1_bio->bios[r1_bio->read_disk];
2643	bio_put(bio);
2644	r1_bio->bios[r1_bio->read_disk] = NULL;
2645
2646	rdev = conf->mirrors[r1_bio->read_disk].rdev;
2647	if (mddev->ro == 0
2648	    && !test_bit(FailFast, &rdev->flags)) {
2649		freeze_array(conf, 1);
2650		fix_read_error(conf, r1_bio);
 
2651		unfreeze_array(conf);
2652	} else if (mddev->ro == 0 && test_bit(FailFast, &rdev->flags)) {
2653		md_error(mddev, rdev);
2654	} else {
2655		r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED;
2656	}
2657
2658	rdev_dec_pending(rdev, conf->mddev);
2659	sector = r1_bio->sector;
2660	bio = r1_bio->master_bio;
2661
2662	/* Reuse the old r1_bio so that the IO_BLOCKED settings are preserved */
2663	r1_bio->state = 0;
2664	raid1_read_request(mddev, bio, r1_bio->sectors, r1_bio);
2665	allow_barrier(conf, sector);
2666}
2667
2668static void raid1d(struct md_thread *thread)
2669{
2670	struct mddev *mddev = thread->mddev;
2671	struct r1bio *r1_bio;
2672	unsigned long flags;
2673	struct r1conf *conf = mddev->private;
2674	struct list_head *head = &conf->retry_list;
2675	struct blk_plug plug;
2676	int idx;
2677
2678	md_check_recovery(mddev);
2679
2680	if (!list_empty_careful(&conf->bio_end_io_list) &&
2681	    !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2682		LIST_HEAD(tmp);
2683		spin_lock_irqsave(&conf->device_lock, flags);
2684		if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags))
2685			list_splice_init(&conf->bio_end_io_list, &tmp);
2686		spin_unlock_irqrestore(&conf->device_lock, flags);
2687		while (!list_empty(&tmp)) {
2688			r1_bio = list_first_entry(&tmp, struct r1bio,
2689						  retry_list);
2690			list_del(&r1_bio->retry_list);
2691			idx = sector_to_idx(r1_bio->sector);
2692			atomic_dec(&conf->nr_queued[idx]);
 
 
2693			if (test_bit(R1BIO_WriteError, &r1_bio->state))
2694				close_write(r1_bio);
2695			raid_end_bio_io(r1_bio);
2696		}
2697	}
2698
2699	blk_start_plug(&plug);
2700	for (;;) {
2701
2702		flush_pending_writes(conf);
2703
2704		spin_lock_irqsave(&conf->device_lock, flags);
2705		if (list_empty(head)) {
2706			spin_unlock_irqrestore(&conf->device_lock, flags);
2707			break;
2708		}
2709		r1_bio = list_entry(head->prev, struct r1bio, retry_list);
2710		list_del(head->prev);
2711		idx = sector_to_idx(r1_bio->sector);
2712		atomic_dec(&conf->nr_queued[idx]);
2713		spin_unlock_irqrestore(&conf->device_lock, flags);
2714
2715		mddev = r1_bio->mddev;
2716		conf = mddev->private;
2717		if (test_bit(R1BIO_IsSync, &r1_bio->state)) {
2718			if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||
2719			    test_bit(R1BIO_WriteError, &r1_bio->state))
2720				handle_sync_write_finished(conf, r1_bio);
2721			else
2722				sync_request_write(mddev, r1_bio);
2723		} else if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||
2724			   test_bit(R1BIO_WriteError, &r1_bio->state))
2725			handle_write_finished(conf, r1_bio);
2726		else if (test_bit(R1BIO_ReadError, &r1_bio->state))
2727			handle_read_error(conf, r1_bio);
2728		else
2729			WARN_ON_ONCE(1);
2730
2731		cond_resched();
2732		if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
2733			md_check_recovery(mddev);
2734	}
2735	blk_finish_plug(&plug);
2736}
2737
2738static int init_resync(struct r1conf *conf)
2739{
2740	int buffs;
2741
2742	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
2743	BUG_ON(mempool_initialized(&conf->r1buf_pool));
2744
2745	return mempool_init(&conf->r1buf_pool, buffs, r1buf_pool_alloc,
2746			    r1buf_pool_free, conf->poolinfo);
2747}
2748
2749static struct r1bio *raid1_alloc_init_r1buf(struct r1conf *conf)
2750{
2751	struct r1bio *r1bio = mempool_alloc(&conf->r1buf_pool, GFP_NOIO);
2752	struct resync_pages *rps;
2753	struct bio *bio;
2754	int i;
2755
2756	for (i = conf->poolinfo->raid_disks; i--; ) {
2757		bio = r1bio->bios[i];
2758		rps = bio->bi_private;
2759		bio_reset(bio, NULL, 0);
2760		bio->bi_private = rps;
2761	}
2762	r1bio->master_bio = NULL;
2763	return r1bio;
2764}
2765
2766/*
2767 * perform a "sync" on one "block"
2768 *
2769 * We need to make sure that no normal I/O request - particularly write
2770 * requests - conflict with active sync requests.
2771 *
2772 * This is achieved by tracking pending requests and a 'barrier' concept
2773 * that can be installed to exclude normal IO requests.
2774 */
2775
2776static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr,
2777				   sector_t max_sector, int *skipped)
2778{
2779	struct r1conf *conf = mddev->private;
2780	struct r1bio *r1_bio;
2781	struct bio *bio;
2782	sector_t nr_sectors;
2783	int disk = -1;
2784	int i;
2785	int wonly = -1;
2786	int write_targets = 0, read_targets = 0;
2787	sector_t sync_blocks;
2788	bool still_degraded = false;
2789	int good_sectors = RESYNC_SECTORS;
2790	int min_bad = 0; /* number of sectors that are bad in all devices */
2791	int idx = sector_to_idx(sector_nr);
2792	int page_idx = 0;
2793
2794	if (!mempool_initialized(&conf->r1buf_pool))
2795		if (init_resync(conf))
2796			return 0;
2797
 
2798	if (sector_nr >= max_sector) {
2799		/* If we aborted, we need to abort the
2800		 * sync on the 'current' bitmap chunk (there will
2801		 * only be one in raid1 resync.
2802		 * We can find the current addess in mddev->curr_resync
2803		 */
2804		if (mddev->curr_resync < max_sector) /* aborted */
2805			mddev->bitmap_ops->end_sync(mddev, mddev->curr_resync,
2806						    &sync_blocks);
2807		else /* completed sync */
2808			conf->fullsync = 0;
2809
2810		mddev->bitmap_ops->close_sync(mddev);
2811		close_sync(conf);
2812
2813		if (mddev_is_clustered(mddev)) {
2814			conf->cluster_sync_low = 0;
2815			conf->cluster_sync_high = 0;
2816		}
2817		return 0;
2818	}
2819
2820	if (mddev->bitmap == NULL &&
2821	    mddev->recovery_cp == MaxSector &&
2822	    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
2823	    conf->fullsync == 0) {
2824		*skipped = 1;
2825		return max_sector - sector_nr;
2826	}
2827	/* before building a request, check if we can skip these blocks..
2828	 * This call the bitmap_start_sync doesn't actually record anything
2829	 */
2830	if (!mddev->bitmap_ops->start_sync(mddev, sector_nr, &sync_blocks, true) &&
2831	    !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
2832		/* We can skip this block, and probably several more */
2833		*skipped = 1;
2834		return sync_blocks;
2835	}
2836
2837	/*
2838	 * If there is non-resync activity waiting for a turn, then let it
2839	 * though before starting on this new sync request.
2840	 */
2841	if (atomic_read(&conf->nr_waiting[idx]))
2842		schedule_timeout_uninterruptible(1);
2843
2844	/* we are incrementing sector_nr below. To be safe, we check against
2845	 * sector_nr + two times RESYNC_SECTORS
2846	 */
2847
2848	mddev->bitmap_ops->cond_end_sync(mddev, sector_nr,
2849		mddev_is_clustered(mddev) &&
2850		(sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high));
2851
2852	if (raise_barrier(conf, sector_nr))
2853		return 0;
2854
2855	r1_bio = raid1_alloc_init_r1buf(conf);
2856
 
2857	/*
2858	 * If we get a correctably read error during resync or recovery,
2859	 * we might want to read from a different device.  So we
2860	 * flag all drives that could conceivably be read from for READ,
2861	 * and any others (which will be non-In_sync devices) for WRITE.
2862	 * If a read fails, we try reading from something else for which READ
2863	 * is OK.
2864	 */
2865
2866	r1_bio->mddev = mddev;
2867	r1_bio->sector = sector_nr;
2868	r1_bio->state = 0;
2869	set_bit(R1BIO_IsSync, &r1_bio->state);
2870	/* make sure good_sectors won't go across barrier unit boundary */
2871	good_sectors = align_to_barrier_unit_end(sector_nr, good_sectors);
2872
2873	for (i = 0; i < conf->raid_disks * 2; i++) {
2874		struct md_rdev *rdev;
2875		bio = r1_bio->bios[i];
2876
2877		rdev = conf->mirrors[i].rdev;
2878		if (rdev == NULL ||
2879		    test_bit(Faulty, &rdev->flags)) {
2880			if (i < conf->raid_disks)
2881				still_degraded = true;
2882		} else if (!test_bit(In_sync, &rdev->flags)) {
2883			bio->bi_opf = REQ_OP_WRITE;
2884			bio->bi_end_io = end_sync_write;
2885			write_targets ++;
2886		} else {
2887			/* may need to read from here */
2888			sector_t first_bad = MaxSector;
2889			int bad_sectors;
2890
2891			if (is_badblock(rdev, sector_nr, good_sectors,
2892					&first_bad, &bad_sectors)) {
2893				if (first_bad > sector_nr)
2894					good_sectors = first_bad - sector_nr;
2895				else {
2896					bad_sectors -= (sector_nr - first_bad);
2897					if (min_bad == 0 ||
2898					    min_bad > bad_sectors)
2899						min_bad = bad_sectors;
2900				}
2901			}
2902			if (sector_nr < first_bad) {
2903				if (test_bit(WriteMostly, &rdev->flags)) {
2904					if (wonly < 0)
2905						wonly = i;
2906				} else {
2907					if (disk < 0)
2908						disk = i;
2909				}
2910				bio->bi_opf = REQ_OP_READ;
2911				bio->bi_end_io = end_sync_read;
2912				read_targets++;
2913			} else if (!test_bit(WriteErrorSeen, &rdev->flags) &&
2914				test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&
2915				!test_bit(MD_RECOVERY_CHECK, &mddev->recovery)) {
2916				/*
2917				 * The device is suitable for reading (InSync),
2918				 * but has bad block(s) here. Let's try to correct them,
2919				 * if we are doing resync or repair. Otherwise, leave
2920				 * this device alone for this sync request.
2921				 */
2922				bio->bi_opf = REQ_OP_WRITE;
2923				bio->bi_end_io = end_sync_write;
2924				write_targets++;
2925			}
2926		}
2927		if (rdev && bio->bi_end_io) {
2928			atomic_inc(&rdev->nr_pending);
2929			bio->bi_iter.bi_sector = sector_nr + rdev->data_offset;
2930			bio_set_dev(bio, rdev->bdev);
2931			if (test_bit(FailFast, &rdev->flags))
2932				bio->bi_opf |= MD_FAILFAST;
2933		}
2934	}
 
2935	if (disk < 0)
2936		disk = wonly;
2937	r1_bio->read_disk = disk;
2938
2939	if (read_targets == 0 && min_bad > 0) {
2940		/* These sectors are bad on all InSync devices, so we
2941		 * need to mark them bad on all write targets
2942		 */
2943		int ok = 1;
2944		for (i = 0 ; i < conf->raid_disks * 2 ; i++)
2945			if (r1_bio->bios[i]->bi_end_io == end_sync_write) {
2946				struct md_rdev *rdev = conf->mirrors[i].rdev;
2947				ok = rdev_set_badblocks(rdev, sector_nr,
2948							min_bad, 0
2949					) && ok;
2950			}
2951		set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2952		*skipped = 1;
2953		put_buf(r1_bio);
2954
2955		if (!ok) {
2956			/* Cannot record the badblocks, so need to
2957			 * abort the resync.
2958			 * If there are multiple read targets, could just
2959			 * fail the really bad ones ???
2960			 */
2961			conf->recovery_disabled = mddev->recovery_disabled;
2962			set_bit(MD_RECOVERY_INTR, &mddev->recovery);
2963			return 0;
2964		} else
2965			return min_bad;
2966
2967	}
2968	if (min_bad > 0 && min_bad < good_sectors) {
2969		/* only resync enough to reach the next bad->good
2970		 * transition */
2971		good_sectors = min_bad;
2972	}
2973
2974	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && read_targets > 0)
2975		/* extra read targets are also write targets */
2976		write_targets += read_targets-1;
2977
2978	if (write_targets == 0 || read_targets == 0) {
2979		/* There is nowhere to write, so all non-sync
2980		 * drives must be failed - so we are finished
2981		 */
2982		sector_t rv;
2983		if (min_bad > 0)
2984			max_sector = sector_nr + min_bad;
2985		rv = max_sector - sector_nr;
2986		*skipped = 1;
2987		put_buf(r1_bio);
2988		return rv;
2989	}
2990
2991	if (max_sector > mddev->resync_max)
2992		max_sector = mddev->resync_max; /* Don't do IO beyond here */
2993	if (max_sector > sector_nr + good_sectors)
2994		max_sector = sector_nr + good_sectors;
2995	nr_sectors = 0;
2996	sync_blocks = 0;
2997	do {
2998		struct page *page;
2999		int len = PAGE_SIZE;
3000		if (sector_nr + (len>>9) > max_sector)
3001			len = (max_sector - sector_nr) << 9;
3002		if (len == 0)
3003			break;
3004		if (sync_blocks == 0) {
3005			if (!mddev->bitmap_ops->start_sync(mddev, sector_nr,
3006						&sync_blocks, still_degraded) &&
3007			    !conf->fullsync &&
3008			    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))
3009				break;
3010			if ((len >> 9) > sync_blocks)
3011				len = sync_blocks<<9;
3012		}
3013
3014		for (i = 0 ; i < conf->raid_disks * 2; i++) {
3015			struct resync_pages *rp;
3016
3017			bio = r1_bio->bios[i];
3018			rp = get_resync_pages(bio);
3019			if (bio->bi_end_io) {
3020				page = resync_fetch_page(rp, page_idx);
3021
3022				/*
3023				 * won't fail because the vec table is big
3024				 * enough to hold all these pages
3025				 */
3026				__bio_add_page(bio, page, len, 0);
3027			}
3028		}
3029		nr_sectors += len>>9;
3030		sector_nr += len>>9;
3031		sync_blocks -= (len>>9);
3032	} while (++page_idx < RESYNC_PAGES);
3033
3034	r1_bio->sectors = nr_sectors;
3035
3036	if (mddev_is_clustered(mddev) &&
3037			conf->cluster_sync_high < sector_nr + nr_sectors) {
3038		conf->cluster_sync_low = mddev->curr_resync_completed;
3039		conf->cluster_sync_high = conf->cluster_sync_low + CLUSTER_RESYNC_WINDOW_SECTORS;
3040		/* Send resync message */
3041		md_cluster_ops->resync_info_update(mddev,
3042				conf->cluster_sync_low,
3043				conf->cluster_sync_high);
3044	}
3045
3046	/* For a user-requested sync, we read all readable devices and do a
3047	 * compare
3048	 */
3049	if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
3050		atomic_set(&r1_bio->remaining, read_targets);
3051		for (i = 0; i < conf->raid_disks * 2 && read_targets; i++) {
3052			bio = r1_bio->bios[i];
3053			if (bio->bi_end_io == end_sync_read) {
3054				read_targets--;
3055				md_sync_acct_bio(bio, nr_sectors);
3056				if (read_targets == 1)
3057					bio->bi_opf &= ~MD_FAILFAST;
3058				submit_bio_noacct(bio);
3059			}
3060		}
3061	} else {
3062		atomic_set(&r1_bio->remaining, 1);
3063		bio = r1_bio->bios[r1_bio->read_disk];
3064		md_sync_acct_bio(bio, nr_sectors);
3065		if (read_targets == 1)
3066			bio->bi_opf &= ~MD_FAILFAST;
3067		submit_bio_noacct(bio);
3068	}
3069	return nr_sectors;
3070}
3071
3072static sector_t raid1_size(struct mddev *mddev, sector_t sectors, int raid_disks)
3073{
3074	if (sectors)
3075		return sectors;
3076
3077	return mddev->dev_sectors;
3078}
3079
3080static struct r1conf *setup_conf(struct mddev *mddev)
3081{
3082	struct r1conf *conf;
3083	int i;
3084	struct raid1_info *disk;
3085	struct md_rdev *rdev;
3086	int err = -ENOMEM;
3087
3088	conf = kzalloc(sizeof(struct r1conf), GFP_KERNEL);
3089	if (!conf)
3090		goto abort;
3091
3092	conf->nr_pending = kcalloc(BARRIER_BUCKETS_NR,
3093				   sizeof(atomic_t), GFP_KERNEL);
3094	if (!conf->nr_pending)
3095		goto abort;
3096
3097	conf->nr_waiting = kcalloc(BARRIER_BUCKETS_NR,
3098				   sizeof(atomic_t), GFP_KERNEL);
3099	if (!conf->nr_waiting)
3100		goto abort;
3101
3102	conf->nr_queued = kcalloc(BARRIER_BUCKETS_NR,
3103				  sizeof(atomic_t), GFP_KERNEL);
3104	if (!conf->nr_queued)
3105		goto abort;
3106
3107	conf->barrier = kcalloc(BARRIER_BUCKETS_NR,
3108				sizeof(atomic_t), GFP_KERNEL);
3109	if (!conf->barrier)
3110		goto abort;
3111
3112	conf->mirrors = kzalloc(array3_size(sizeof(struct raid1_info),
3113					    mddev->raid_disks, 2),
3114				GFP_KERNEL);
3115	if (!conf->mirrors)
3116		goto abort;
3117
3118	conf->tmppage = alloc_page(GFP_KERNEL);
3119	if (!conf->tmppage)
3120		goto abort;
3121
3122	conf->poolinfo = kzalloc(sizeof(*conf->poolinfo), GFP_KERNEL);
3123	if (!conf->poolinfo)
3124		goto abort;
3125	conf->poolinfo->raid_disks = mddev->raid_disks * 2;
3126	err = mempool_init(&conf->r1bio_pool, NR_RAID_BIOS, r1bio_pool_alloc,
3127			   rbio_pool_free, conf->poolinfo);
3128	if (err)
3129		goto abort;
3130
3131	err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0);
3132	if (err)
3133		goto abort;
3134
3135	conf->poolinfo->mddev = mddev;
3136
3137	err = -EINVAL;
3138	spin_lock_init(&conf->device_lock);
3139	conf->raid_disks = mddev->raid_disks;
3140	rdev_for_each(rdev, mddev) {
3141		int disk_idx = rdev->raid_disk;
3142
3143		if (disk_idx >= conf->raid_disks || disk_idx < 0)
3144			continue;
 
 
 
 
3145
3146		if (!raid1_add_conf(conf, rdev, disk_idx,
3147				    test_bit(Replacement, &rdev->flags)))
3148			goto abort;
 
 
 
3149	}
 
3150	conf->mddev = mddev;
3151	INIT_LIST_HEAD(&conf->retry_list);
3152	INIT_LIST_HEAD(&conf->bio_end_io_list);
3153
3154	spin_lock_init(&conf->resync_lock);
3155	init_waitqueue_head(&conf->wait_barrier);
3156
3157	bio_list_init(&conf->pending_bio_list);
 
3158	conf->recovery_disabled = mddev->recovery_disabled - 1;
3159
3160	err = -EIO;
3161	for (i = 0; i < conf->raid_disks * 2; i++) {
3162
3163		disk = conf->mirrors + i;
3164
3165		if (i < conf->raid_disks &&
3166		    disk[conf->raid_disks].rdev) {
3167			/* This slot has a replacement. */
3168			if (!disk->rdev) {
3169				/* No original, just make the replacement
3170				 * a recovering spare
3171				 */
3172				disk->rdev =
3173					disk[conf->raid_disks].rdev;
3174				disk[conf->raid_disks].rdev = NULL;
3175			} else if (!test_bit(In_sync, &disk->rdev->flags))
3176				/* Original is not in_sync - bad */
3177				goto abort;
3178		}
3179
3180		if (!disk->rdev ||
3181		    !test_bit(In_sync, &disk->rdev->flags)) {
3182			disk->head_position = 0;
3183			if (disk->rdev &&
3184			    (disk->rdev->saved_raid_disk < 0))
3185				conf->fullsync = 1;
3186		}
3187	}
3188
3189	err = -ENOMEM;
3190	rcu_assign_pointer(conf->thread,
3191			   md_register_thread(raid1d, mddev, "raid1"));
3192	if (!conf->thread)
3193		goto abort;
3194
3195	return conf;
3196
3197 abort:
3198	if (conf) {
3199		mempool_exit(&conf->r1bio_pool);
3200		kfree(conf->mirrors);
3201		safe_put_page(conf->tmppage);
3202		kfree(conf->poolinfo);
3203		kfree(conf->nr_pending);
3204		kfree(conf->nr_waiting);
3205		kfree(conf->nr_queued);
3206		kfree(conf->barrier);
3207		bioset_exit(&conf->bio_split);
3208		kfree(conf);
3209	}
3210	return ERR_PTR(err);
3211}
3212
3213static int raid1_set_limits(struct mddev *mddev)
3214{
3215	struct queue_limits lim;
3216	int err;
3217
3218	md_init_stacking_limits(&lim);
3219	lim.max_write_zeroes_sectors = 0;
3220	lim.features |= BLK_FEAT_ATOMIC_WRITES_STACKED;
3221	err = mddev_stack_rdev_limits(mddev, &lim, MDDEV_STACK_INTEGRITY);
3222	if (err)
3223		return err;
3224	return queue_limits_set(mddev->gendisk->queue, &lim);
3225}
3226
3227static int raid1_run(struct mddev *mddev)
3228{
3229	struct r1conf *conf;
3230	int i;
 
3231	int ret;
 
3232
3233	if (mddev->level != 1) {
3234		pr_warn("md/raid1:%s: raid level not set to mirroring (%d)\n",
3235			mdname(mddev), mddev->level);
3236		return -EIO;
3237	}
3238	if (mddev->reshape_position != MaxSector) {
3239		pr_warn("md/raid1:%s: reshape_position set but not supported\n",
3240			mdname(mddev));
3241		return -EIO;
3242	}
3243
 
3244	/*
3245	 * copy the already verified devices into our private RAID1
3246	 * bookkeeping area. [whatever we allocate in run(),
3247	 * should be freed in raid1_free()]
3248	 */
3249	if (mddev->private == NULL)
3250		conf = setup_conf(mddev);
3251	else
3252		conf = mddev->private;
3253
3254	if (IS_ERR(conf))
3255		return PTR_ERR(conf);
3256
3257	if (!mddev_is_dm(mddev)) {
3258		ret = raid1_set_limits(mddev);
3259		if (ret)
3260			return ret;
 
 
 
 
 
 
 
 
3261	}
3262
3263	mddev->degraded = 0;
3264	for (i = 0; i < conf->raid_disks; i++)
3265		if (conf->mirrors[i].rdev == NULL ||
3266		    !test_bit(In_sync, &conf->mirrors[i].rdev->flags) ||
3267		    test_bit(Faulty, &conf->mirrors[i].rdev->flags))
3268			mddev->degraded++;
3269	/*
3270	 * RAID1 needs at least one disk in active
3271	 */
3272	if (conf->raid_disks - mddev->degraded < 1) {
3273		md_unregister_thread(mddev, &conf->thread);
3274		return -EINVAL;
3275	}
3276
3277	if (conf->raid_disks - mddev->degraded == 1)
3278		mddev->recovery_cp = MaxSector;
3279
3280	if (mddev->recovery_cp != MaxSector)
3281		pr_info("md/raid1:%s: not clean -- starting background reconstruction\n",
3282			mdname(mddev));
3283	pr_info("md/raid1:%s: active with %d out of %d mirrors\n",
3284		mdname(mddev), mddev->raid_disks - mddev->degraded,
3285		mddev->raid_disks);
3286
3287	/*
3288	 * Ok, everything is just fine now
3289	 */
3290	rcu_assign_pointer(mddev->thread, conf->thread);
3291	rcu_assign_pointer(conf->thread, NULL);
3292	mddev->private = conf;
3293	set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
3294
3295	md_set_array_sectors(mddev, raid1_size(mddev, 0, 0));
3296
 
 
 
 
 
 
 
 
 
3297	ret = md_integrity_register(mddev);
3298	if (ret)
3299		md_unregister_thread(mddev, &mddev->thread);
 
 
 
 
 
 
3300	return ret;
3301}
3302
3303static void raid1_free(struct mddev *mddev, void *priv)
3304{
3305	struct r1conf *conf = priv;
3306
3307	mempool_exit(&conf->r1bio_pool);
3308	kfree(conf->mirrors);
3309	safe_put_page(conf->tmppage);
3310	kfree(conf->poolinfo);
3311	kfree(conf->nr_pending);
3312	kfree(conf->nr_waiting);
3313	kfree(conf->nr_queued);
3314	kfree(conf->barrier);
3315	bioset_exit(&conf->bio_split);
3316	kfree(conf);
3317}
3318
3319static int raid1_resize(struct mddev *mddev, sector_t sectors)
3320{
3321	/* no resync is happening, and there is enough space
3322	 * on all devices, so we can resize.
3323	 * We need to make sure resync covers any new space.
3324	 * If the array is shrinking we should possibly wait until
3325	 * any io in the removed space completes, but it hardly seems
3326	 * worth it.
3327	 */
3328	sector_t newsize = raid1_size(mddev, sectors, 0);
3329	int ret;
3330
3331	if (mddev->external_size &&
3332	    mddev->array_sectors > newsize)
3333		return -EINVAL;
3334
3335	ret = mddev->bitmap_ops->resize(mddev, newsize, 0, false);
3336	if (ret)
3337		return ret;
3338
3339	md_set_array_sectors(mddev, newsize);
3340	if (sectors > mddev->dev_sectors &&
3341	    mddev->recovery_cp > mddev->dev_sectors) {
3342		mddev->recovery_cp = mddev->dev_sectors;
3343		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
3344	}
3345	mddev->dev_sectors = sectors;
3346	mddev->resync_max_sectors = sectors;
3347	return 0;
3348}
3349
3350static int raid1_reshape(struct mddev *mddev)
3351{
3352	/* We need to:
3353	 * 1/ resize the r1bio_pool
3354	 * 2/ resize conf->mirrors
3355	 *
3356	 * We allocate a new r1bio_pool if we can.
3357	 * Then raise a device barrier and wait until all IO stops.
3358	 * Then resize conf->mirrors and swap in the new r1bio pool.
3359	 *
3360	 * At the same time, we "pack" the devices so that all the missing
3361	 * devices have the higher raid_disk numbers.
3362	 */
3363	mempool_t newpool, oldpool;
3364	struct pool_info *newpoolinfo;
3365	struct raid1_info *newmirrors;
3366	struct r1conf *conf = mddev->private;
3367	int cnt, raid_disks;
3368	unsigned long flags;
3369	int d, d2;
3370	int ret;
3371
3372	memset(&newpool, 0, sizeof(newpool));
3373	memset(&oldpool, 0, sizeof(oldpool));
3374
3375	/* Cannot change chunk_size, layout, or level */
3376	if (mddev->chunk_sectors != mddev->new_chunk_sectors ||
3377	    mddev->layout != mddev->new_layout ||
3378	    mddev->level != mddev->new_level) {
3379		mddev->new_chunk_sectors = mddev->chunk_sectors;
3380		mddev->new_layout = mddev->layout;
3381		mddev->new_level = mddev->level;
3382		return -EINVAL;
3383	}
3384
3385	if (!mddev_is_clustered(mddev))
3386		md_allow_write(mddev);
3387
3388	raid_disks = mddev->raid_disks + mddev->delta_disks;
3389
3390	if (raid_disks < conf->raid_disks) {
3391		cnt=0;
3392		for (d= 0; d < conf->raid_disks; d++)
3393			if (conf->mirrors[d].rdev)
3394				cnt++;
3395		if (cnt > raid_disks)
3396			return -EBUSY;
3397	}
3398
3399	newpoolinfo = kmalloc(sizeof(*newpoolinfo), GFP_KERNEL);
3400	if (!newpoolinfo)
3401		return -ENOMEM;
3402	newpoolinfo->mddev = mddev;
3403	newpoolinfo->raid_disks = raid_disks * 2;
3404
3405	ret = mempool_init(&newpool, NR_RAID_BIOS, r1bio_pool_alloc,
3406			   rbio_pool_free, newpoolinfo);
3407	if (ret) {
3408		kfree(newpoolinfo);
3409		return ret;
3410	}
3411	newmirrors = kzalloc(array3_size(sizeof(struct raid1_info),
3412					 raid_disks, 2),
3413			     GFP_KERNEL);
3414	if (!newmirrors) {
3415		kfree(newpoolinfo);
3416		mempool_exit(&newpool);
3417		return -ENOMEM;
3418	}
3419
3420	freeze_array(conf, 0);
3421
3422	/* ok, everything is stopped */
3423	oldpool = conf->r1bio_pool;
3424	conf->r1bio_pool = newpool;
3425
3426	for (d = d2 = 0; d < conf->raid_disks; d++) {
3427		struct md_rdev *rdev = conf->mirrors[d].rdev;
3428		if (rdev && rdev->raid_disk != d2) {
3429			sysfs_unlink_rdev(mddev, rdev);
3430			rdev->raid_disk = d2;
3431			sysfs_unlink_rdev(mddev, rdev);
3432			if (sysfs_link_rdev(mddev, rdev))
3433				pr_warn("md/raid1:%s: cannot register rd%d\n",
3434					mdname(mddev), rdev->raid_disk);
3435		}
3436		if (rdev)
3437			newmirrors[d2++].rdev = rdev;
3438	}
3439	kfree(conf->mirrors);
3440	conf->mirrors = newmirrors;
3441	kfree(conf->poolinfo);
3442	conf->poolinfo = newpoolinfo;
3443
3444	spin_lock_irqsave(&conf->device_lock, flags);
3445	mddev->degraded += (raid_disks - conf->raid_disks);
3446	spin_unlock_irqrestore(&conf->device_lock, flags);
3447	conf->raid_disks = mddev->raid_disks = raid_disks;
3448	mddev->delta_disks = 0;
3449
3450	unfreeze_array(conf);
3451
3452	set_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
3453	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
3454	md_wakeup_thread(mddev->thread);
3455
3456	mempool_exit(&oldpool);
3457	return 0;
3458}
3459
3460static void raid1_quiesce(struct mddev *mddev, int quiesce)
3461{
3462	struct r1conf *conf = mddev->private;
3463
3464	if (quiesce)
3465		freeze_array(conf, 0);
3466	else
3467		unfreeze_array(conf);
3468}
3469
3470static void *raid1_takeover(struct mddev *mddev)
3471{
3472	/* raid1 can take over:
3473	 *  raid5 with 2 devices, any layout or chunk size
3474	 */
3475	if (mddev->level == 5 && mddev->raid_disks == 2) {
3476		struct r1conf *conf;
3477		mddev->new_level = 1;
3478		mddev->new_layout = 0;
3479		mddev->new_chunk_sectors = 0;
3480		conf = setup_conf(mddev);
3481		if (!IS_ERR(conf)) {
3482			/* Array must appear to be quiesced */
3483			conf->array_frozen = 1;
3484			mddev_clear_unsupported_flags(mddev,
3485				UNSUPPORTED_MDDEV_FLAGS);
3486		}
3487		return conf;
3488	}
3489	return ERR_PTR(-EINVAL);
3490}
3491
3492static struct md_personality raid1_personality =
3493{
3494	.name		= "raid1",
3495	.level		= 1,
3496	.owner		= THIS_MODULE,
3497	.make_request	= raid1_make_request,
3498	.run		= raid1_run,
3499	.free		= raid1_free,
3500	.status		= raid1_status,
3501	.error_handler	= raid1_error,
3502	.hot_add_disk	= raid1_add_disk,
3503	.hot_remove_disk= raid1_remove_disk,
3504	.spare_active	= raid1_spare_active,
3505	.sync_request	= raid1_sync_request,
3506	.resize		= raid1_resize,
3507	.size		= raid1_size,
3508	.check_reshape	= raid1_reshape,
3509	.quiesce	= raid1_quiesce,
3510	.takeover	= raid1_takeover,
3511};
3512
3513static int __init raid_init(void)
3514{
3515	return register_md_personality(&raid1_personality);
3516}
3517
3518static void raid_exit(void)
3519{
3520	unregister_md_personality(&raid1_personality);
3521}
3522
3523module_init(raid_init);
3524module_exit(raid_exit);
3525MODULE_LICENSE("GPL");
3526MODULE_DESCRIPTION("RAID1 (mirroring) personality for MD");
3527MODULE_ALIAS("md-personality-3"); /* RAID1 */
3528MODULE_ALIAS("md-raid1");
3529MODULE_ALIAS("md-level-1");
v5.9
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * raid1.c : Multiple Devices driver for Linux
   4 *
   5 * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat
   6 *
   7 * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman
   8 *
   9 * RAID-1 management functions.
  10 *
  11 * Better read-balancing code written by Mika Kuoppala <miku@iki.fi>, 2000
  12 *
  13 * Fixes to reconstruction by Jakob Østergaard" <jakob@ostenfeld.dk>
  14 * Various fixes by Neil Brown <neilb@cse.unsw.edu.au>
  15 *
  16 * Changes by Peter T. Breuer <ptb@it.uc3m.es> 31/1/2003 to support
  17 * bitmapped intelligence in resync:
  18 *
  19 *      - bitmap marked during normal i/o
  20 *      - bitmap used to skip nondirty blocks during sync
  21 *
  22 * Additions to bitmap code, (C) 2003-2004 Paul Clements, SteelEye Technology:
  23 * - persistent bitmap code
  24 */
  25
  26#include <linux/slab.h>
  27#include <linux/delay.h>
  28#include <linux/blkdev.h>
  29#include <linux/module.h>
  30#include <linux/seq_file.h>
  31#include <linux/ratelimit.h>
  32#include <linux/interval_tree_generic.h>
  33
  34#include <trace/events/block.h>
  35
  36#include "md.h"
  37#include "raid1.h"
  38#include "md-bitmap.h"
  39
  40#define UNSUPPORTED_MDDEV_FLAGS		\
  41	((1L << MD_HAS_JOURNAL) |	\
  42	 (1L << MD_JOURNAL_CLEAN) |	\
  43	 (1L << MD_HAS_PPL) |		\
  44	 (1L << MD_HAS_MULTIPLE_PPLS))
  45
  46static void allow_barrier(struct r1conf *conf, sector_t sector_nr);
  47static void lower_barrier(struct r1conf *conf, sector_t sector_nr);
  48
  49#define raid1_log(md, fmt, args...)				\
  50	do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid1 " fmt, ##args); } while (0)
  51
  52#include "raid1-10.c"
  53
  54#define START(node) ((node)->start)
  55#define LAST(node) ((node)->last)
  56INTERVAL_TREE_DEFINE(struct serial_info, node, sector_t, _subtree_last,
  57		     START, LAST, static inline, raid1_rb);
  58
  59static int check_and_add_serial(struct md_rdev *rdev, struct r1bio *r1_bio,
  60				struct serial_info *si, int idx)
  61{
  62	unsigned long flags;
  63	int ret = 0;
  64	sector_t lo = r1_bio->sector;
  65	sector_t hi = lo + r1_bio->sectors;
  66	struct serial_in_rdev *serial = &rdev->serial[idx];
  67
  68	spin_lock_irqsave(&serial->serial_lock, flags);
  69	/* collision happened */
  70	if (raid1_rb_iter_first(&serial->serial_rb, lo, hi))
  71		ret = -EBUSY;
  72	else {
  73		si->start = lo;
  74		si->last = hi;
  75		raid1_rb_insert(si, &serial->serial_rb);
  76	}
  77	spin_unlock_irqrestore(&serial->serial_lock, flags);
  78
  79	return ret;
  80}
  81
  82static void wait_for_serialization(struct md_rdev *rdev, struct r1bio *r1_bio)
  83{
  84	struct mddev *mddev = rdev->mddev;
  85	struct serial_info *si;
  86	int idx = sector_to_idx(r1_bio->sector);
  87	struct serial_in_rdev *serial = &rdev->serial[idx];
  88
  89	if (WARN_ON(!mddev->serial_info_pool))
  90		return;
  91	si = mempool_alloc(mddev->serial_info_pool, GFP_NOIO);
  92	wait_event(serial->serial_io_wait,
  93		   check_and_add_serial(rdev, r1_bio, si, idx) == 0);
  94}
  95
  96static void remove_serial(struct md_rdev *rdev, sector_t lo, sector_t hi)
  97{
  98	struct serial_info *si;
  99	unsigned long flags;
 100	int found = 0;
 101	struct mddev *mddev = rdev->mddev;
 102	int idx = sector_to_idx(lo);
 103	struct serial_in_rdev *serial = &rdev->serial[idx];
 104
 105	spin_lock_irqsave(&serial->serial_lock, flags);
 106	for (si = raid1_rb_iter_first(&serial->serial_rb, lo, hi);
 107	     si; si = raid1_rb_iter_next(si, lo, hi)) {
 108		if (si->start == lo && si->last == hi) {
 109			raid1_rb_remove(si, &serial->serial_rb);
 110			mempool_free(si, mddev->serial_info_pool);
 111			found = 1;
 112			break;
 113		}
 114	}
 115	if (!found)
 116		WARN(1, "The write IO is not recorded for serialization\n");
 117	spin_unlock_irqrestore(&serial->serial_lock, flags);
 118	wake_up(&serial->serial_io_wait);
 119}
 120
 121/*
 122 * for resync bio, r1bio pointer can be retrieved from the per-bio
 123 * 'struct resync_pages'.
 124 */
 125static inline struct r1bio *get_resync_r1bio(struct bio *bio)
 126{
 127	return get_resync_pages(bio)->raid_bio;
 128}
 129
 130static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data)
 131{
 132	struct pool_info *pi = data;
 133	int size = offsetof(struct r1bio, bios[pi->raid_disks]);
 134
 135	/* allocate a r1bio with room for raid_disks entries in the bios array */
 136	return kzalloc(size, gfp_flags);
 137}
 138
 139#define RESYNC_DEPTH 32
 140#define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
 141#define RESYNC_WINDOW (RESYNC_BLOCK_SIZE * RESYNC_DEPTH)
 142#define RESYNC_WINDOW_SECTORS (RESYNC_WINDOW >> 9)
 143#define CLUSTER_RESYNC_WINDOW (16 * RESYNC_WINDOW)
 144#define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9)
 145
 146static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data)
 147{
 148	struct pool_info *pi = data;
 149	struct r1bio *r1_bio;
 150	struct bio *bio;
 151	int need_pages;
 152	int j;
 153	struct resync_pages *rps;
 154
 155	r1_bio = r1bio_pool_alloc(gfp_flags, pi);
 156	if (!r1_bio)
 157		return NULL;
 158
 159	rps = kmalloc_array(pi->raid_disks, sizeof(struct resync_pages),
 160			    gfp_flags);
 161	if (!rps)
 162		goto out_free_r1bio;
 163
 164	/*
 165	 * Allocate bios : 1 for reading, n-1 for writing
 166	 */
 167	for (j = pi->raid_disks ; j-- ; ) {
 168		bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
 169		if (!bio)
 170			goto out_free_bio;
 
 171		r1_bio->bios[j] = bio;
 172	}
 173	/*
 174	 * Allocate RESYNC_PAGES data pages and attach them to
 175	 * the first bio.
 176	 * If this is a user-requested check/repair, allocate
 177	 * RESYNC_PAGES for each bio.
 178	 */
 179	if (test_bit(MD_RECOVERY_REQUESTED, &pi->mddev->recovery))
 180		need_pages = pi->raid_disks;
 181	else
 182		need_pages = 1;
 183	for (j = 0; j < pi->raid_disks; j++) {
 184		struct resync_pages *rp = &rps[j];
 185
 186		bio = r1_bio->bios[j];
 187
 188		if (j < need_pages) {
 189			if (resync_alloc_pages(rp, gfp_flags))
 190				goto out_free_pages;
 191		} else {
 192			memcpy(rp, &rps[0], sizeof(*rp));
 193			resync_get_all_pages(rp);
 194		}
 195
 196		rp->raid_bio = r1_bio;
 197		bio->bi_private = rp;
 198	}
 199
 200	r1_bio->master_bio = NULL;
 201
 202	return r1_bio;
 203
 204out_free_pages:
 205	while (--j >= 0)
 206		resync_free_pages(&rps[j]);
 207
 208out_free_bio:
 209	while (++j < pi->raid_disks)
 210		bio_put(r1_bio->bios[j]);
 
 
 211	kfree(rps);
 212
 213out_free_r1bio:
 214	rbio_pool_free(r1_bio, data);
 215	return NULL;
 216}
 217
 218static void r1buf_pool_free(void *__r1_bio, void *data)
 219{
 220	struct pool_info *pi = data;
 221	int i;
 222	struct r1bio *r1bio = __r1_bio;
 223	struct resync_pages *rp = NULL;
 224
 225	for (i = pi->raid_disks; i--; ) {
 226		rp = get_resync_pages(r1bio->bios[i]);
 227		resync_free_pages(rp);
 228		bio_put(r1bio->bios[i]);
 
 229	}
 230
 231	/* resync pages array stored in the 1st bio's .bi_private */
 232	kfree(rp);
 233
 234	rbio_pool_free(r1bio, data);
 235}
 236
 237static void put_all_bios(struct r1conf *conf, struct r1bio *r1_bio)
 238{
 239	int i;
 240
 241	for (i = 0; i < conf->raid_disks * 2; i++) {
 242		struct bio **bio = r1_bio->bios + i;
 243		if (!BIO_SPECIAL(*bio))
 244			bio_put(*bio);
 245		*bio = NULL;
 246	}
 247}
 248
 249static void free_r1bio(struct r1bio *r1_bio)
 250{
 251	struct r1conf *conf = r1_bio->mddev->private;
 252
 253	put_all_bios(conf, r1_bio);
 254	mempool_free(r1_bio, &conf->r1bio_pool);
 255}
 256
 257static void put_buf(struct r1bio *r1_bio)
 258{
 259	struct r1conf *conf = r1_bio->mddev->private;
 260	sector_t sect = r1_bio->sector;
 261	int i;
 262
 263	for (i = 0; i < conf->raid_disks * 2; i++) {
 264		struct bio *bio = r1_bio->bios[i];
 265		if (bio->bi_end_io)
 266			rdev_dec_pending(conf->mirrors[i].rdev, r1_bio->mddev);
 267	}
 268
 269	mempool_free(r1_bio, &conf->r1buf_pool);
 270
 271	lower_barrier(conf, sect);
 272}
 273
 274static void reschedule_retry(struct r1bio *r1_bio)
 275{
 276	unsigned long flags;
 277	struct mddev *mddev = r1_bio->mddev;
 278	struct r1conf *conf = mddev->private;
 279	int idx;
 280
 281	idx = sector_to_idx(r1_bio->sector);
 282	spin_lock_irqsave(&conf->device_lock, flags);
 283	list_add(&r1_bio->retry_list, &conf->retry_list);
 284	atomic_inc(&conf->nr_queued[idx]);
 285	spin_unlock_irqrestore(&conf->device_lock, flags);
 286
 287	wake_up(&conf->wait_barrier);
 288	md_wakeup_thread(mddev->thread);
 289}
 290
 291/*
 292 * raid_end_bio_io() is called when we have finished servicing a mirrored
 293 * operation and are ready to return a success/failure code to the buffer
 294 * cache layer.
 295 */
 296static void call_bio_endio(struct r1bio *r1_bio)
 297{
 298	struct bio *bio = r1_bio->master_bio;
 299
 300	if (!test_bit(R1BIO_Uptodate, &r1_bio->state))
 301		bio->bi_status = BLK_STS_IOERR;
 302
 303	bio_endio(bio);
 304}
 305
 306static void raid_end_bio_io(struct r1bio *r1_bio)
 307{
 308	struct bio *bio = r1_bio->master_bio;
 309	struct r1conf *conf = r1_bio->mddev->private;
 
 310
 311	/* if nobody has done the final endio yet, do it now */
 312	if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) {
 313		pr_debug("raid1: sync end %s on sectors %llu-%llu\n",
 314			 (bio_data_dir(bio) == WRITE) ? "write" : "read",
 315			 (unsigned long long) bio->bi_iter.bi_sector,
 316			 (unsigned long long) bio_end_sector(bio) - 1);
 317
 318		call_bio_endio(r1_bio);
 319	}
 
 
 320	/*
 321	 * Wake up any possible resync thread that waits for the device
 322	 * to go idle.  All I/Os, even write-behind writes, are done.
 323	 */
 324	allow_barrier(conf, r1_bio->sector);
 325
 326	free_r1bio(r1_bio);
 327}
 328
 329/*
 330 * Update disk head position estimator based on IRQ completion info.
 331 */
 332static inline void update_head_pos(int disk, struct r1bio *r1_bio)
 333{
 334	struct r1conf *conf = r1_bio->mddev->private;
 335
 336	conf->mirrors[disk].head_position =
 337		r1_bio->sector + (r1_bio->sectors);
 338}
 339
 340/*
 341 * Find the disk number which triggered given bio
 342 */
 343static int find_bio_disk(struct r1bio *r1_bio, struct bio *bio)
 344{
 345	int mirror;
 346	struct r1conf *conf = r1_bio->mddev->private;
 347	int raid_disks = conf->raid_disks;
 348
 349	for (mirror = 0; mirror < raid_disks * 2; mirror++)
 350		if (r1_bio->bios[mirror] == bio)
 351			break;
 352
 353	BUG_ON(mirror == raid_disks * 2);
 354	update_head_pos(mirror, r1_bio);
 355
 356	return mirror;
 357}
 358
 359static void raid1_end_read_request(struct bio *bio)
 360{
 361	int uptodate = !bio->bi_status;
 362	struct r1bio *r1_bio = bio->bi_private;
 363	struct r1conf *conf = r1_bio->mddev->private;
 364	struct md_rdev *rdev = conf->mirrors[r1_bio->read_disk].rdev;
 365
 366	/*
 367	 * this branch is our 'one mirror IO has finished' event handler:
 368	 */
 369	update_head_pos(r1_bio->read_disk, r1_bio);
 370
 371	if (uptodate)
 372		set_bit(R1BIO_Uptodate, &r1_bio->state);
 373	else if (test_bit(FailFast, &rdev->flags) &&
 374		 test_bit(R1BIO_FailFast, &r1_bio->state))
 375		/* This was a fail-fast read so we definitely
 376		 * want to retry */
 377		;
 378	else {
 379		/* If all other devices have failed, we want to return
 380		 * the error upwards rather than fail the last device.
 381		 * Here we redefine "uptodate" to mean "Don't want to retry"
 382		 */
 383		unsigned long flags;
 384		spin_lock_irqsave(&conf->device_lock, flags);
 385		if (r1_bio->mddev->degraded == conf->raid_disks ||
 386		    (r1_bio->mddev->degraded == conf->raid_disks-1 &&
 387		     test_bit(In_sync, &rdev->flags)))
 388			uptodate = 1;
 389		spin_unlock_irqrestore(&conf->device_lock, flags);
 390	}
 391
 392	if (uptodate) {
 393		raid_end_bio_io(r1_bio);
 394		rdev_dec_pending(rdev, conf->mddev);
 395	} else {
 396		/*
 397		 * oops, read error:
 398		 */
 399		char b[BDEVNAME_SIZE];
 400		pr_err_ratelimited("md/raid1:%s: %s: rescheduling sector %llu\n",
 401				   mdname(conf->mddev),
 402				   bdevname(rdev->bdev, b),
 403				   (unsigned long long)r1_bio->sector);
 404		set_bit(R1BIO_ReadError, &r1_bio->state);
 405		reschedule_retry(r1_bio);
 406		/* don't drop the reference on read_disk yet */
 407	}
 408}
 409
 410static void close_write(struct r1bio *r1_bio)
 411{
 
 
 412	/* it really is the end of this request */
 413	if (test_bit(R1BIO_BehindIO, &r1_bio->state)) {
 414		bio_free_pages(r1_bio->behind_master_bio);
 415		bio_put(r1_bio->behind_master_bio);
 416		r1_bio->behind_master_bio = NULL;
 417	}
 418	/* clear the bitmap if all writes complete successfully */
 419	md_bitmap_endwrite(r1_bio->mddev->bitmap, r1_bio->sector,
 420			   r1_bio->sectors,
 421			   !test_bit(R1BIO_Degraded, &r1_bio->state),
 422			   test_bit(R1BIO_BehindIO, &r1_bio->state));
 423	md_write_end(r1_bio->mddev);
 424}
 425
 426static void r1_bio_write_done(struct r1bio *r1_bio)
 427{
 428	if (!atomic_dec_and_test(&r1_bio->remaining))
 429		return;
 430
 431	if (test_bit(R1BIO_WriteError, &r1_bio->state))
 432		reschedule_retry(r1_bio);
 433	else {
 434		close_write(r1_bio);
 435		if (test_bit(R1BIO_MadeGood, &r1_bio->state))
 436			reschedule_retry(r1_bio);
 437		else
 438			raid_end_bio_io(r1_bio);
 439	}
 440}
 441
 442static void raid1_end_write_request(struct bio *bio)
 443{
 444	struct r1bio *r1_bio = bio->bi_private;
 445	int behind = test_bit(R1BIO_BehindIO, &r1_bio->state);
 446	struct r1conf *conf = r1_bio->mddev->private;
 447	struct bio *to_put = NULL;
 448	int mirror = find_bio_disk(r1_bio, bio);
 449	struct md_rdev *rdev = conf->mirrors[mirror].rdev;
 450	bool discard_error;
 451	sector_t lo = r1_bio->sector;
 452	sector_t hi = r1_bio->sector + r1_bio->sectors;
 453
 454	discard_error = bio->bi_status && bio_op(bio) == REQ_OP_DISCARD;
 455
 456	/*
 457	 * 'one mirror IO has finished' event handler:
 458	 */
 459	if (bio->bi_status && !discard_error) {
 460		set_bit(WriteErrorSeen,	&rdev->flags);
 461		if (!test_and_set_bit(WantReplacement, &rdev->flags))
 462			set_bit(MD_RECOVERY_NEEDED, &
 463				conf->mddev->recovery);
 464
 465		if (test_bit(FailFast, &rdev->flags) &&
 466		    (bio->bi_opf & MD_FAILFAST) &&
 467		    /* We never try FailFast to WriteMostly devices */
 468		    !test_bit(WriteMostly, &rdev->flags)) {
 469			md_error(r1_bio->mddev, rdev);
 470		}
 471
 472		/*
 473		 * When the device is faulty, it is not necessary to
 474		 * handle write error.
 475		 * For failfast, this is the only remaining device,
 476		 * We need to retry the write without FailFast.
 477		 */
 478		if (!test_bit(Faulty, &rdev->flags))
 479			set_bit(R1BIO_WriteError, &r1_bio->state);
 480		else {
 481			/* Finished with this branch */
 482			r1_bio->bios[mirror] = NULL;
 483			to_put = bio;
 484		}
 485	} else {
 486		/*
 487		 * Set R1BIO_Uptodate in our master bio, so that we
 488		 * will return a good error code for to the higher
 489		 * levels even if IO on some other mirrored buffer
 490		 * fails.
 491		 *
 492		 * The 'master' represents the composite IO operation
 493		 * to user-side. So if something waits for IO, then it
 494		 * will wait for the 'master' bio.
 495		 */
 496		sector_t first_bad;
 497		int bad_sectors;
 498
 499		r1_bio->bios[mirror] = NULL;
 500		to_put = bio;
 501		/*
 502		 * Do not set R1BIO_Uptodate if the current device is
 503		 * rebuilding or Faulty. This is because we cannot use
 504		 * such device for properly reading the data back (we could
 505		 * potentially use it, if the current write would have felt
 506		 * before rdev->recovery_offset, but for simplicity we don't
 507		 * check this here.
 508		 */
 509		if (test_bit(In_sync, &rdev->flags) &&
 510		    !test_bit(Faulty, &rdev->flags))
 511			set_bit(R1BIO_Uptodate, &r1_bio->state);
 512
 513		/* Maybe we can clear some bad blocks. */
 514		if (is_badblock(rdev, r1_bio->sector, r1_bio->sectors,
 515				&first_bad, &bad_sectors) && !discard_error) {
 516			r1_bio->bios[mirror] = IO_MADE_GOOD;
 517			set_bit(R1BIO_MadeGood, &r1_bio->state);
 518		}
 519	}
 520
 521	if (behind) {
 522		if (test_bit(CollisionCheck, &rdev->flags))
 523			remove_serial(rdev, lo, hi);
 524		if (test_bit(WriteMostly, &rdev->flags))
 525			atomic_dec(&r1_bio->behind_remaining);
 526
 527		/*
 528		 * In behind mode, we ACK the master bio once the I/O
 529		 * has safely reached all non-writemostly
 530		 * disks. Setting the Returned bit ensures that this
 531		 * gets done only once -- we don't ever want to return
 532		 * -EIO here, instead we'll wait
 533		 */
 534		if (atomic_read(&r1_bio->behind_remaining) >= (atomic_read(&r1_bio->remaining)-1) &&
 535		    test_bit(R1BIO_Uptodate, &r1_bio->state)) {
 536			/* Maybe we can return now */
 537			if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) {
 538				struct bio *mbio = r1_bio->master_bio;
 539				pr_debug("raid1: behind end write sectors"
 540					 " %llu-%llu\n",
 541					 (unsigned long long) mbio->bi_iter.bi_sector,
 542					 (unsigned long long) bio_end_sector(mbio) - 1);
 543				call_bio_endio(r1_bio);
 544			}
 545		}
 546	} else if (rdev->mddev->serialize_policy)
 547		remove_serial(rdev, lo, hi);
 548	if (r1_bio->bios[mirror] == NULL)
 549		rdev_dec_pending(rdev, conf->mddev);
 550
 551	/*
 552	 * Let's see if all mirrored write operations have finished
 553	 * already.
 554	 */
 555	r1_bio_write_done(r1_bio);
 556
 557	if (to_put)
 558		bio_put(to_put);
 559}
 560
 561static sector_t align_to_barrier_unit_end(sector_t start_sector,
 562					  sector_t sectors)
 563{
 564	sector_t len;
 565
 566	WARN_ON(sectors == 0);
 567	/*
 568	 * len is the number of sectors from start_sector to end of the
 569	 * barrier unit which start_sector belongs to.
 570	 */
 571	len = round_up(start_sector + 1, BARRIER_UNIT_SECTOR_SIZE) -
 572	      start_sector;
 573
 574	if (len > sectors)
 575		len = sectors;
 576
 577	return len;
 578}
 579
 580/*
 581 * This routine returns the disk from which the requested read should
 582 * be done. There is a per-array 'next expected sequential IO' sector
 583 * number - if this matches on the next IO then we use the last disk.
 584 * There is also a per-disk 'last know head position' sector that is
 585 * maintained from IRQ contexts, both the normal and the resync IO
 586 * completion handlers update this position correctly. If there is no
 587 * perfect sequential match then we pick the disk whose head is closest.
 588 *
 589 * If there are 2 mirrors in the same 2 devices, performance degrades
 590 * because position is mirror, not device based.
 591 *
 592 * The rdev for the device selected will have nr_pending incremented.
 593 */
 594static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, int *max_sectors)
 595{
 596	const sector_t this_sector = r1_bio->sector;
 597	int sectors;
 598	int best_good_sectors;
 599	int best_disk, best_dist_disk, best_pending_disk;
 600	int has_nonrot_disk;
 601	int disk;
 602	sector_t best_dist;
 603	unsigned int min_pending;
 604	struct md_rdev *rdev;
 605	int choose_first;
 606	int choose_next_idle;
 607
 608	rcu_read_lock();
 609	/*
 610	 * Check if we can balance. We can balance on the whole
 611	 * device if no resync is going on, or below the resync window.
 612	 * We take the first readable disk when above the resync window.
 613	 */
 614 retry:
 615	sectors = r1_bio->sectors;
 616	best_disk = -1;
 617	best_dist_disk = -1;
 618	best_dist = MaxSector;
 619	best_pending_disk = -1;
 620	min_pending = UINT_MAX;
 621	best_good_sectors = 0;
 622	has_nonrot_disk = 0;
 623	choose_next_idle = 0;
 624	clear_bit(R1BIO_FailFast, &r1_bio->state);
 
 
 
 
 
 
 
 
 
 
 
 625
 626	if ((conf->mddev->recovery_cp < this_sector + sectors) ||
 627	    (mddev_is_clustered(conf->mddev) &&
 628	    md_cluster_ops->area_resyncing(conf->mddev, READ, this_sector,
 629		    this_sector + sectors)))
 630		choose_first = 1;
 631	else
 632		choose_first = 0;
 633
 634	for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) {
 635		sector_t dist;
 636		sector_t first_bad;
 637		int bad_sectors;
 638		unsigned int pending;
 639		bool nonrot;
 
 640
 641		rdev = rcu_dereference(conf->mirrors[disk].rdev);
 642		if (r1_bio->bios[disk] == IO_BLOCKED
 643		    || rdev == NULL
 644		    || test_bit(Faulty, &rdev->flags))
 645			continue;
 646		if (!test_bit(In_sync, &rdev->flags) &&
 647		    rdev->recovery_offset < this_sector + sectors)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 648			continue;
 649		if (test_bit(WriteMostly, &rdev->flags)) {
 650			/* Don't balance among write-mostly, just
 651			 * use the first as a last resort */
 652			if (best_dist_disk < 0) {
 653				if (is_badblock(rdev, this_sector, sectors,
 654						&first_bad, &bad_sectors)) {
 655					if (first_bad <= this_sector)
 656						/* Cannot use this */
 657						continue;
 658					best_good_sectors = first_bad - this_sector;
 659				} else
 660					best_good_sectors = sectors;
 661				best_dist_disk = disk;
 662				best_pending_disk = disk;
 663			}
 664			continue;
 
 
 
 
 
 
 
 
 665		}
 666		/* This is a reasonable device to use.  It might
 667		 * even be best.
 
 
 668		 */
 669		if (is_badblock(rdev, this_sector, sectors,
 670				&first_bad, &bad_sectors)) {
 671			if (best_dist < MaxSector)
 672				/* already have a better device */
 673				continue;
 674			if (first_bad <= this_sector) {
 675				/* cannot read here. If this is the 'primary'
 676				 * device, then we must not read beyond
 677				 * bad_sectors from another device..
 678				 */
 679				bad_sectors -= (this_sector - first_bad);
 680				if (choose_first && sectors > bad_sectors)
 681					sectors = bad_sectors;
 682				if (best_good_sectors > sectors)
 683					best_good_sectors = sectors;
 684
 685			} else {
 686				sector_t good_sectors = first_bad - this_sector;
 687				if (good_sectors > best_good_sectors) {
 688					best_good_sectors = good_sectors;
 689					best_disk = disk;
 690				}
 691				if (choose_first)
 692					break;
 693			}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 694			continue;
 695		} else {
 696			if ((sectors > best_good_sectors) && (best_disk >= 0))
 697				best_disk = -1;
 698			best_good_sectors = sectors;
 699		}
 700
 701		if (best_disk >= 0)
 702			/* At least two disks to choose from so failfast is OK */
 703			set_bit(R1BIO_FailFast, &r1_bio->state);
 704
 705		nonrot = blk_queue_nonrot(bdev_get_queue(rdev->bdev));
 706		has_nonrot_disk |= nonrot;
 707		pending = atomic_read(&rdev->nr_pending);
 708		dist = abs(this_sector - conf->mirrors[disk].head_position);
 709		if (choose_first) {
 710			best_disk = disk;
 711			break;
 712		}
 713		/* Don't change to another disk for sequential reads */
 714		if (conf->mirrors[disk].next_seq_sect == this_sector
 715		    || dist == 0) {
 716			int opt_iosize = bdev_io_opt(rdev->bdev) >> 9;
 717			struct raid1_info *mirror = &conf->mirrors[disk];
 718
 719			best_disk = disk;
 720			/*
 721			 * If buffered sequential IO size exceeds optimal
 722			 * iosize, check if there is idle disk. If yes, choose
 723			 * the idle disk. read_balance could already choose an
 724			 * idle disk before noticing it's a sequential IO in
 725			 * this disk. This doesn't matter because this disk
 726			 * will idle, next time it will be utilized after the
 727			 * first disk has IO size exceeds optimal iosize. In
 728			 * this way, iosize of the first disk will be optimal
 729			 * iosize at least. iosize of the second disk might be
 730			 * small, but not a big deal since when the second disk
 731			 * starts IO, the first disk is likely still busy.
 732			 */
 733			if (nonrot && opt_iosize > 0 &&
 734			    mirror->seq_start != MaxSector &&
 735			    mirror->next_seq_sect > opt_iosize &&
 736			    mirror->next_seq_sect - opt_iosize >=
 737			    mirror->seq_start) {
 738				choose_next_idle = 1;
 739				continue;
 740			}
 741			break;
 742		}
 743
 744		if (choose_next_idle)
 745			continue;
 746
 747		if (min_pending > pending) {
 748			min_pending = pending;
 749			best_pending_disk = disk;
 750		}
 751
 752		if (dist < best_dist) {
 753			best_dist = dist;
 754			best_dist_disk = disk;
 755		}
 756	}
 757
 758	/*
 
 
 
 
 
 
 
 759	 * If all disks are rotational, choose the closest disk. If any disk is
 760	 * non-rotational, choose the disk with less pending request even the
 761	 * disk is rotational, which might/might not be optimal for raids with
 762	 * mixed ratation/non-rotational disks depending on workload.
 763	 */
 764	if (best_disk == -1) {
 765		if (has_nonrot_disk || min_pending == 0)
 766			best_disk = best_pending_disk;
 767		else
 768			best_disk = best_dist_disk;
 769	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 770
 771	if (best_disk >= 0) {
 772		rdev = rcu_dereference(conf->mirrors[best_disk].rdev);
 773		if (!rdev)
 774			goto retry;
 775		atomic_inc(&rdev->nr_pending);
 776		sectors = best_good_sectors;
 777
 778		if (conf->mirrors[best_disk].next_seq_sect != this_sector)
 779			conf->mirrors[best_disk].seq_start = this_sector;
 
 780
 781		conf->mirrors[best_disk].next_seq_sect = this_sector + sectors;
 
 
 
 
 
 782	}
 783	rcu_read_unlock();
 784	*max_sectors = sectors;
 785
 786	return best_disk;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 787}
 788
 789static void flush_bio_list(struct r1conf *conf, struct bio *bio)
 790{
 791	/* flush any pending bitmap writes to disk before proceeding w/ I/O */
 792	md_bitmap_unplug(conf->mddev->bitmap);
 793	wake_up(&conf->wait_barrier);
 794
 795	while (bio) { /* submit pending writes */
 796		struct bio *next = bio->bi_next;
 797		struct md_rdev *rdev = (void *)bio->bi_disk;
 798		bio->bi_next = NULL;
 799		bio_set_dev(bio, rdev->bdev);
 800		if (test_bit(Faulty, &rdev->flags)) {
 801			bio_io_error(bio);
 802		} else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
 803				    !blk_queue_discard(bio->bi_disk->queue)))
 804			/* Just ignore it */
 805			bio_endio(bio);
 806		else
 807			submit_bio_noacct(bio);
 808		bio = next;
 809		cond_resched();
 810	}
 811}
 812
 813static void flush_pending_writes(struct r1conf *conf)
 814{
 815	/* Any writes that have been queued but are awaiting
 816	 * bitmap updates get flushed here.
 817	 */
 818	spin_lock_irq(&conf->device_lock);
 819
 820	if (conf->pending_bio_list.head) {
 821		struct blk_plug plug;
 822		struct bio *bio;
 823
 824		bio = bio_list_get(&conf->pending_bio_list);
 825		conf->pending_count = 0;
 826		spin_unlock_irq(&conf->device_lock);
 827
 828		/*
 829		 * As this is called in a wait_event() loop (see freeze_array),
 830		 * current->state might be TASK_UNINTERRUPTIBLE which will
 831		 * cause a warning when we prepare to wait again.  As it is
 832		 * rare that this path is taken, it is perfectly safe to force
 833		 * us to go around the wait_event() loop again, so the warning
 834		 * is a false-positive.  Silence the warning by resetting
 835		 * thread state
 836		 */
 837		__set_current_state(TASK_RUNNING);
 838		blk_start_plug(&plug);
 839		flush_bio_list(conf, bio);
 840		blk_finish_plug(&plug);
 841	} else
 842		spin_unlock_irq(&conf->device_lock);
 843}
 844
 845/* Barriers....
 846 * Sometimes we need to suspend IO while we do something else,
 847 * either some resync/recovery, or reconfigure the array.
 848 * To do this we raise a 'barrier'.
 849 * The 'barrier' is a counter that can be raised multiple times
 850 * to count how many activities are happening which preclude
 851 * normal IO.
 852 * We can only raise the barrier if there is no pending IO.
 853 * i.e. if nr_pending == 0.
 854 * We choose only to raise the barrier if no-one is waiting for the
 855 * barrier to go down.  This means that as soon as an IO request
 856 * is ready, no other operations which require a barrier will start
 857 * until the IO request has had a chance.
 858 *
 859 * So: regular IO calls 'wait_barrier'.  When that returns there
 860 *    is no backgroup IO happening,  It must arrange to call
 861 *    allow_barrier when it has finished its IO.
 862 * backgroup IO calls must call raise_barrier.  Once that returns
 863 *    there is no normal IO happeing.  It must arrange to call
 864 *    lower_barrier when the particular background IO completes.
 865 *
 866 * If resync/recovery is interrupted, returns -EINTR;
 867 * Otherwise, returns 0.
 868 */
 869static int raise_barrier(struct r1conf *conf, sector_t sector_nr)
 870{
 871	int idx = sector_to_idx(sector_nr);
 872
 873	spin_lock_irq(&conf->resync_lock);
 874
 875	/* Wait until no block IO is waiting */
 876	wait_event_lock_irq(conf->wait_barrier,
 877			    !atomic_read(&conf->nr_waiting[idx]),
 878			    conf->resync_lock);
 879
 880	/* block any new IO from starting */
 881	atomic_inc(&conf->barrier[idx]);
 882	/*
 883	 * In raise_barrier() we firstly increase conf->barrier[idx] then
 884	 * check conf->nr_pending[idx]. In _wait_barrier() we firstly
 885	 * increase conf->nr_pending[idx] then check conf->barrier[idx].
 886	 * A memory barrier here to make sure conf->nr_pending[idx] won't
 887	 * be fetched before conf->barrier[idx] is increased. Otherwise
 888	 * there will be a race between raise_barrier() and _wait_barrier().
 889	 */
 890	smp_mb__after_atomic();
 891
 892	/* For these conditions we must wait:
 893	 * A: while the array is in frozen state
 894	 * B: while conf->nr_pending[idx] is not 0, meaning regular I/O
 895	 *    existing in corresponding I/O barrier bucket.
 896	 * C: while conf->barrier[idx] >= RESYNC_DEPTH, meaning reaches
 897	 *    max resync count which allowed on current I/O barrier bucket.
 898	 */
 899	wait_event_lock_irq(conf->wait_barrier,
 900			    (!conf->array_frozen &&
 901			     !atomic_read(&conf->nr_pending[idx]) &&
 902			     atomic_read(&conf->barrier[idx]) < RESYNC_DEPTH) ||
 903				test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery),
 904			    conf->resync_lock);
 905
 906	if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
 907		atomic_dec(&conf->barrier[idx]);
 908		spin_unlock_irq(&conf->resync_lock);
 909		wake_up(&conf->wait_barrier);
 910		return -EINTR;
 911	}
 912
 913	atomic_inc(&conf->nr_sync_pending);
 914	spin_unlock_irq(&conf->resync_lock);
 915
 916	return 0;
 917}
 918
 919static void lower_barrier(struct r1conf *conf, sector_t sector_nr)
 920{
 921	int idx = sector_to_idx(sector_nr);
 922
 923	BUG_ON(atomic_read(&conf->barrier[idx]) <= 0);
 924
 925	atomic_dec(&conf->barrier[idx]);
 926	atomic_dec(&conf->nr_sync_pending);
 927	wake_up(&conf->wait_barrier);
 928}
 929
 930static void _wait_barrier(struct r1conf *conf, int idx)
 931{
 
 
 932	/*
 933	 * We need to increase conf->nr_pending[idx] very early here,
 934	 * then raise_barrier() can be blocked when it waits for
 935	 * conf->nr_pending[idx] to be 0. Then we can avoid holding
 936	 * conf->resync_lock when there is no barrier raised in same
 937	 * barrier unit bucket. Also if the array is frozen, I/O
 938	 * should be blocked until array is unfrozen.
 939	 */
 940	atomic_inc(&conf->nr_pending[idx]);
 941	/*
 942	 * In _wait_barrier() we firstly increase conf->nr_pending[idx], then
 943	 * check conf->barrier[idx]. In raise_barrier() we firstly increase
 944	 * conf->barrier[idx], then check conf->nr_pending[idx]. A memory
 945	 * barrier is necessary here to make sure conf->barrier[idx] won't be
 946	 * fetched before conf->nr_pending[idx] is increased. Otherwise there
 947	 * will be a race between _wait_barrier() and raise_barrier().
 948	 */
 949	smp_mb__after_atomic();
 950
 951	/*
 952	 * Don't worry about checking two atomic_t variables at same time
 953	 * here. If during we check conf->barrier[idx], the array is
 954	 * frozen (conf->array_frozen is 1), and chonf->barrier[idx] is
 955	 * 0, it is safe to return and make the I/O continue. Because the
 956	 * array is frozen, all I/O returned here will eventually complete
 957	 * or be queued, no race will happen. See code comment in
 958	 * frozen_array().
 959	 */
 960	if (!READ_ONCE(conf->array_frozen) &&
 961	    !atomic_read(&conf->barrier[idx]))
 962		return;
 963
 964	/*
 965	 * After holding conf->resync_lock, conf->nr_pending[idx]
 966	 * should be decreased before waiting for barrier to drop.
 967	 * Otherwise, we may encounter a race condition because
 968	 * raise_barrer() might be waiting for conf->nr_pending[idx]
 969	 * to be 0 at same time.
 970	 */
 971	spin_lock_irq(&conf->resync_lock);
 972	atomic_inc(&conf->nr_waiting[idx]);
 973	atomic_dec(&conf->nr_pending[idx]);
 974	/*
 975	 * In case freeze_array() is waiting for
 976	 * get_unqueued_pending() == extra
 977	 */
 978	wake_up(&conf->wait_barrier);
 979	/* Wait for the barrier in same barrier unit bucket to drop. */
 980	wait_event_lock_irq(conf->wait_barrier,
 981			    !conf->array_frozen &&
 982			     !atomic_read(&conf->barrier[idx]),
 983			    conf->resync_lock);
 984	atomic_inc(&conf->nr_pending[idx]);
 
 
 
 
 
 
 
 985	atomic_dec(&conf->nr_waiting[idx]);
 986	spin_unlock_irq(&conf->resync_lock);
 
 987}
 988
 989static void wait_read_barrier(struct r1conf *conf, sector_t sector_nr)
 990{
 991	int idx = sector_to_idx(sector_nr);
 
 992
 993	/*
 994	 * Very similar to _wait_barrier(). The difference is, for read
 995	 * I/O we don't need wait for sync I/O, but if the whole array
 996	 * is frozen, the read I/O still has to wait until the array is
 997	 * unfrozen. Since there is no ordering requirement with
 998	 * conf->barrier[idx] here, memory barrier is unnecessary as well.
 999	 */
1000	atomic_inc(&conf->nr_pending[idx]);
1001
1002	if (!READ_ONCE(conf->array_frozen))
1003		return;
1004
1005	spin_lock_irq(&conf->resync_lock);
1006	atomic_inc(&conf->nr_waiting[idx]);
1007	atomic_dec(&conf->nr_pending[idx]);
1008	/*
1009	 * In case freeze_array() is waiting for
1010	 * get_unqueued_pending() == extra
1011	 */
1012	wake_up(&conf->wait_barrier);
1013	/* Wait for array to be unfrozen */
1014	wait_event_lock_irq(conf->wait_barrier,
1015			    !conf->array_frozen,
1016			    conf->resync_lock);
1017	atomic_inc(&conf->nr_pending[idx]);
 
 
 
 
 
 
 
 
1018	atomic_dec(&conf->nr_waiting[idx]);
1019	spin_unlock_irq(&conf->resync_lock);
 
1020}
1021
1022static void wait_barrier(struct r1conf *conf, sector_t sector_nr)
1023{
1024	int idx = sector_to_idx(sector_nr);
1025
1026	_wait_barrier(conf, idx);
1027}
1028
1029static void _allow_barrier(struct r1conf *conf, int idx)
1030{
1031	atomic_dec(&conf->nr_pending[idx]);
1032	wake_up(&conf->wait_barrier);
1033}
1034
1035static void allow_barrier(struct r1conf *conf, sector_t sector_nr)
1036{
1037	int idx = sector_to_idx(sector_nr);
1038
1039	_allow_barrier(conf, idx);
1040}
1041
1042/* conf->resync_lock should be held */
1043static int get_unqueued_pending(struct r1conf *conf)
1044{
1045	int idx, ret;
1046
1047	ret = atomic_read(&conf->nr_sync_pending);
1048	for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++)
1049		ret += atomic_read(&conf->nr_pending[idx]) -
1050			atomic_read(&conf->nr_queued[idx]);
1051
1052	return ret;
1053}
1054
1055static void freeze_array(struct r1conf *conf, int extra)
1056{
1057	/* Stop sync I/O and normal I/O and wait for everything to
1058	 * go quiet.
1059	 * This is called in two situations:
1060	 * 1) management command handlers (reshape, remove disk, quiesce).
1061	 * 2) one normal I/O request failed.
1062
1063	 * After array_frozen is set to 1, new sync IO will be blocked at
1064	 * raise_barrier(), and new normal I/O will blocked at _wait_barrier()
1065	 * or wait_read_barrier(). The flying I/Os will either complete or be
1066	 * queued. When everything goes quite, there are only queued I/Os left.
1067
1068	 * Every flying I/O contributes to a conf->nr_pending[idx], idx is the
1069	 * barrier bucket index which this I/O request hits. When all sync and
1070	 * normal I/O are queued, sum of all conf->nr_pending[] will match sum
1071	 * of all conf->nr_queued[]. But normal I/O failure is an exception,
1072	 * in handle_read_error(), we may call freeze_array() before trying to
1073	 * fix the read error. In this case, the error read I/O is not queued,
1074	 * so get_unqueued_pending() == 1.
1075	 *
1076	 * Therefore before this function returns, we need to wait until
1077	 * get_unqueued_pendings(conf) gets equal to extra. For
1078	 * normal I/O context, extra is 1, in rested situations extra is 0.
1079	 */
1080	spin_lock_irq(&conf->resync_lock);
1081	conf->array_frozen = 1;
1082	raid1_log(conf->mddev, "wait freeze");
1083	wait_event_lock_irq_cmd(
1084		conf->wait_barrier,
1085		get_unqueued_pending(conf) == extra,
1086		conf->resync_lock,
1087		flush_pending_writes(conf));
1088	spin_unlock_irq(&conf->resync_lock);
1089}
1090static void unfreeze_array(struct r1conf *conf)
1091{
1092	/* reverse the effect of the freeze */
1093	spin_lock_irq(&conf->resync_lock);
1094	conf->array_frozen = 0;
1095	spin_unlock_irq(&conf->resync_lock);
1096	wake_up(&conf->wait_barrier);
1097}
1098
1099static void alloc_behind_master_bio(struct r1bio *r1_bio,
1100					   struct bio *bio)
1101{
1102	int size = bio->bi_iter.bi_size;
1103	unsigned vcnt = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1104	int i = 0;
1105	struct bio *behind_bio = NULL;
1106
1107	behind_bio = bio_alloc_mddev(GFP_NOIO, vcnt, r1_bio->mddev);
1108	if (!behind_bio)
1109		return;
1110
1111	/* discard op, we don't support writezero/writesame yet */
1112	if (!bio_has_data(bio)) {
1113		behind_bio->bi_iter.bi_size = size;
1114		goto skip_copy;
1115	}
1116
1117	behind_bio->bi_write_hint = bio->bi_write_hint;
1118
1119	while (i < vcnt && size) {
1120		struct page *page;
1121		int len = min_t(int, PAGE_SIZE, size);
1122
1123		page = alloc_page(GFP_NOIO);
1124		if (unlikely(!page))
1125			goto free_pages;
1126
1127		bio_add_page(behind_bio, page, len, 0);
 
 
 
1128
1129		size -= len;
1130		i++;
1131	}
1132
1133	bio_copy_data(behind_bio, bio);
1134skip_copy:
1135	r1_bio->behind_master_bio = behind_bio;
1136	set_bit(R1BIO_BehindIO, &r1_bio->state);
1137
1138	return;
1139
1140free_pages:
1141	pr_debug("%dB behind alloc failed, doing sync I/O\n",
1142		 bio->bi_iter.bi_size);
1143	bio_free_pages(behind_bio);
1144	bio_put(behind_bio);
1145}
1146
1147struct raid1_plug_cb {
1148	struct blk_plug_cb	cb;
1149	struct bio_list		pending;
1150	int			pending_cnt;
1151};
1152
1153static void raid1_unplug(struct blk_plug_cb *cb, bool from_schedule)
1154{
1155	struct raid1_plug_cb *plug = container_of(cb, struct raid1_plug_cb,
1156						  cb);
1157	struct mddev *mddev = plug->cb.data;
1158	struct r1conf *conf = mddev->private;
1159	struct bio *bio;
1160
1161	if (from_schedule || current->bio_list) {
1162		spin_lock_irq(&conf->device_lock);
1163		bio_list_merge(&conf->pending_bio_list, &plug->pending);
1164		conf->pending_count += plug->pending_cnt;
1165		spin_unlock_irq(&conf->device_lock);
1166		wake_up(&conf->wait_barrier);
1167		md_wakeup_thread(mddev->thread);
1168		kfree(plug);
1169		return;
1170	}
1171
1172	/* we aren't scheduling, so we can do the write-out directly. */
1173	bio = bio_list_get(&plug->pending);
1174	flush_bio_list(conf, bio);
1175	kfree(plug);
1176}
1177
1178static void init_r1bio(struct r1bio *r1_bio, struct mddev *mddev, struct bio *bio)
1179{
1180	r1_bio->master_bio = bio;
1181	r1_bio->sectors = bio_sectors(bio);
1182	r1_bio->state = 0;
1183	r1_bio->mddev = mddev;
1184	r1_bio->sector = bio->bi_iter.bi_sector;
1185}
1186
1187static inline struct r1bio *
1188alloc_r1bio(struct mddev *mddev, struct bio *bio)
1189{
1190	struct r1conf *conf = mddev->private;
1191	struct r1bio *r1_bio;
1192
1193	r1_bio = mempool_alloc(&conf->r1bio_pool, GFP_NOIO);
1194	/* Ensure no bio records IO_BLOCKED */
1195	memset(r1_bio->bios, 0, conf->raid_disks * sizeof(r1_bio->bios[0]));
1196	init_r1bio(r1_bio, mddev, bio);
1197	return r1_bio;
1198}
1199
1200static void raid1_read_request(struct mddev *mddev, struct bio *bio,
1201			       int max_read_sectors, struct r1bio *r1_bio)
1202{
1203	struct r1conf *conf = mddev->private;
1204	struct raid1_info *mirror;
1205	struct bio *read_bio;
1206	struct bitmap *bitmap = mddev->bitmap;
1207	const int op = bio_op(bio);
1208	const unsigned long do_sync = (bio->bi_opf & REQ_SYNC);
1209	int max_sectors;
1210	int rdisk;
1211	bool print_msg = !!r1_bio;
1212	char b[BDEVNAME_SIZE];
1213
1214	/*
1215	 * If r1_bio is set, we are blocking the raid1d thread
1216	 * so there is a tiny risk of deadlock.  So ask for
1217	 * emergency memory if needed.
1218	 */
1219	gfp_t gfp = r1_bio ? (GFP_NOIO | __GFP_HIGH) : GFP_NOIO;
1220
1221	if (print_msg) {
1222		/* Need to get the block device name carefully */
1223		struct md_rdev *rdev;
1224		rcu_read_lock();
1225		rdev = rcu_dereference(conf->mirrors[r1_bio->read_disk].rdev);
1226		if (rdev)
1227			bdevname(rdev->bdev, b);
1228		else
1229			strcpy(b, "???");
1230		rcu_read_unlock();
1231	}
1232
1233	/*
1234	 * Still need barrier for READ in case that whole
1235	 * array is frozen.
1236	 */
1237	wait_read_barrier(conf, bio->bi_iter.bi_sector);
 
 
 
 
1238
1239	if (!r1_bio)
1240		r1_bio = alloc_r1bio(mddev, bio);
1241	else
1242		init_r1bio(r1_bio, mddev, bio);
1243	r1_bio->sectors = max_read_sectors;
1244
1245	/*
1246	 * make_request() can abort the operation when read-ahead is being
1247	 * used and no empty request is available.
1248	 */
1249	rdisk = read_balance(conf, r1_bio, &max_sectors);
1250
1251	if (rdisk < 0) {
1252		/* couldn't find anywhere to read from */
1253		if (print_msg) {
1254			pr_crit_ratelimited("md/raid1:%s: %s: unrecoverable I/O read error for block %llu\n",
1255					    mdname(mddev),
1256					    b,
1257					    (unsigned long long)r1_bio->sector);
1258		}
1259		raid_end_bio_io(r1_bio);
1260		return;
1261	}
1262	mirror = conf->mirrors + rdisk;
1263
1264	if (print_msg)
1265		pr_info_ratelimited("md/raid1:%s: redirecting sector %llu to other mirror: %s\n",
1266				    mdname(mddev),
1267				    (unsigned long long)r1_bio->sector,
1268				    bdevname(mirror->rdev->bdev, b));
1269
1270	if (test_bit(WriteMostly, &mirror->rdev->flags) &&
1271	    bitmap) {
1272		/*
1273		 * Reading from a write-mostly device must take care not to
1274		 * over-take any writes that are 'behind'
1275		 */
1276		raid1_log(mddev, "wait behind writes");
1277		wait_event(bitmap->behind_wait,
1278			   atomic_read(&bitmap->behind_writes) == 0);
1279	}
1280
1281	if (max_sectors < bio_sectors(bio)) {
1282		struct bio *split = bio_split(bio, max_sectors,
1283					      gfp, &conf->bio_split);
 
 
 
 
 
1284		bio_chain(split, bio);
1285		submit_bio_noacct(bio);
1286		bio = split;
1287		r1_bio->master_bio = bio;
1288		r1_bio->sectors = max_sectors;
1289	}
1290
1291	r1_bio->read_disk = rdisk;
1292
1293	read_bio = bio_clone_fast(bio, gfp, &mddev->bio_set);
 
 
 
 
1294
1295	r1_bio->bios[rdisk] = read_bio;
1296
1297	read_bio->bi_iter.bi_sector = r1_bio->sector +
1298		mirror->rdev->data_offset;
1299	bio_set_dev(read_bio, mirror->rdev->bdev);
1300	read_bio->bi_end_io = raid1_end_read_request;
1301	bio_set_op_attrs(read_bio, op, do_sync);
1302	if (test_bit(FailFast, &mirror->rdev->flags) &&
1303	    test_bit(R1BIO_FailFast, &r1_bio->state))
1304	        read_bio->bi_opf |= MD_FAILFAST;
1305	read_bio->bi_private = r1_bio;
 
 
 
1306
1307	if (mddev->gendisk)
1308	        trace_block_bio_remap(read_bio->bi_disk->queue, read_bio,
1309				disk_devt(mddev->gendisk), r1_bio->sector);
 
 
 
1310
1311	submit_bio_noacct(read_bio);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1312}
1313
1314static void raid1_write_request(struct mddev *mddev, struct bio *bio,
1315				int max_write_sectors)
1316{
1317	struct r1conf *conf = mddev->private;
1318	struct r1bio *r1_bio;
1319	int i, disks;
1320	struct bitmap *bitmap = mddev->bitmap;
1321	unsigned long flags;
1322	struct md_rdev *blocked_rdev;
1323	struct blk_plug_cb *cb;
1324	struct raid1_plug_cb *plug = NULL;
1325	int first_clone;
1326	int max_sectors;
 
 
1327
1328	if (mddev_is_clustered(mddev) &&
1329	     md_cluster_ops->area_resyncing(mddev, WRITE,
1330		     bio->bi_iter.bi_sector, bio_end_sector(bio))) {
1331
1332		DEFINE_WAIT(w);
 
 
 
 
1333		for (;;) {
1334			prepare_to_wait(&conf->wait_barrier,
1335					&w, TASK_IDLE);
1336			if (!md_cluster_ops->area_resyncing(mddev, WRITE,
1337							bio->bi_iter.bi_sector,
1338							bio_end_sector(bio)))
1339				break;
1340			schedule();
1341		}
1342		finish_wait(&conf->wait_barrier, &w);
1343	}
1344
1345	/*
1346	 * Register the new request and wait if the reconstruction
1347	 * thread has put up a bar for new requests.
1348	 * Continue immediately if no resync is active currently.
1349	 */
1350	wait_barrier(conf, bio->bi_iter.bi_sector);
 
 
 
 
 
 
 
 
 
1351
1352	r1_bio = alloc_r1bio(mddev, bio);
1353	r1_bio->sectors = max_write_sectors;
1354
1355	if (conf->pending_count >= max_queued_requests) {
1356		md_wakeup_thread(mddev->thread);
1357		raid1_log(mddev, "wait queued");
1358		wait_event(conf->wait_barrier,
1359			   conf->pending_count < max_queued_requests);
1360	}
1361	/* first select target devices under rcu_lock and
1362	 * inc refcount on their rdev.  Record them by setting
1363	 * bios[x] to bio
1364	 * If there are known/acknowledged bad blocks on any device on
1365	 * which we have seen a write error, we want to avoid writing those
1366	 * blocks.
1367	 * This potentially requires several writes to write around
1368	 * the bad blocks.  Each set of writes gets it's own r1bio
1369	 * with a set of bios attached.
1370	 */
1371
1372	disks = conf->raid_disks * 2;
1373 retry_write:
1374	blocked_rdev = NULL;
1375	rcu_read_lock();
1376	max_sectors = r1_bio->sectors;
1377	for (i = 0;  i < disks; i++) {
1378		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
1379		if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
1380			atomic_inc(&rdev->nr_pending);
1381			blocked_rdev = rdev;
1382			break;
1383		}
 
 
 
 
1384		r1_bio->bios[i] = NULL;
1385		if (!rdev || test_bit(Faulty, &rdev->flags)) {
1386			if (i < conf->raid_disks)
1387				set_bit(R1BIO_Degraded, &r1_bio->state);
1388			continue;
1389		}
1390
1391		atomic_inc(&rdev->nr_pending);
1392		if (test_bit(WriteErrorSeen, &rdev->flags)) {
1393			sector_t first_bad;
1394			int bad_sectors;
1395			int is_bad;
1396
1397			is_bad = is_badblock(rdev, r1_bio->sector, max_sectors,
1398					     &first_bad, &bad_sectors);
1399			if (is_bad < 0) {
1400				/* mustn't write here until the bad block is
1401				 * acknowledged*/
1402				set_bit(BlockedBadBlocks, &rdev->flags);
1403				blocked_rdev = rdev;
1404				break;
1405			}
1406			if (is_bad && first_bad <= r1_bio->sector) {
1407				/* Cannot write here at all */
1408				bad_sectors -= (r1_bio->sector - first_bad);
1409				if (bad_sectors < max_sectors)
1410					/* mustn't write more than bad_sectors
1411					 * to other devices yet
1412					 */
1413					max_sectors = bad_sectors;
1414				rdev_dec_pending(rdev, mddev);
1415				/* We don't set R1BIO_Degraded as that
1416				 * only applies if the disk is
1417				 * missing, so it might be re-added,
1418				 * and we want to know to recover this
1419				 * chunk.
1420				 * In this case the device is here,
1421				 * and the fact that this chunk is not
1422				 * in-sync is recorded in the bad
1423				 * block log
1424				 */
1425				continue;
1426			}
1427			if (is_bad) {
1428				int good_sectors = first_bad - r1_bio->sector;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1429				if (good_sectors < max_sectors)
1430					max_sectors = good_sectors;
1431			}
1432		}
1433		r1_bio->bios[i] = bio;
1434	}
1435	rcu_read_unlock();
1436
1437	if (unlikely(blocked_rdev)) {
1438		/* Wait for this device to become unblocked */
1439		int j;
1440
1441		for (j = 0; j < i; j++)
1442			if (r1_bio->bios[j])
1443				rdev_dec_pending(conf->mirrors[j].rdev, mddev);
1444		r1_bio->state = 0;
1445		allow_barrier(conf, bio->bi_iter.bi_sector);
1446		raid1_log(mddev, "wait rdev %d blocked", blocked_rdev->raid_disk);
1447		md_wait_for_blocked_rdev(blocked_rdev, mddev);
1448		wait_barrier(conf, bio->bi_iter.bi_sector);
1449		goto retry_write;
1450	}
1451
 
 
 
 
 
 
 
 
 
1452	if (max_sectors < bio_sectors(bio)) {
1453		struct bio *split = bio_split(bio, max_sectors,
1454					      GFP_NOIO, &conf->bio_split);
 
 
 
 
 
1455		bio_chain(split, bio);
1456		submit_bio_noacct(bio);
1457		bio = split;
1458		r1_bio->master_bio = bio;
1459		r1_bio->sectors = max_sectors;
1460	}
1461
 
 
1462	atomic_set(&r1_bio->remaining, 1);
1463	atomic_set(&r1_bio->behind_remaining, 0);
1464
1465	first_clone = 1;
1466
1467	for (i = 0; i < disks; i++) {
1468		struct bio *mbio = NULL;
1469		struct md_rdev *rdev = conf->mirrors[i].rdev;
1470		if (!r1_bio->bios[i])
1471			continue;
1472
1473		if (first_clone) {
 
 
 
 
 
1474			/* do behind I/O ?
1475			 * Not if there are too many, or cannot
1476			 * allocate memory, or a reader on WriteMostly
1477			 * is waiting for behind writes to flush */
1478			if (bitmap &&
1479			    (atomic_read(&bitmap->behind_writes)
1480			     < mddev->bitmap_info.max_write_behind) &&
1481			    !waitqueue_active(&bitmap->behind_wait)) {
1482				alloc_behind_master_bio(r1_bio, bio);
1483			}
1484
1485			md_bitmap_startwrite(bitmap, r1_bio->sector, r1_bio->sectors,
1486					     test_bit(R1BIO_BehindIO, &r1_bio->state));
1487			first_clone = 0;
1488		}
1489
1490		if (r1_bio->behind_master_bio)
1491			mbio = bio_clone_fast(r1_bio->behind_master_bio,
1492					      GFP_NOIO, &mddev->bio_set);
1493		else
1494			mbio = bio_clone_fast(bio, GFP_NOIO, &mddev->bio_set);
1495
1496		if (r1_bio->behind_master_bio) {
 
 
 
1497			if (test_bit(CollisionCheck, &rdev->flags))
1498				wait_for_serialization(rdev, r1_bio);
1499			if (test_bit(WriteMostly, &rdev->flags))
1500				atomic_inc(&r1_bio->behind_remaining);
1501		} else if (mddev->serialize_policy)
1502			wait_for_serialization(rdev, r1_bio);
 
 
 
 
 
1503
1504		r1_bio->bios[i] = mbio;
1505
1506		mbio->bi_iter.bi_sector	= (r1_bio->sector +
1507				   conf->mirrors[i].rdev->data_offset);
1508		bio_set_dev(mbio, conf->mirrors[i].rdev->bdev);
1509		mbio->bi_end_io	= raid1_end_write_request;
1510		mbio->bi_opf = bio_op(bio) | (bio->bi_opf & (REQ_SYNC | REQ_FUA));
1511		if (test_bit(FailFast, &conf->mirrors[i].rdev->flags) &&
1512		    !test_bit(WriteMostly, &conf->mirrors[i].rdev->flags) &&
 
1513		    conf->raid_disks - mddev->degraded > 1)
1514			mbio->bi_opf |= MD_FAILFAST;
1515		mbio->bi_private = r1_bio;
1516
1517		atomic_inc(&r1_bio->remaining);
1518
1519		if (mddev->gendisk)
1520			trace_block_bio_remap(mbio->bi_disk->queue,
1521					      mbio, disk_devt(mddev->gendisk),
1522					      r1_bio->sector);
1523		/* flush_pending_writes() needs access to the rdev so...*/
1524		mbio->bi_disk = (void *)conf->mirrors[i].rdev;
1525
1526		cb = blk_check_plugged(raid1_unplug, mddev, sizeof(*plug));
1527		if (cb)
1528			plug = container_of(cb, struct raid1_plug_cb, cb);
1529		else
1530			plug = NULL;
1531		if (plug) {
1532			bio_list_add(&plug->pending, mbio);
1533			plug->pending_cnt++;
1534		} else {
1535			spin_lock_irqsave(&conf->device_lock, flags);
1536			bio_list_add(&conf->pending_bio_list, mbio);
1537			conf->pending_count++;
1538			spin_unlock_irqrestore(&conf->device_lock, flags);
1539			md_wakeup_thread(mddev->thread);
1540		}
1541	}
1542
1543	r1_bio_write_done(r1_bio);
1544
1545	/* In case raid1d snuck in to freeze_array */
1546	wake_up(&conf->wait_barrier);
 
 
 
 
 
 
 
 
 
 
 
 
1547}
1548
1549static bool raid1_make_request(struct mddev *mddev, struct bio *bio)
1550{
1551	sector_t sectors;
1552
1553	if (unlikely(bio->bi_opf & REQ_PREFLUSH)
1554	    && md_flush_request(mddev, bio))
1555		return true;
1556
1557	/*
1558	 * There is a limit to the maximum size, but
1559	 * the read/write handler might find a lower limit
1560	 * due to bad blocks.  To avoid multiple splits,
1561	 * we pass the maximum number of sectors down
1562	 * and let the lower level perform the split.
1563	 */
1564	sectors = align_to_barrier_unit_end(
1565		bio->bi_iter.bi_sector, bio_sectors(bio));
1566
1567	if (bio_data_dir(bio) == READ)
1568		raid1_read_request(mddev, bio, sectors, NULL);
1569	else {
1570		if (!md_write_start(mddev,bio))
1571			return false;
1572		raid1_write_request(mddev, bio, sectors);
1573	}
1574	return true;
1575}
1576
1577static void raid1_status(struct seq_file *seq, struct mddev *mddev)
1578{
1579	struct r1conf *conf = mddev->private;
1580	int i;
1581
 
 
1582	seq_printf(seq, " [%d/%d] [", conf->raid_disks,
1583		   conf->raid_disks - mddev->degraded);
1584	rcu_read_lock();
1585	for (i = 0; i < conf->raid_disks; i++) {
1586		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
 
1587		seq_printf(seq, "%s",
1588			   rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
1589	}
1590	rcu_read_unlock();
1591	seq_printf(seq, "]");
1592}
1593
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1594static void raid1_error(struct mddev *mddev, struct md_rdev *rdev)
1595{
1596	char b[BDEVNAME_SIZE];
1597	struct r1conf *conf = mddev->private;
1598	unsigned long flags;
1599
1600	/*
1601	 * If it is not operational, then we have already marked it as dead
1602	 * else if it is the last working disks with "fail_last_dev == false",
1603	 * ignore the error, let the next level up know.
1604	 * else mark the drive as failed
1605	 */
1606	spin_lock_irqsave(&conf->device_lock, flags);
1607	if (test_bit(In_sync, &rdev->flags) && !mddev->fail_last_dev
1608	    && (conf->raid_disks - mddev->degraded) == 1) {
1609		/*
1610		 * Don't fail the drive, act as though we were just a
1611		 * normal single drive.
1612		 * However don't try a recovery from this drive as
1613		 * it is very likely to fail.
1614		 */
1615		conf->recovery_disabled = mddev->recovery_disabled;
1616		spin_unlock_irqrestore(&conf->device_lock, flags);
1617		return;
1618	}
1619	set_bit(Blocked, &rdev->flags);
1620	if (test_and_clear_bit(In_sync, &rdev->flags))
1621		mddev->degraded++;
1622	set_bit(Faulty, &rdev->flags);
1623	spin_unlock_irqrestore(&conf->device_lock, flags);
1624	/*
1625	 * if recovery is running, make sure it aborts.
1626	 */
1627	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1628	set_mask_bits(&mddev->sb_flags, 0,
1629		      BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
1630	pr_crit("md/raid1:%s: Disk failure on %s, disabling device.\n"
1631		"md/raid1:%s: Operation continuing on %d devices.\n",
1632		mdname(mddev), bdevname(rdev->bdev, b),
1633		mdname(mddev), conf->raid_disks - mddev->degraded);
1634}
1635
1636static void print_conf(struct r1conf *conf)
1637{
1638	int i;
1639
1640	pr_debug("RAID1 conf printout:\n");
1641	if (!conf) {
1642		pr_debug("(!conf)\n");
1643		return;
1644	}
1645	pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
1646		 conf->raid_disks);
1647
1648	rcu_read_lock();
1649	for (i = 0; i < conf->raid_disks; i++) {
1650		char b[BDEVNAME_SIZE];
1651		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
1652		if (rdev)
1653			pr_debug(" disk %d, wo:%d, o:%d, dev:%s\n",
1654				 i, !test_bit(In_sync, &rdev->flags),
1655				 !test_bit(Faulty, &rdev->flags),
1656				 bdevname(rdev->bdev,b));
1657	}
1658	rcu_read_unlock();
1659}
1660
1661static void close_sync(struct r1conf *conf)
1662{
1663	int idx;
1664
1665	for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) {
1666		_wait_barrier(conf, idx);
1667		_allow_barrier(conf, idx);
1668	}
1669
1670	mempool_exit(&conf->r1buf_pool);
1671}
1672
1673static int raid1_spare_active(struct mddev *mddev)
1674{
1675	int i;
1676	struct r1conf *conf = mddev->private;
1677	int count = 0;
1678	unsigned long flags;
1679
1680	/*
1681	 * Find all failed disks within the RAID1 configuration
1682	 * and mark them readable.
1683	 * Called under mddev lock, so rcu protection not needed.
1684	 * device_lock used to avoid races with raid1_end_read_request
1685	 * which expects 'In_sync' flags and ->degraded to be consistent.
1686	 */
1687	spin_lock_irqsave(&conf->device_lock, flags);
1688	for (i = 0; i < conf->raid_disks; i++) {
1689		struct md_rdev *rdev = conf->mirrors[i].rdev;
1690		struct md_rdev *repl = conf->mirrors[conf->raid_disks + i].rdev;
1691		if (repl
1692		    && !test_bit(Candidate, &repl->flags)
1693		    && repl->recovery_offset == MaxSector
1694		    && !test_bit(Faulty, &repl->flags)
1695		    && !test_and_set_bit(In_sync, &repl->flags)) {
1696			/* replacement has just become active */
1697			if (!rdev ||
1698			    !test_and_clear_bit(In_sync, &rdev->flags))
1699				count++;
1700			if (rdev) {
1701				/* Replaced device not technically
1702				 * faulty, but we need to be sure
1703				 * it gets removed and never re-added
1704				 */
1705				set_bit(Faulty, &rdev->flags);
1706				sysfs_notify_dirent_safe(
1707					rdev->sysfs_state);
1708			}
1709		}
1710		if (rdev
1711		    && rdev->recovery_offset == MaxSector
1712		    && !test_bit(Faulty, &rdev->flags)
1713		    && !test_and_set_bit(In_sync, &rdev->flags)) {
1714			count++;
1715			sysfs_notify_dirent_safe(rdev->sysfs_state);
1716		}
1717	}
1718	mddev->degraded -= count;
1719	spin_unlock_irqrestore(&conf->device_lock, flags);
1720
1721	print_conf(conf);
1722	return count;
1723}
1724
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1725static int raid1_add_disk(struct mddev *mddev, struct md_rdev *rdev)
1726{
1727	struct r1conf *conf = mddev->private;
1728	int err = -EEXIST;
1729	int mirror = 0;
1730	struct raid1_info *p;
1731	int first = 0;
1732	int last = conf->raid_disks - 1;
1733
1734	if (mddev->recovery_disabled == conf->recovery_disabled)
1735		return -EBUSY;
1736
1737	if (md_integrity_add_rdev(rdev, mddev))
1738		return -ENXIO;
1739
1740	if (rdev->raid_disk >= 0)
1741		first = last = rdev->raid_disk;
1742
1743	/*
1744	 * find the disk ... but prefer rdev->saved_raid_disk
1745	 * if possible.
1746	 */
1747	if (rdev->saved_raid_disk >= 0 &&
1748	    rdev->saved_raid_disk >= first &&
1749	    rdev->saved_raid_disk < conf->raid_disks &&
1750	    conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
1751		first = last = rdev->saved_raid_disk;
1752
1753	for (mirror = first; mirror <= last; mirror++) {
1754		p = conf->mirrors + mirror;
1755		if (!p->rdev) {
1756			if (mddev->gendisk)
1757				disk_stack_limits(mddev->gendisk, rdev->bdev,
1758						  rdev->data_offset << 9);
1759
1760			p->head_position = 0;
1761			rdev->raid_disk = mirror;
1762			err = 0;
1763			/* As all devices are equivalent, we don't need a full recovery
1764			 * if this was recently any drive of the array
1765			 */
1766			if (rdev->saved_raid_disk < 0)
1767				conf->fullsync = 1;
1768			rcu_assign_pointer(p->rdev, rdev);
1769			break;
1770		}
1771		if (test_bit(WantReplacement, &p->rdev->flags) &&
1772		    p[conf->raid_disks].rdev == NULL) {
1773			/* Add this device as a replacement */
1774			clear_bit(In_sync, &rdev->flags);
1775			set_bit(Replacement, &rdev->flags);
1776			rdev->raid_disk = mirror;
1777			err = 0;
1778			conf->fullsync = 1;
1779			rcu_assign_pointer(p[conf->raid_disks].rdev, rdev);
1780			break;
1781		}
 
1782	}
1783	if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev)))
1784		blk_queue_flag_set(QUEUE_FLAG_DISCARD, mddev->queue);
1785	print_conf(conf);
1786	return err;
1787}
1788
1789static int raid1_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
1790{
1791	struct r1conf *conf = mddev->private;
1792	int err = 0;
1793	int number = rdev->raid_disk;
1794	struct raid1_info *p = conf->mirrors + number;
1795
1796	if (rdev != p->rdev)
1797		p = conf->mirrors + conf->raid_disks + number;
 
 
 
 
 
1798
1799	print_conf(conf);
1800	if (rdev == p->rdev) {
1801		if (test_bit(In_sync, &rdev->flags) ||
1802		    atomic_read(&rdev->nr_pending)) {
1803			err = -EBUSY;
1804			goto abort;
1805		}
1806		/* Only remove non-faulty devices if recovery
1807		 * is not possible.
1808		 */
1809		if (!test_bit(Faulty, &rdev->flags) &&
1810		    mddev->recovery_disabled != conf->recovery_disabled &&
1811		    mddev->degraded < conf->raid_disks) {
1812			err = -EBUSY;
1813			goto abort;
1814		}
1815		p->rdev = NULL;
1816		if (!test_bit(RemoveSynchronized, &rdev->flags)) {
1817			synchronize_rcu();
1818			if (atomic_read(&rdev->nr_pending)) {
1819				/* lost the race, try later */
1820				err = -EBUSY;
1821				p->rdev = rdev;
1822				goto abort;
1823			}
1824		}
1825		if (conf->mirrors[conf->raid_disks + number].rdev) {
1826			/* We just removed a device that is being replaced.
1827			 * Move down the replacement.  We drain all IO before
1828			 * doing this to avoid confusion.
1829			 */
1830			struct md_rdev *repl =
1831				conf->mirrors[conf->raid_disks + number].rdev;
1832			freeze_array(conf, 0);
1833			if (atomic_read(&repl->nr_pending)) {
1834				/* It means that some queued IO of retry_list
1835				 * hold repl. Thus, we cannot set replacement
1836				 * as NULL, avoiding rdev NULL pointer
1837				 * dereference in sync_request_write and
1838				 * handle_write_finished.
1839				 */
1840				err = -EBUSY;
1841				unfreeze_array(conf);
1842				goto abort;
1843			}
1844			clear_bit(Replacement, &repl->flags);
1845			p->rdev = repl;
1846			conf->mirrors[conf->raid_disks + number].rdev = NULL;
1847			unfreeze_array(conf);
1848		}
1849
1850		clear_bit(WantReplacement, &rdev->flags);
1851		err = md_integrity_register(mddev);
1852	}
1853abort:
1854
1855	print_conf(conf);
1856	return err;
1857}
1858
1859static void end_sync_read(struct bio *bio)
1860{
1861	struct r1bio *r1_bio = get_resync_r1bio(bio);
1862
1863	update_head_pos(r1_bio->read_disk, r1_bio);
1864
1865	/*
1866	 * we have read a block, now it needs to be re-written,
1867	 * or re-read if the read failed.
1868	 * We don't do much here, just schedule handling by raid1d
1869	 */
1870	if (!bio->bi_status)
1871		set_bit(R1BIO_Uptodate, &r1_bio->state);
1872
1873	if (atomic_dec_and_test(&r1_bio->remaining))
1874		reschedule_retry(r1_bio);
1875}
1876
1877static void abort_sync_write(struct mddev *mddev, struct r1bio *r1_bio)
1878{
1879	sector_t sync_blocks = 0;
1880	sector_t s = r1_bio->sector;
1881	long sectors_to_go = r1_bio->sectors;
1882
1883	/* make sure these bits don't get cleared. */
1884	do {
1885		md_bitmap_end_sync(mddev->bitmap, s, &sync_blocks, 1);
1886		s += sync_blocks;
1887		sectors_to_go -= sync_blocks;
1888	} while (sectors_to_go > 0);
1889}
1890
1891static void put_sync_write_buf(struct r1bio *r1_bio, int uptodate)
1892{
1893	if (atomic_dec_and_test(&r1_bio->remaining)) {
1894		struct mddev *mddev = r1_bio->mddev;
1895		int s = r1_bio->sectors;
1896
1897		if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||
1898		    test_bit(R1BIO_WriteError, &r1_bio->state))
1899			reschedule_retry(r1_bio);
1900		else {
1901			put_buf(r1_bio);
1902			md_done_sync(mddev, s, uptodate);
1903		}
1904	}
1905}
1906
1907static void end_sync_write(struct bio *bio)
1908{
1909	int uptodate = !bio->bi_status;
1910	struct r1bio *r1_bio = get_resync_r1bio(bio);
1911	struct mddev *mddev = r1_bio->mddev;
1912	struct r1conf *conf = mddev->private;
1913	sector_t first_bad;
1914	int bad_sectors;
1915	struct md_rdev *rdev = conf->mirrors[find_bio_disk(r1_bio, bio)].rdev;
1916
1917	if (!uptodate) {
1918		abort_sync_write(mddev, r1_bio);
1919		set_bit(WriteErrorSeen, &rdev->flags);
1920		if (!test_and_set_bit(WantReplacement, &rdev->flags))
1921			set_bit(MD_RECOVERY_NEEDED, &
1922				mddev->recovery);
1923		set_bit(R1BIO_WriteError, &r1_bio->state);
1924	} else if (is_badblock(rdev, r1_bio->sector, r1_bio->sectors,
1925			       &first_bad, &bad_sectors) &&
1926		   !is_badblock(conf->mirrors[r1_bio->read_disk].rdev,
1927				r1_bio->sector,
1928				r1_bio->sectors,
1929				&first_bad, &bad_sectors)
1930		)
1931		set_bit(R1BIO_MadeGood, &r1_bio->state);
 
1932
1933	put_sync_write_buf(r1_bio, uptodate);
1934}
1935
1936static int r1_sync_page_io(struct md_rdev *rdev, sector_t sector,
1937			    int sectors, struct page *page, int rw)
1938{
1939	if (sync_page_io(rdev, sector, sectors << 9, page, rw, 0, false))
1940		/* success */
1941		return 1;
1942	if (rw == WRITE) {
1943		set_bit(WriteErrorSeen, &rdev->flags);
1944		if (!test_and_set_bit(WantReplacement,
1945				      &rdev->flags))
1946			set_bit(MD_RECOVERY_NEEDED, &
1947				rdev->mddev->recovery);
1948	}
1949	/* need to record an error - either for the block or the device */
1950	if (!rdev_set_badblocks(rdev, sector, sectors, 0))
1951		md_error(rdev->mddev, rdev);
1952	return 0;
1953}
1954
1955static int fix_sync_read_error(struct r1bio *r1_bio)
1956{
1957	/* Try some synchronous reads of other devices to get
1958	 * good data, much like with normal read errors.  Only
1959	 * read into the pages we already have so we don't
1960	 * need to re-issue the read request.
1961	 * We don't need to freeze the array, because being in an
1962	 * active sync request, there is no normal IO, and
1963	 * no overlapping syncs.
1964	 * We don't need to check is_badblock() again as we
1965	 * made sure that anything with a bad block in range
1966	 * will have bi_end_io clear.
1967	 */
1968	struct mddev *mddev = r1_bio->mddev;
1969	struct r1conf *conf = mddev->private;
1970	struct bio *bio = r1_bio->bios[r1_bio->read_disk];
1971	struct page **pages = get_resync_pages(bio)->pages;
1972	sector_t sect = r1_bio->sector;
1973	int sectors = r1_bio->sectors;
1974	int idx = 0;
1975	struct md_rdev *rdev;
1976
1977	rdev = conf->mirrors[r1_bio->read_disk].rdev;
1978	if (test_bit(FailFast, &rdev->flags)) {
1979		/* Don't try recovering from here - just fail it
1980		 * ... unless it is the last working device of course */
1981		md_error(mddev, rdev);
1982		if (test_bit(Faulty, &rdev->flags))
1983			/* Don't try to read from here, but make sure
1984			 * put_buf does it's thing
1985			 */
1986			bio->bi_end_io = end_sync_write;
1987	}
1988
1989	while(sectors) {
1990		int s = sectors;
1991		int d = r1_bio->read_disk;
1992		int success = 0;
1993		int start;
1994
1995		if (s > (PAGE_SIZE>>9))
1996			s = PAGE_SIZE >> 9;
1997		do {
1998			if (r1_bio->bios[d]->bi_end_io == end_sync_read) {
1999				/* No rcu protection needed here devices
2000				 * can only be removed when no resync is
2001				 * active, and resync is currently active
2002				 */
2003				rdev = conf->mirrors[d].rdev;
2004				if (sync_page_io(rdev, sect, s<<9,
2005						 pages[idx],
2006						 REQ_OP_READ, 0, false)) {
2007					success = 1;
2008					break;
2009				}
2010			}
2011			d++;
2012			if (d == conf->raid_disks * 2)
2013				d = 0;
2014		} while (!success && d != r1_bio->read_disk);
2015
2016		if (!success) {
2017			char b[BDEVNAME_SIZE];
2018			int abort = 0;
2019			/* Cannot read from anywhere, this block is lost.
2020			 * Record a bad block on each device.  If that doesn't
2021			 * work just disable and interrupt the recovery.
2022			 * Don't fail devices as that won't really help.
2023			 */
2024			pr_crit_ratelimited("md/raid1:%s: %s: unrecoverable I/O read error for block %llu\n",
2025					    mdname(mddev), bio_devname(bio, b),
2026					    (unsigned long long)r1_bio->sector);
2027			for (d = 0; d < conf->raid_disks * 2; d++) {
2028				rdev = conf->mirrors[d].rdev;
2029				if (!rdev || test_bit(Faulty, &rdev->flags))
2030					continue;
2031				if (!rdev_set_badblocks(rdev, sect, s, 0))
2032					abort = 1;
2033			}
2034			if (abort) {
2035				conf->recovery_disabled =
2036					mddev->recovery_disabled;
2037				set_bit(MD_RECOVERY_INTR, &mddev->recovery);
2038				md_done_sync(mddev, r1_bio->sectors, 0);
2039				put_buf(r1_bio);
2040				return 0;
2041			}
2042			/* Try next page */
2043			sectors -= s;
2044			sect += s;
2045			idx++;
2046			continue;
2047		}
2048
2049		start = d;
2050		/* write it back and re-read */
2051		while (d != r1_bio->read_disk) {
2052			if (d == 0)
2053				d = conf->raid_disks * 2;
2054			d--;
2055			if (r1_bio->bios[d]->bi_end_io != end_sync_read)
2056				continue;
2057			rdev = conf->mirrors[d].rdev;
2058			if (r1_sync_page_io(rdev, sect, s,
2059					    pages[idx],
2060					    WRITE) == 0) {
2061				r1_bio->bios[d]->bi_end_io = NULL;
2062				rdev_dec_pending(rdev, mddev);
2063			}
2064		}
2065		d = start;
2066		while (d != r1_bio->read_disk) {
2067			if (d == 0)
2068				d = conf->raid_disks * 2;
2069			d--;
2070			if (r1_bio->bios[d]->bi_end_io != end_sync_read)
2071				continue;
2072			rdev = conf->mirrors[d].rdev;
2073			if (r1_sync_page_io(rdev, sect, s,
2074					    pages[idx],
2075					    READ) != 0)
2076				atomic_add(s, &rdev->corrected_errors);
2077		}
2078		sectors -= s;
2079		sect += s;
2080		idx ++;
2081	}
2082	set_bit(R1BIO_Uptodate, &r1_bio->state);
2083	bio->bi_status = 0;
2084	return 1;
2085}
2086
2087static void process_checks(struct r1bio *r1_bio)
2088{
2089	/* We have read all readable devices.  If we haven't
2090	 * got the block, then there is no hope left.
2091	 * If we have, then we want to do a comparison
2092	 * and skip the write if everything is the same.
2093	 * If any blocks failed to read, then we need to
2094	 * attempt an over-write
2095	 */
2096	struct mddev *mddev = r1_bio->mddev;
2097	struct r1conf *conf = mddev->private;
2098	int primary;
2099	int i;
2100	int vcnt;
2101
2102	/* Fix variable parts of all bios */
2103	vcnt = (r1_bio->sectors + PAGE_SIZE / 512 - 1) >> (PAGE_SHIFT - 9);
2104	for (i = 0; i < conf->raid_disks * 2; i++) {
2105		blk_status_t status;
2106		struct bio *b = r1_bio->bios[i];
2107		struct resync_pages *rp = get_resync_pages(b);
2108		if (b->bi_end_io != end_sync_read)
2109			continue;
2110		/* fixup the bio for reuse, but preserve errno */
2111		status = b->bi_status;
2112		bio_reset(b);
2113		b->bi_status = status;
2114		b->bi_iter.bi_sector = r1_bio->sector +
2115			conf->mirrors[i].rdev->data_offset;
2116		bio_set_dev(b, conf->mirrors[i].rdev->bdev);
2117		b->bi_end_io = end_sync_read;
2118		rp->raid_bio = r1_bio;
2119		b->bi_private = rp;
2120
2121		/* initialize bvec table again */
2122		md_bio_reset_resync_pages(b, rp, r1_bio->sectors << 9);
2123	}
2124	for (primary = 0; primary < conf->raid_disks * 2; primary++)
2125		if (r1_bio->bios[primary]->bi_end_io == end_sync_read &&
2126		    !r1_bio->bios[primary]->bi_status) {
2127			r1_bio->bios[primary]->bi_end_io = NULL;
2128			rdev_dec_pending(conf->mirrors[primary].rdev, mddev);
2129			break;
2130		}
2131	r1_bio->read_disk = primary;
2132	for (i = 0; i < conf->raid_disks * 2; i++) {
2133		int j = 0;
2134		struct bio *pbio = r1_bio->bios[primary];
2135		struct bio *sbio = r1_bio->bios[i];
2136		blk_status_t status = sbio->bi_status;
2137		struct page **ppages = get_resync_pages(pbio)->pages;
2138		struct page **spages = get_resync_pages(sbio)->pages;
2139		struct bio_vec *bi;
2140		int page_len[RESYNC_PAGES] = { 0 };
2141		struct bvec_iter_all iter_all;
2142
2143		if (sbio->bi_end_io != end_sync_read)
2144			continue;
2145		/* Now we can 'fixup' the error value */
2146		sbio->bi_status = 0;
2147
2148		bio_for_each_segment_all(bi, sbio, iter_all)
2149			page_len[j++] = bi->bv_len;
2150
2151		if (!status) {
2152			for (j = vcnt; j-- ; ) {
2153				if (memcmp(page_address(ppages[j]),
2154					   page_address(spages[j]),
2155					   page_len[j]))
2156					break;
2157			}
2158		} else
2159			j = 0;
2160		if (j >= 0)
2161			atomic64_add(r1_bio->sectors, &mddev->resync_mismatches);
2162		if (j < 0 || (test_bit(MD_RECOVERY_CHECK, &mddev->recovery)
2163			      && !status)) {
2164			/* No need to write to this device. */
2165			sbio->bi_end_io = NULL;
2166			rdev_dec_pending(conf->mirrors[i].rdev, mddev);
2167			continue;
2168		}
2169
2170		bio_copy_data(sbio, pbio);
2171	}
2172}
2173
2174static void sync_request_write(struct mddev *mddev, struct r1bio *r1_bio)
2175{
2176	struct r1conf *conf = mddev->private;
2177	int i;
2178	int disks = conf->raid_disks * 2;
2179	struct bio *wbio;
2180
2181	if (!test_bit(R1BIO_Uptodate, &r1_bio->state))
2182		/* ouch - failed to read all of that. */
2183		if (!fix_sync_read_error(r1_bio))
2184			return;
2185
2186	if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))
2187		process_checks(r1_bio);
2188
2189	/*
2190	 * schedule writes
2191	 */
2192	atomic_set(&r1_bio->remaining, 1);
2193	for (i = 0; i < disks ; i++) {
2194		wbio = r1_bio->bios[i];
2195		if (wbio->bi_end_io == NULL ||
2196		    (wbio->bi_end_io == end_sync_read &&
2197		     (i == r1_bio->read_disk ||
2198		      !test_bit(MD_RECOVERY_SYNC, &mddev->recovery))))
2199			continue;
2200		if (test_bit(Faulty, &conf->mirrors[i].rdev->flags)) {
2201			abort_sync_write(mddev, r1_bio);
2202			continue;
2203		}
2204
2205		bio_set_op_attrs(wbio, REQ_OP_WRITE, 0);
2206		if (test_bit(FailFast, &conf->mirrors[i].rdev->flags))
2207			wbio->bi_opf |= MD_FAILFAST;
2208
2209		wbio->bi_end_io = end_sync_write;
2210		atomic_inc(&r1_bio->remaining);
2211		md_sync_acct(conf->mirrors[i].rdev->bdev, bio_sectors(wbio));
2212
2213		submit_bio_noacct(wbio);
2214	}
2215
2216	put_sync_write_buf(r1_bio, 1);
2217}
2218
2219/*
2220 * This is a kernel thread which:
2221 *
2222 *	1.	Retries failed read operations on working mirrors.
2223 *	2.	Updates the raid superblock when problems encounter.
2224 *	3.	Performs writes following reads for array synchronising.
2225 */
2226
2227static void fix_read_error(struct r1conf *conf, int read_disk,
2228			   sector_t sect, int sectors)
2229{
 
 
 
2230	struct mddev *mddev = conf->mddev;
 
 
 
 
 
 
 
2231	while(sectors) {
2232		int s = sectors;
2233		int d = read_disk;
2234		int success = 0;
2235		int start;
2236		struct md_rdev *rdev;
2237
2238		if (s > (PAGE_SIZE>>9))
2239			s = PAGE_SIZE >> 9;
2240
2241		do {
2242			sector_t first_bad;
2243			int bad_sectors;
2244
2245			rcu_read_lock();
2246			rdev = rcu_dereference(conf->mirrors[d].rdev);
2247			if (rdev &&
2248			    (test_bit(In_sync, &rdev->flags) ||
2249			     (!test_bit(Faulty, &rdev->flags) &&
2250			      rdev->recovery_offset >= sect + s)) &&
2251			    is_badblock(rdev, sect, s,
2252					&first_bad, &bad_sectors) == 0) {
2253				atomic_inc(&rdev->nr_pending);
2254				rcu_read_unlock();
2255				if (sync_page_io(rdev, sect, s<<9,
2256					 conf->tmppage, REQ_OP_READ, 0, false))
2257					success = 1;
2258				rdev_dec_pending(rdev, mddev);
2259				if (success)
2260					break;
2261			} else
2262				rcu_read_unlock();
2263			d++;
2264			if (d == conf->raid_disks * 2)
2265				d = 0;
2266		} while (!success && d != read_disk);
2267
2268		if (!success) {
2269			/* Cannot read from anywhere - mark it bad */
2270			struct md_rdev *rdev = conf->mirrors[read_disk].rdev;
2271			if (!rdev_set_badblocks(rdev, sect, s, 0))
2272				md_error(mddev, rdev);
2273			break;
2274		}
2275		/* write it back and re-read */
2276		start = d;
2277		while (d != read_disk) {
2278			if (d==0)
2279				d = conf->raid_disks * 2;
2280			d--;
2281			rcu_read_lock();
2282			rdev = rcu_dereference(conf->mirrors[d].rdev);
2283			if (rdev &&
2284			    !test_bit(Faulty, &rdev->flags)) {
2285				atomic_inc(&rdev->nr_pending);
2286				rcu_read_unlock();
2287				r1_sync_page_io(rdev, sect, s,
2288						conf->tmppage, WRITE);
2289				rdev_dec_pending(rdev, mddev);
2290			} else
2291				rcu_read_unlock();
2292		}
2293		d = start;
2294		while (d != read_disk) {
2295			char b[BDEVNAME_SIZE];
2296			if (d==0)
2297				d = conf->raid_disks * 2;
2298			d--;
2299			rcu_read_lock();
2300			rdev = rcu_dereference(conf->mirrors[d].rdev);
2301			if (rdev &&
2302			    !test_bit(Faulty, &rdev->flags)) {
2303				atomic_inc(&rdev->nr_pending);
2304				rcu_read_unlock();
2305				if (r1_sync_page_io(rdev, sect, s,
2306						    conf->tmppage, READ)) {
2307					atomic_add(s, &rdev->corrected_errors);
2308					pr_info("md/raid1:%s: read error corrected (%d sectors at %llu on %s)\n",
2309						mdname(mddev), s,
2310						(unsigned long long)(sect +
2311								     rdev->data_offset),
2312						bdevname(rdev->bdev, b));
2313				}
2314				rdev_dec_pending(rdev, mddev);
2315			} else
2316				rcu_read_unlock();
2317		}
2318		sectors -= s;
2319		sect += s;
2320	}
2321}
2322
2323static int narrow_write_error(struct r1bio *r1_bio, int i)
2324{
2325	struct mddev *mddev = r1_bio->mddev;
2326	struct r1conf *conf = mddev->private;
2327	struct md_rdev *rdev = conf->mirrors[i].rdev;
2328
2329	/* bio has the data to be written to device 'i' where
2330	 * we just recently had a write error.
2331	 * We repeatedly clone the bio and trim down to one block,
2332	 * then try the write.  Where the write fails we record
2333	 * a bad block.
2334	 * It is conceivable that the bio doesn't exactly align with
2335	 * blocks.  We must handle this somehow.
2336	 *
2337	 * We currently own a reference on the rdev.
2338	 */
2339
2340	int block_sectors;
2341	sector_t sector;
2342	int sectors;
2343	int sect_to_write = r1_bio->sectors;
2344	int ok = 1;
2345
2346	if (rdev->badblocks.shift < 0)
2347		return 0;
2348
2349	block_sectors = roundup(1 << rdev->badblocks.shift,
2350				bdev_logical_block_size(rdev->bdev) >> 9);
2351	sector = r1_bio->sector;
2352	sectors = ((sector + block_sectors)
2353		   & ~(sector_t)(block_sectors - 1))
2354		- sector;
2355
2356	while (sect_to_write) {
2357		struct bio *wbio;
2358		if (sectors > sect_to_write)
2359			sectors = sect_to_write;
2360		/* Write at 'sector' for 'sectors'*/
2361
2362		if (test_bit(R1BIO_BehindIO, &r1_bio->state)) {
2363			wbio = bio_clone_fast(r1_bio->behind_master_bio,
2364					      GFP_NOIO,
2365					      &mddev->bio_set);
2366		} else {
2367			wbio = bio_clone_fast(r1_bio->master_bio, GFP_NOIO,
2368					      &mddev->bio_set);
2369		}
2370
2371		bio_set_op_attrs(wbio, REQ_OP_WRITE, 0);
2372		wbio->bi_iter.bi_sector = r1_bio->sector;
2373		wbio->bi_iter.bi_size = r1_bio->sectors << 9;
2374
2375		bio_trim(wbio, sector - r1_bio->sector, sectors);
2376		wbio->bi_iter.bi_sector += rdev->data_offset;
2377		bio_set_dev(wbio, rdev->bdev);
2378
2379		if (submit_bio_wait(wbio) < 0)
2380			/* failure! */
2381			ok = rdev_set_badblocks(rdev, sector,
2382						sectors, 0)
2383				&& ok;
2384
2385		bio_put(wbio);
2386		sect_to_write -= sectors;
2387		sector += sectors;
2388		sectors = block_sectors;
2389	}
2390	return ok;
2391}
2392
2393static void handle_sync_write_finished(struct r1conf *conf, struct r1bio *r1_bio)
2394{
2395	int m;
2396	int s = r1_bio->sectors;
2397	for (m = 0; m < conf->raid_disks * 2 ; m++) {
2398		struct md_rdev *rdev = conf->mirrors[m].rdev;
2399		struct bio *bio = r1_bio->bios[m];
2400		if (bio->bi_end_io == NULL)
2401			continue;
2402		if (!bio->bi_status &&
2403		    test_bit(R1BIO_MadeGood, &r1_bio->state)) {
2404			rdev_clear_badblocks(rdev, r1_bio->sector, s, 0);
2405		}
2406		if (bio->bi_status &&
2407		    test_bit(R1BIO_WriteError, &r1_bio->state)) {
2408			if (!rdev_set_badblocks(rdev, r1_bio->sector, s, 0))
2409				md_error(conf->mddev, rdev);
2410		}
2411	}
2412	put_buf(r1_bio);
2413	md_done_sync(conf->mddev, s, 1);
2414}
2415
2416static void handle_write_finished(struct r1conf *conf, struct r1bio *r1_bio)
2417{
2418	int m, idx;
2419	bool fail = false;
2420
2421	for (m = 0; m < conf->raid_disks * 2 ; m++)
2422		if (r1_bio->bios[m] == IO_MADE_GOOD) {
2423			struct md_rdev *rdev = conf->mirrors[m].rdev;
2424			rdev_clear_badblocks(rdev,
2425					     r1_bio->sector,
2426					     r1_bio->sectors, 0);
2427			rdev_dec_pending(rdev, conf->mddev);
2428		} else if (r1_bio->bios[m] != NULL) {
2429			/* This drive got a write error.  We need to
2430			 * narrow down and record precise write
2431			 * errors.
2432			 */
2433			fail = true;
2434			if (!narrow_write_error(r1_bio, m)) {
2435				md_error(conf->mddev,
2436					 conf->mirrors[m].rdev);
2437				/* an I/O failed, we can't clear the bitmap */
2438				set_bit(R1BIO_Degraded, &r1_bio->state);
2439			}
2440			rdev_dec_pending(conf->mirrors[m].rdev,
2441					 conf->mddev);
2442		}
2443	if (fail) {
2444		spin_lock_irq(&conf->device_lock);
2445		list_add(&r1_bio->retry_list, &conf->bio_end_io_list);
2446		idx = sector_to_idx(r1_bio->sector);
2447		atomic_inc(&conf->nr_queued[idx]);
2448		spin_unlock_irq(&conf->device_lock);
2449		/*
2450		 * In case freeze_array() is waiting for condition
2451		 * get_unqueued_pending() == extra to be true.
2452		 */
2453		wake_up(&conf->wait_barrier);
2454		md_wakeup_thread(conf->mddev->thread);
2455	} else {
2456		if (test_bit(R1BIO_WriteError, &r1_bio->state))
2457			close_write(r1_bio);
2458		raid_end_bio_io(r1_bio);
2459	}
2460}
2461
2462static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio)
2463{
2464	struct mddev *mddev = conf->mddev;
2465	struct bio *bio;
2466	struct md_rdev *rdev;
 
2467
2468	clear_bit(R1BIO_ReadError, &r1_bio->state);
2469	/* we got a read error. Maybe the drive is bad.  Maybe just
2470	 * the block and we can fix it.
2471	 * We freeze all other IO, and try reading the block from
2472	 * other devices.  When we find one, we re-write
2473	 * and check it that fixes the read error.
2474	 * This is all done synchronously while the array is
2475	 * frozen
2476	 */
2477
2478	bio = r1_bio->bios[r1_bio->read_disk];
2479	bio_put(bio);
2480	r1_bio->bios[r1_bio->read_disk] = NULL;
2481
2482	rdev = conf->mirrors[r1_bio->read_disk].rdev;
2483	if (mddev->ro == 0
2484	    && !test_bit(FailFast, &rdev->flags)) {
2485		freeze_array(conf, 1);
2486		fix_read_error(conf, r1_bio->read_disk,
2487			       r1_bio->sector, r1_bio->sectors);
2488		unfreeze_array(conf);
2489	} else if (mddev->ro == 0 && test_bit(FailFast, &rdev->flags)) {
2490		md_error(mddev, rdev);
2491	} else {
2492		r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED;
2493	}
2494
2495	rdev_dec_pending(rdev, conf->mddev);
2496	allow_barrier(conf, r1_bio->sector);
2497	bio = r1_bio->master_bio;
2498
2499	/* Reuse the old r1_bio so that the IO_BLOCKED settings are preserved */
2500	r1_bio->state = 0;
2501	raid1_read_request(mddev, bio, r1_bio->sectors, r1_bio);
 
2502}
2503
2504static void raid1d(struct md_thread *thread)
2505{
2506	struct mddev *mddev = thread->mddev;
2507	struct r1bio *r1_bio;
2508	unsigned long flags;
2509	struct r1conf *conf = mddev->private;
2510	struct list_head *head = &conf->retry_list;
2511	struct blk_plug plug;
2512	int idx;
2513
2514	md_check_recovery(mddev);
2515
2516	if (!list_empty_careful(&conf->bio_end_io_list) &&
2517	    !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2518		LIST_HEAD(tmp);
2519		spin_lock_irqsave(&conf->device_lock, flags);
2520		if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags))
2521			list_splice_init(&conf->bio_end_io_list, &tmp);
2522		spin_unlock_irqrestore(&conf->device_lock, flags);
2523		while (!list_empty(&tmp)) {
2524			r1_bio = list_first_entry(&tmp, struct r1bio,
2525						  retry_list);
2526			list_del(&r1_bio->retry_list);
2527			idx = sector_to_idx(r1_bio->sector);
2528			atomic_dec(&conf->nr_queued[idx]);
2529			if (mddev->degraded)
2530				set_bit(R1BIO_Degraded, &r1_bio->state);
2531			if (test_bit(R1BIO_WriteError, &r1_bio->state))
2532				close_write(r1_bio);
2533			raid_end_bio_io(r1_bio);
2534		}
2535	}
2536
2537	blk_start_plug(&plug);
2538	for (;;) {
2539
2540		flush_pending_writes(conf);
2541
2542		spin_lock_irqsave(&conf->device_lock, flags);
2543		if (list_empty(head)) {
2544			spin_unlock_irqrestore(&conf->device_lock, flags);
2545			break;
2546		}
2547		r1_bio = list_entry(head->prev, struct r1bio, retry_list);
2548		list_del(head->prev);
2549		idx = sector_to_idx(r1_bio->sector);
2550		atomic_dec(&conf->nr_queued[idx]);
2551		spin_unlock_irqrestore(&conf->device_lock, flags);
2552
2553		mddev = r1_bio->mddev;
2554		conf = mddev->private;
2555		if (test_bit(R1BIO_IsSync, &r1_bio->state)) {
2556			if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||
2557			    test_bit(R1BIO_WriteError, &r1_bio->state))
2558				handle_sync_write_finished(conf, r1_bio);
2559			else
2560				sync_request_write(mddev, r1_bio);
2561		} else if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||
2562			   test_bit(R1BIO_WriteError, &r1_bio->state))
2563			handle_write_finished(conf, r1_bio);
2564		else if (test_bit(R1BIO_ReadError, &r1_bio->state))
2565			handle_read_error(conf, r1_bio);
2566		else
2567			WARN_ON_ONCE(1);
2568
2569		cond_resched();
2570		if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
2571			md_check_recovery(mddev);
2572	}
2573	blk_finish_plug(&plug);
2574}
2575
2576static int init_resync(struct r1conf *conf)
2577{
2578	int buffs;
2579
2580	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
2581	BUG_ON(mempool_initialized(&conf->r1buf_pool));
2582
2583	return mempool_init(&conf->r1buf_pool, buffs, r1buf_pool_alloc,
2584			    r1buf_pool_free, conf->poolinfo);
2585}
2586
2587static struct r1bio *raid1_alloc_init_r1buf(struct r1conf *conf)
2588{
2589	struct r1bio *r1bio = mempool_alloc(&conf->r1buf_pool, GFP_NOIO);
2590	struct resync_pages *rps;
2591	struct bio *bio;
2592	int i;
2593
2594	for (i = conf->poolinfo->raid_disks; i--; ) {
2595		bio = r1bio->bios[i];
2596		rps = bio->bi_private;
2597		bio_reset(bio);
2598		bio->bi_private = rps;
2599	}
2600	r1bio->master_bio = NULL;
2601	return r1bio;
2602}
2603
2604/*
2605 * perform a "sync" on one "block"
2606 *
2607 * We need to make sure that no normal I/O request - particularly write
2608 * requests - conflict with active sync requests.
2609 *
2610 * This is achieved by tracking pending requests and a 'barrier' concept
2611 * that can be installed to exclude normal IO requests.
2612 */
2613
2614static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr,
2615				   int *skipped)
2616{
2617	struct r1conf *conf = mddev->private;
2618	struct r1bio *r1_bio;
2619	struct bio *bio;
2620	sector_t max_sector, nr_sectors;
2621	int disk = -1;
2622	int i;
2623	int wonly = -1;
2624	int write_targets = 0, read_targets = 0;
2625	sector_t sync_blocks;
2626	int still_degraded = 0;
2627	int good_sectors = RESYNC_SECTORS;
2628	int min_bad = 0; /* number of sectors that are bad in all devices */
2629	int idx = sector_to_idx(sector_nr);
2630	int page_idx = 0;
2631
2632	if (!mempool_initialized(&conf->r1buf_pool))
2633		if (init_resync(conf))
2634			return 0;
2635
2636	max_sector = mddev->dev_sectors;
2637	if (sector_nr >= max_sector) {
2638		/* If we aborted, we need to abort the
2639		 * sync on the 'current' bitmap chunk (there will
2640		 * only be one in raid1 resync.
2641		 * We can find the current addess in mddev->curr_resync
2642		 */
2643		if (mddev->curr_resync < max_sector) /* aborted */
2644			md_bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
2645					   &sync_blocks, 1);
2646		else /* completed sync */
2647			conf->fullsync = 0;
2648
2649		md_bitmap_close_sync(mddev->bitmap);
2650		close_sync(conf);
2651
2652		if (mddev_is_clustered(mddev)) {
2653			conf->cluster_sync_low = 0;
2654			conf->cluster_sync_high = 0;
2655		}
2656		return 0;
2657	}
2658
2659	if (mddev->bitmap == NULL &&
2660	    mddev->recovery_cp == MaxSector &&
2661	    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
2662	    conf->fullsync == 0) {
2663		*skipped = 1;
2664		return max_sector - sector_nr;
2665	}
2666	/* before building a request, check if we can skip these blocks..
2667	 * This call the bitmap_start_sync doesn't actually record anything
2668	 */
2669	if (!md_bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
2670	    !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
2671		/* We can skip this block, and probably several more */
2672		*skipped = 1;
2673		return sync_blocks;
2674	}
2675
2676	/*
2677	 * If there is non-resync activity waiting for a turn, then let it
2678	 * though before starting on this new sync request.
2679	 */
2680	if (atomic_read(&conf->nr_waiting[idx]))
2681		schedule_timeout_uninterruptible(1);
2682
2683	/* we are incrementing sector_nr below. To be safe, we check against
2684	 * sector_nr + two times RESYNC_SECTORS
2685	 */
2686
2687	md_bitmap_cond_end_sync(mddev->bitmap, sector_nr,
2688		mddev_is_clustered(mddev) && (sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high));
2689
2690
2691	if (raise_barrier(conf, sector_nr))
2692		return 0;
2693
2694	r1_bio = raid1_alloc_init_r1buf(conf);
2695
2696	rcu_read_lock();
2697	/*
2698	 * If we get a correctably read error during resync or recovery,
2699	 * we might want to read from a different device.  So we
2700	 * flag all drives that could conceivably be read from for READ,
2701	 * and any others (which will be non-In_sync devices) for WRITE.
2702	 * If a read fails, we try reading from something else for which READ
2703	 * is OK.
2704	 */
2705
2706	r1_bio->mddev = mddev;
2707	r1_bio->sector = sector_nr;
2708	r1_bio->state = 0;
2709	set_bit(R1BIO_IsSync, &r1_bio->state);
2710	/* make sure good_sectors won't go across barrier unit boundary */
2711	good_sectors = align_to_barrier_unit_end(sector_nr, good_sectors);
2712
2713	for (i = 0; i < conf->raid_disks * 2; i++) {
2714		struct md_rdev *rdev;
2715		bio = r1_bio->bios[i];
2716
2717		rdev = rcu_dereference(conf->mirrors[i].rdev);
2718		if (rdev == NULL ||
2719		    test_bit(Faulty, &rdev->flags)) {
2720			if (i < conf->raid_disks)
2721				still_degraded = 1;
2722		} else if (!test_bit(In_sync, &rdev->flags)) {
2723			bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
2724			bio->bi_end_io = end_sync_write;
2725			write_targets ++;
2726		} else {
2727			/* may need to read from here */
2728			sector_t first_bad = MaxSector;
2729			int bad_sectors;
2730
2731			if (is_badblock(rdev, sector_nr, good_sectors,
2732					&first_bad, &bad_sectors)) {
2733				if (first_bad > sector_nr)
2734					good_sectors = first_bad - sector_nr;
2735				else {
2736					bad_sectors -= (sector_nr - first_bad);
2737					if (min_bad == 0 ||
2738					    min_bad > bad_sectors)
2739						min_bad = bad_sectors;
2740				}
2741			}
2742			if (sector_nr < first_bad) {
2743				if (test_bit(WriteMostly, &rdev->flags)) {
2744					if (wonly < 0)
2745						wonly = i;
2746				} else {
2747					if (disk < 0)
2748						disk = i;
2749				}
2750				bio_set_op_attrs(bio, REQ_OP_READ, 0);
2751				bio->bi_end_io = end_sync_read;
2752				read_targets++;
2753			} else if (!test_bit(WriteErrorSeen, &rdev->flags) &&
2754				test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&
2755				!test_bit(MD_RECOVERY_CHECK, &mddev->recovery)) {
2756				/*
2757				 * The device is suitable for reading (InSync),
2758				 * but has bad block(s) here. Let's try to correct them,
2759				 * if we are doing resync or repair. Otherwise, leave
2760				 * this device alone for this sync request.
2761				 */
2762				bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
2763				bio->bi_end_io = end_sync_write;
2764				write_targets++;
2765			}
2766		}
2767		if (rdev && bio->bi_end_io) {
2768			atomic_inc(&rdev->nr_pending);
2769			bio->bi_iter.bi_sector = sector_nr + rdev->data_offset;
2770			bio_set_dev(bio, rdev->bdev);
2771			if (test_bit(FailFast, &rdev->flags))
2772				bio->bi_opf |= MD_FAILFAST;
2773		}
2774	}
2775	rcu_read_unlock();
2776	if (disk < 0)
2777		disk = wonly;
2778	r1_bio->read_disk = disk;
2779
2780	if (read_targets == 0 && min_bad > 0) {
2781		/* These sectors are bad on all InSync devices, so we
2782		 * need to mark them bad on all write targets
2783		 */
2784		int ok = 1;
2785		for (i = 0 ; i < conf->raid_disks * 2 ; i++)
2786			if (r1_bio->bios[i]->bi_end_io == end_sync_write) {
2787				struct md_rdev *rdev = conf->mirrors[i].rdev;
2788				ok = rdev_set_badblocks(rdev, sector_nr,
2789							min_bad, 0
2790					) && ok;
2791			}
2792		set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2793		*skipped = 1;
2794		put_buf(r1_bio);
2795
2796		if (!ok) {
2797			/* Cannot record the badblocks, so need to
2798			 * abort the resync.
2799			 * If there are multiple read targets, could just
2800			 * fail the really bad ones ???
2801			 */
2802			conf->recovery_disabled = mddev->recovery_disabled;
2803			set_bit(MD_RECOVERY_INTR, &mddev->recovery);
2804			return 0;
2805		} else
2806			return min_bad;
2807
2808	}
2809	if (min_bad > 0 && min_bad < good_sectors) {
2810		/* only resync enough to reach the next bad->good
2811		 * transition */
2812		good_sectors = min_bad;
2813	}
2814
2815	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && read_targets > 0)
2816		/* extra read targets are also write targets */
2817		write_targets += read_targets-1;
2818
2819	if (write_targets == 0 || read_targets == 0) {
2820		/* There is nowhere to write, so all non-sync
2821		 * drives must be failed - so we are finished
2822		 */
2823		sector_t rv;
2824		if (min_bad > 0)
2825			max_sector = sector_nr + min_bad;
2826		rv = max_sector - sector_nr;
2827		*skipped = 1;
2828		put_buf(r1_bio);
2829		return rv;
2830	}
2831
2832	if (max_sector > mddev->resync_max)
2833		max_sector = mddev->resync_max; /* Don't do IO beyond here */
2834	if (max_sector > sector_nr + good_sectors)
2835		max_sector = sector_nr + good_sectors;
2836	nr_sectors = 0;
2837	sync_blocks = 0;
2838	do {
2839		struct page *page;
2840		int len = PAGE_SIZE;
2841		if (sector_nr + (len>>9) > max_sector)
2842			len = (max_sector - sector_nr) << 9;
2843		if (len == 0)
2844			break;
2845		if (sync_blocks == 0) {
2846			if (!md_bitmap_start_sync(mddev->bitmap, sector_nr,
2847						  &sync_blocks, still_degraded) &&
2848			    !conf->fullsync &&
2849			    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))
2850				break;
2851			if ((len >> 9) > sync_blocks)
2852				len = sync_blocks<<9;
2853		}
2854
2855		for (i = 0 ; i < conf->raid_disks * 2; i++) {
2856			struct resync_pages *rp;
2857
2858			bio = r1_bio->bios[i];
2859			rp = get_resync_pages(bio);
2860			if (bio->bi_end_io) {
2861				page = resync_fetch_page(rp, page_idx);
2862
2863				/*
2864				 * won't fail because the vec table is big
2865				 * enough to hold all these pages
2866				 */
2867				bio_add_page(bio, page, len, 0);
2868			}
2869		}
2870		nr_sectors += len>>9;
2871		sector_nr += len>>9;
2872		sync_blocks -= (len>>9);
2873	} while (++page_idx < RESYNC_PAGES);
2874
2875	r1_bio->sectors = nr_sectors;
2876
2877	if (mddev_is_clustered(mddev) &&
2878			conf->cluster_sync_high < sector_nr + nr_sectors) {
2879		conf->cluster_sync_low = mddev->curr_resync_completed;
2880		conf->cluster_sync_high = conf->cluster_sync_low + CLUSTER_RESYNC_WINDOW_SECTORS;
2881		/* Send resync message */
2882		md_cluster_ops->resync_info_update(mddev,
2883				conf->cluster_sync_low,
2884				conf->cluster_sync_high);
2885	}
2886
2887	/* For a user-requested sync, we read all readable devices and do a
2888	 * compare
2889	 */
2890	if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
2891		atomic_set(&r1_bio->remaining, read_targets);
2892		for (i = 0; i < conf->raid_disks * 2 && read_targets; i++) {
2893			bio = r1_bio->bios[i];
2894			if (bio->bi_end_io == end_sync_read) {
2895				read_targets--;
2896				md_sync_acct_bio(bio, nr_sectors);
2897				if (read_targets == 1)
2898					bio->bi_opf &= ~MD_FAILFAST;
2899				submit_bio_noacct(bio);
2900			}
2901		}
2902	} else {
2903		atomic_set(&r1_bio->remaining, 1);
2904		bio = r1_bio->bios[r1_bio->read_disk];
2905		md_sync_acct_bio(bio, nr_sectors);
2906		if (read_targets == 1)
2907			bio->bi_opf &= ~MD_FAILFAST;
2908		submit_bio_noacct(bio);
2909	}
2910	return nr_sectors;
2911}
2912
2913static sector_t raid1_size(struct mddev *mddev, sector_t sectors, int raid_disks)
2914{
2915	if (sectors)
2916		return sectors;
2917
2918	return mddev->dev_sectors;
2919}
2920
2921static struct r1conf *setup_conf(struct mddev *mddev)
2922{
2923	struct r1conf *conf;
2924	int i;
2925	struct raid1_info *disk;
2926	struct md_rdev *rdev;
2927	int err = -ENOMEM;
2928
2929	conf = kzalloc(sizeof(struct r1conf), GFP_KERNEL);
2930	if (!conf)
2931		goto abort;
2932
2933	conf->nr_pending = kcalloc(BARRIER_BUCKETS_NR,
2934				   sizeof(atomic_t), GFP_KERNEL);
2935	if (!conf->nr_pending)
2936		goto abort;
2937
2938	conf->nr_waiting = kcalloc(BARRIER_BUCKETS_NR,
2939				   sizeof(atomic_t), GFP_KERNEL);
2940	if (!conf->nr_waiting)
2941		goto abort;
2942
2943	conf->nr_queued = kcalloc(BARRIER_BUCKETS_NR,
2944				  sizeof(atomic_t), GFP_KERNEL);
2945	if (!conf->nr_queued)
2946		goto abort;
2947
2948	conf->barrier = kcalloc(BARRIER_BUCKETS_NR,
2949				sizeof(atomic_t), GFP_KERNEL);
2950	if (!conf->barrier)
2951		goto abort;
2952
2953	conf->mirrors = kzalloc(array3_size(sizeof(struct raid1_info),
2954					    mddev->raid_disks, 2),
2955				GFP_KERNEL);
2956	if (!conf->mirrors)
2957		goto abort;
2958
2959	conf->tmppage = alloc_page(GFP_KERNEL);
2960	if (!conf->tmppage)
2961		goto abort;
2962
2963	conf->poolinfo = kzalloc(sizeof(*conf->poolinfo), GFP_KERNEL);
2964	if (!conf->poolinfo)
2965		goto abort;
2966	conf->poolinfo->raid_disks = mddev->raid_disks * 2;
2967	err = mempool_init(&conf->r1bio_pool, NR_RAID_BIOS, r1bio_pool_alloc,
2968			   rbio_pool_free, conf->poolinfo);
2969	if (err)
2970		goto abort;
2971
2972	err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0);
2973	if (err)
2974		goto abort;
2975
2976	conf->poolinfo->mddev = mddev;
2977
2978	err = -EINVAL;
2979	spin_lock_init(&conf->device_lock);
 
2980	rdev_for_each(rdev, mddev) {
2981		int disk_idx = rdev->raid_disk;
2982		if (disk_idx >= mddev->raid_disks
2983		    || disk_idx < 0)
2984			continue;
2985		if (test_bit(Replacement, &rdev->flags))
2986			disk = conf->mirrors + mddev->raid_disks + disk_idx;
2987		else
2988			disk = conf->mirrors + disk_idx;
2989
2990		if (disk->rdev)
 
2991			goto abort;
2992		disk->rdev = rdev;
2993		disk->head_position = 0;
2994		disk->seq_start = MaxSector;
2995	}
2996	conf->raid_disks = mddev->raid_disks;
2997	conf->mddev = mddev;
2998	INIT_LIST_HEAD(&conf->retry_list);
2999	INIT_LIST_HEAD(&conf->bio_end_io_list);
3000
3001	spin_lock_init(&conf->resync_lock);
3002	init_waitqueue_head(&conf->wait_barrier);
3003
3004	bio_list_init(&conf->pending_bio_list);
3005	conf->pending_count = 0;
3006	conf->recovery_disabled = mddev->recovery_disabled - 1;
3007
3008	err = -EIO;
3009	for (i = 0; i < conf->raid_disks * 2; i++) {
3010
3011		disk = conf->mirrors + i;
3012
3013		if (i < conf->raid_disks &&
3014		    disk[conf->raid_disks].rdev) {
3015			/* This slot has a replacement. */
3016			if (!disk->rdev) {
3017				/* No original, just make the replacement
3018				 * a recovering spare
3019				 */
3020				disk->rdev =
3021					disk[conf->raid_disks].rdev;
3022				disk[conf->raid_disks].rdev = NULL;
3023			} else if (!test_bit(In_sync, &disk->rdev->flags))
3024				/* Original is not in_sync - bad */
3025				goto abort;
3026		}
3027
3028		if (!disk->rdev ||
3029		    !test_bit(In_sync, &disk->rdev->flags)) {
3030			disk->head_position = 0;
3031			if (disk->rdev &&
3032			    (disk->rdev->saved_raid_disk < 0))
3033				conf->fullsync = 1;
3034		}
3035	}
3036
3037	err = -ENOMEM;
3038	conf->thread = md_register_thread(raid1d, mddev, "raid1");
 
3039	if (!conf->thread)
3040		goto abort;
3041
3042	return conf;
3043
3044 abort:
3045	if (conf) {
3046		mempool_exit(&conf->r1bio_pool);
3047		kfree(conf->mirrors);
3048		safe_put_page(conf->tmppage);
3049		kfree(conf->poolinfo);
3050		kfree(conf->nr_pending);
3051		kfree(conf->nr_waiting);
3052		kfree(conf->nr_queued);
3053		kfree(conf->barrier);
3054		bioset_exit(&conf->bio_split);
3055		kfree(conf);
3056	}
3057	return ERR_PTR(err);
3058}
3059
3060static void raid1_free(struct mddev *mddev, void *priv);
 
 
 
 
 
 
 
 
 
 
 
 
 
3061static int raid1_run(struct mddev *mddev)
3062{
3063	struct r1conf *conf;
3064	int i;
3065	struct md_rdev *rdev;
3066	int ret;
3067	bool discard_supported = false;
3068
3069	if (mddev->level != 1) {
3070		pr_warn("md/raid1:%s: raid level not set to mirroring (%d)\n",
3071			mdname(mddev), mddev->level);
3072		return -EIO;
3073	}
3074	if (mddev->reshape_position != MaxSector) {
3075		pr_warn("md/raid1:%s: reshape_position set but not supported\n",
3076			mdname(mddev));
3077		return -EIO;
3078	}
3079	if (mddev_init_writes_pending(mddev) < 0)
3080		return -ENOMEM;
3081	/*
3082	 * copy the already verified devices into our private RAID1
3083	 * bookkeeping area. [whatever we allocate in run(),
3084	 * should be freed in raid1_free()]
3085	 */
3086	if (mddev->private == NULL)
3087		conf = setup_conf(mddev);
3088	else
3089		conf = mddev->private;
3090
3091	if (IS_ERR(conf))
3092		return PTR_ERR(conf);
3093
3094	if (mddev->queue) {
3095		blk_queue_max_write_same_sectors(mddev->queue, 0);
3096		blk_queue_max_write_zeroes_sectors(mddev->queue, 0);
3097	}
3098
3099	rdev_for_each(rdev, mddev) {
3100		if (!mddev->gendisk)
3101			continue;
3102		disk_stack_limits(mddev->gendisk, rdev->bdev,
3103				  rdev->data_offset << 9);
3104		if (blk_queue_discard(bdev_get_queue(rdev->bdev)))
3105			discard_supported = true;
3106	}
3107
3108	mddev->degraded = 0;
3109	for (i = 0; i < conf->raid_disks; i++)
3110		if (conf->mirrors[i].rdev == NULL ||
3111		    !test_bit(In_sync, &conf->mirrors[i].rdev->flags) ||
3112		    test_bit(Faulty, &conf->mirrors[i].rdev->flags))
3113			mddev->degraded++;
3114	/*
3115	 * RAID1 needs at least one disk in active
3116	 */
3117	if (conf->raid_disks - mddev->degraded < 1) {
3118		ret = -EINVAL;
3119		goto abort;
3120	}
3121
3122	if (conf->raid_disks - mddev->degraded == 1)
3123		mddev->recovery_cp = MaxSector;
3124
3125	if (mddev->recovery_cp != MaxSector)
3126		pr_info("md/raid1:%s: not clean -- starting background reconstruction\n",
3127			mdname(mddev));
3128	pr_info("md/raid1:%s: active with %d out of %d mirrors\n",
3129		mdname(mddev), mddev->raid_disks - mddev->degraded,
3130		mddev->raid_disks);
3131
3132	/*
3133	 * Ok, everything is just fine now
3134	 */
3135	mddev->thread = conf->thread;
3136	conf->thread = NULL;
3137	mddev->private = conf;
3138	set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
3139
3140	md_set_array_sectors(mddev, raid1_size(mddev, 0, 0));
3141
3142	if (mddev->queue) {
3143		if (discard_supported)
3144			blk_queue_flag_set(QUEUE_FLAG_DISCARD,
3145						mddev->queue);
3146		else
3147			blk_queue_flag_clear(QUEUE_FLAG_DISCARD,
3148						  mddev->queue);
3149	}
3150
3151	ret = md_integrity_register(mddev);
3152	if (ret) {
3153		md_unregister_thread(&mddev->thread);
3154		goto abort;
3155	}
3156	return 0;
3157
3158abort:
3159	raid1_free(mddev, conf);
3160	return ret;
3161}
3162
3163static void raid1_free(struct mddev *mddev, void *priv)
3164{
3165	struct r1conf *conf = priv;
3166
3167	mempool_exit(&conf->r1bio_pool);
3168	kfree(conf->mirrors);
3169	safe_put_page(conf->tmppage);
3170	kfree(conf->poolinfo);
3171	kfree(conf->nr_pending);
3172	kfree(conf->nr_waiting);
3173	kfree(conf->nr_queued);
3174	kfree(conf->barrier);
3175	bioset_exit(&conf->bio_split);
3176	kfree(conf);
3177}
3178
3179static int raid1_resize(struct mddev *mddev, sector_t sectors)
3180{
3181	/* no resync is happening, and there is enough space
3182	 * on all devices, so we can resize.
3183	 * We need to make sure resync covers any new space.
3184	 * If the array is shrinking we should possibly wait until
3185	 * any io in the removed space completes, but it hardly seems
3186	 * worth it.
3187	 */
3188	sector_t newsize = raid1_size(mddev, sectors, 0);
 
 
3189	if (mddev->external_size &&
3190	    mddev->array_sectors > newsize)
3191		return -EINVAL;
3192	if (mddev->bitmap) {
3193		int ret = md_bitmap_resize(mddev->bitmap, newsize, 0, 0);
3194		if (ret)
3195			return ret;
3196	}
3197	md_set_array_sectors(mddev, newsize);
3198	if (sectors > mddev->dev_sectors &&
3199	    mddev->recovery_cp > mddev->dev_sectors) {
3200		mddev->recovery_cp = mddev->dev_sectors;
3201		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
3202	}
3203	mddev->dev_sectors = sectors;
3204	mddev->resync_max_sectors = sectors;
3205	return 0;
3206}
3207
3208static int raid1_reshape(struct mddev *mddev)
3209{
3210	/* We need to:
3211	 * 1/ resize the r1bio_pool
3212	 * 2/ resize conf->mirrors
3213	 *
3214	 * We allocate a new r1bio_pool if we can.
3215	 * Then raise a device barrier and wait until all IO stops.
3216	 * Then resize conf->mirrors and swap in the new r1bio pool.
3217	 *
3218	 * At the same time, we "pack" the devices so that all the missing
3219	 * devices have the higher raid_disk numbers.
3220	 */
3221	mempool_t newpool, oldpool;
3222	struct pool_info *newpoolinfo;
3223	struct raid1_info *newmirrors;
3224	struct r1conf *conf = mddev->private;
3225	int cnt, raid_disks;
3226	unsigned long flags;
3227	int d, d2;
3228	int ret;
3229
3230	memset(&newpool, 0, sizeof(newpool));
3231	memset(&oldpool, 0, sizeof(oldpool));
3232
3233	/* Cannot change chunk_size, layout, or level */
3234	if (mddev->chunk_sectors != mddev->new_chunk_sectors ||
3235	    mddev->layout != mddev->new_layout ||
3236	    mddev->level != mddev->new_level) {
3237		mddev->new_chunk_sectors = mddev->chunk_sectors;
3238		mddev->new_layout = mddev->layout;
3239		mddev->new_level = mddev->level;
3240		return -EINVAL;
3241	}
3242
3243	if (!mddev_is_clustered(mddev))
3244		md_allow_write(mddev);
3245
3246	raid_disks = mddev->raid_disks + mddev->delta_disks;
3247
3248	if (raid_disks < conf->raid_disks) {
3249		cnt=0;
3250		for (d= 0; d < conf->raid_disks; d++)
3251			if (conf->mirrors[d].rdev)
3252				cnt++;
3253		if (cnt > raid_disks)
3254			return -EBUSY;
3255	}
3256
3257	newpoolinfo = kmalloc(sizeof(*newpoolinfo), GFP_KERNEL);
3258	if (!newpoolinfo)
3259		return -ENOMEM;
3260	newpoolinfo->mddev = mddev;
3261	newpoolinfo->raid_disks = raid_disks * 2;
3262
3263	ret = mempool_init(&newpool, NR_RAID_BIOS, r1bio_pool_alloc,
3264			   rbio_pool_free, newpoolinfo);
3265	if (ret) {
3266		kfree(newpoolinfo);
3267		return ret;
3268	}
3269	newmirrors = kzalloc(array3_size(sizeof(struct raid1_info),
3270					 raid_disks, 2),
3271			     GFP_KERNEL);
3272	if (!newmirrors) {
3273		kfree(newpoolinfo);
3274		mempool_exit(&newpool);
3275		return -ENOMEM;
3276	}
3277
3278	freeze_array(conf, 0);
3279
3280	/* ok, everything is stopped */
3281	oldpool = conf->r1bio_pool;
3282	conf->r1bio_pool = newpool;
3283
3284	for (d = d2 = 0; d < conf->raid_disks; d++) {
3285		struct md_rdev *rdev = conf->mirrors[d].rdev;
3286		if (rdev && rdev->raid_disk != d2) {
3287			sysfs_unlink_rdev(mddev, rdev);
3288			rdev->raid_disk = d2;
3289			sysfs_unlink_rdev(mddev, rdev);
3290			if (sysfs_link_rdev(mddev, rdev))
3291				pr_warn("md/raid1:%s: cannot register rd%d\n",
3292					mdname(mddev), rdev->raid_disk);
3293		}
3294		if (rdev)
3295			newmirrors[d2++].rdev = rdev;
3296	}
3297	kfree(conf->mirrors);
3298	conf->mirrors = newmirrors;
3299	kfree(conf->poolinfo);
3300	conf->poolinfo = newpoolinfo;
3301
3302	spin_lock_irqsave(&conf->device_lock, flags);
3303	mddev->degraded += (raid_disks - conf->raid_disks);
3304	spin_unlock_irqrestore(&conf->device_lock, flags);
3305	conf->raid_disks = mddev->raid_disks = raid_disks;
3306	mddev->delta_disks = 0;
3307
3308	unfreeze_array(conf);
3309
3310	set_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
3311	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
3312	md_wakeup_thread(mddev->thread);
3313
3314	mempool_exit(&oldpool);
3315	return 0;
3316}
3317
3318static void raid1_quiesce(struct mddev *mddev, int quiesce)
3319{
3320	struct r1conf *conf = mddev->private;
3321
3322	if (quiesce)
3323		freeze_array(conf, 0);
3324	else
3325		unfreeze_array(conf);
3326}
3327
3328static void *raid1_takeover(struct mddev *mddev)
3329{
3330	/* raid1 can take over:
3331	 *  raid5 with 2 devices, any layout or chunk size
3332	 */
3333	if (mddev->level == 5 && mddev->raid_disks == 2) {
3334		struct r1conf *conf;
3335		mddev->new_level = 1;
3336		mddev->new_layout = 0;
3337		mddev->new_chunk_sectors = 0;
3338		conf = setup_conf(mddev);
3339		if (!IS_ERR(conf)) {
3340			/* Array must appear to be quiesced */
3341			conf->array_frozen = 1;
3342			mddev_clear_unsupported_flags(mddev,
3343				UNSUPPORTED_MDDEV_FLAGS);
3344		}
3345		return conf;
3346	}
3347	return ERR_PTR(-EINVAL);
3348}
3349
3350static struct md_personality raid1_personality =
3351{
3352	.name		= "raid1",
3353	.level		= 1,
3354	.owner		= THIS_MODULE,
3355	.make_request	= raid1_make_request,
3356	.run		= raid1_run,
3357	.free		= raid1_free,
3358	.status		= raid1_status,
3359	.error_handler	= raid1_error,
3360	.hot_add_disk	= raid1_add_disk,
3361	.hot_remove_disk= raid1_remove_disk,
3362	.spare_active	= raid1_spare_active,
3363	.sync_request	= raid1_sync_request,
3364	.resize		= raid1_resize,
3365	.size		= raid1_size,
3366	.check_reshape	= raid1_reshape,
3367	.quiesce	= raid1_quiesce,
3368	.takeover	= raid1_takeover,
3369};
3370
3371static int __init raid_init(void)
3372{
3373	return register_md_personality(&raid1_personality);
3374}
3375
3376static void raid_exit(void)
3377{
3378	unregister_md_personality(&raid1_personality);
3379}
3380
3381module_init(raid_init);
3382module_exit(raid_exit);
3383MODULE_LICENSE("GPL");
3384MODULE_DESCRIPTION("RAID1 (mirroring) personality for MD");
3385MODULE_ALIAS("md-personality-3"); /* RAID1 */
3386MODULE_ALIAS("md-raid1");
3387MODULE_ALIAS("md-level-1");
3388
3389module_param(max_queued_requests, int, S_IRUGO|S_IWUSR);