1/*
   2 * raid5.c : Multiple Devices driver for Linux
   3 *	   Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
   4 *	   Copyright (C) 1999, 2000 Ingo Molnar
   5 *	   Copyright (C) 2002, 2003 H. Peter Anvin
   6 *
   7 * RAID-4/5/6 management functions.
   8 * Thanks to Penguin Computing for making the RAID-6 development possible
   9 * by donating a test server!
  10 *
  11 * This program is free software; you can redistribute it and/or modify
  12 * it under the terms of the GNU General Public License as published by
  13 * the Free Software Foundation; either version 2, or (at your option)
  14 * any later version.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * (for example /usr/src/linux/COPYING); if not, write to the Free
  18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 */
  20
  21/*
  22 * BITMAP UNPLUGGING:
  23 *
  24 * The sequencing for updating the bitmap reliably is a little
  25 * subtle (and I got it wrong the first time) so it deserves some
  26 * explanation.
  27 *
  28 * We group bitmap updates into batches.  Each batch has a number.
  29 * We may write out several batches at once, but that isn't very important.
  30 * conf->seq_write is the number of the last batch successfully written.
  31 * conf->seq_flush is the number of the last batch that was closed to
  32 *    new additions.
  33 * When we discover that we will need to write to any block in a stripe
  34 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
  35 * the number of the batch it will be in. This is seq_flush+1.
  36 * When we are ready to do a write, if that batch hasn't been written yet,
  37 *   we plug the array and queue the stripe for later.
  38 * When an unplug happens, we increment bm_flush, thus closing the current
  39 *   batch.
  40 * When we notice that bm_flush > bm_write, we write out all pending updates
  41 * to the bitmap, and advance bm_write to where bm_flush was.
  42 * This may occasionally write a bit out twice, but is sure never to
  43 * miss any bits.
  44 */
  45
  46#include <linux/blkdev.h>
  47#include <linux/kthread.h>
  48#include <linux/raid/pq.h>
  49#include <linux/async_tx.h>
  50#include <linux/module.h>
  51#include <linux/async.h>
  52#include <linux/seq_file.h>
  53#include <linux/cpu.h>
  54#include <linux/slab.h>
  55#include <linux/ratelimit.h>
  56#include <linux/nodemask.h>
 
  57#include <trace/events/block.h>
 
  58
  59#include "md.h"
  60#include "raid5.h"
  61#include "raid0.h"
  62#include "bitmap.h"
 
 
 
  63
  64#define cpu_to_group(cpu) cpu_to_node(cpu)
  65#define ANY_GROUP NUMA_NO_NODE
  66
 
 
 
 
  67static struct workqueue_struct *raid5_wq;
  68/*
  69 * Stripe cache
  70 */
  71
  72#define NR_STRIPES		256
  73#define STRIPE_SIZE		PAGE_SIZE
  74#define STRIPE_SHIFT		(PAGE_SHIFT - 9)
  75#define STRIPE_SECTORS		(STRIPE_SIZE>>9)
  76#define	IO_THRESHOLD		1
  77#define BYPASS_THRESHOLD	1
  78#define NR_HASH			(PAGE_SIZE / sizeof(struct hlist_head))
  79#define HASH_MASK		(NR_HASH - 1)
  80#define MAX_STRIPE_BATCH	8
  81
  82static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
  83{
  84	int hash = (sect >> STRIPE_SHIFT) & HASH_MASK;
  85	return &conf->stripe_hashtbl[hash];
  86}
  87
  88static inline int stripe_hash_locks_hash(sector_t sect)
  89{
  90	return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK;
  91}
  92
  93static inline void lock_device_hash_lock(struct r5conf *conf, int hash)
  94{
  95	spin_lock_irq(conf->hash_locks + hash);
  96	spin_lock(&conf->device_lock);
  97}
  98
  99static inline void unlock_device_hash_lock(struct r5conf *conf, int hash)
 100{
 101	spin_unlock(&conf->device_lock);
 102	spin_unlock_irq(conf->hash_locks + hash);
 103}
 104
 105static inline void lock_all_device_hash_locks_irq(struct r5conf *conf)
 106{
 107	int i;
 108	local_irq_disable();
 109	spin_lock(conf->hash_locks);
 110	for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
 111		spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks);
 112	spin_lock(&conf->device_lock);
 113}
 114
 115static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf)
 116{
 117	int i;
 118	spin_unlock(&conf->device_lock);
 119	for (i = NR_STRIPE_HASH_LOCKS; i; i--)
 120		spin_unlock(conf->hash_locks + i - 1);
 121	local_irq_enable();
 122}
 123
 124/* bio's attached to a stripe+device for I/O are linked together in bi_sector
 125 * order without overlap.  There may be several bio's per stripe+device, and
 126 * a bio could span several devices.
 127 * When walking this list for a particular stripe+device, we must never proceed
 128 * beyond a bio that extends past this device, as the next bio might no longer
 129 * be valid.
 130 * This function is used to determine the 'next' bio in the list, given the sector
 131 * of the current stripe+device
 132 */
 133static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector)
 134{
 135	int sectors = bio_sectors(bio);
 136	if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS)
 137		return bio->bi_next;
 138	else
 139		return NULL;
 140}
 141
 142/*
 143 * We maintain a biased count of active stripes in the bottom 16 bits of
 144 * bi_phys_segments, and a count of processed stripes in the upper 16 bits
 145 */
 146static inline int raid5_bi_processed_stripes(struct bio *bio)
 147{
 148	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
 149	return (atomic_read(segments) >> 16) & 0xffff;
 150}
 151
 152static inline int raid5_dec_bi_active_stripes(struct bio *bio)
 153{
 154	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
 155	return atomic_sub_return(1, segments) & 0xffff;
 156}
 157
 158static inline void raid5_inc_bi_active_stripes(struct bio *bio)
 159{
 160	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
 161	atomic_inc(segments);
 162}
 163
 164static inline void raid5_set_bi_processed_stripes(struct bio *bio,
 165	unsigned int cnt)
 166{
 167	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
 168	int old, new;
 169
 170	do {
 171		old = atomic_read(segments);
 172		new = (old & 0xffff) | (cnt << 16);
 173	} while (atomic_cmpxchg(segments, old, new) != old);
 174}
 175
 176static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt)
 177{
 178	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
 179	atomic_set(segments, cnt);
 180}
 181
 182/* Find first data disk in a raid6 stripe */
 183static inline int raid6_d0(struct stripe_head *sh)
 184{
 185	if (sh->ddf_layout)
 186		/* ddf always start from first device */
 187		return 0;
 188	/* md starts just after Q block */
 189	if (sh->qd_idx == sh->disks - 1)
 190		return 0;
 191	else
 192		return sh->qd_idx + 1;
 193}
 194static inline int raid6_next_disk(int disk, int raid_disks)
 195{
 196	disk++;
 197	return (disk < raid_disks) ? disk : 0;
 198}
 199
 200/* When walking through the disks in a raid5, starting at raid6_d0,
 201 * We need to map each disk to a 'slot', where the data disks are slot
 202 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
 203 * is raid_disks-1.  This help does that mapping.
 204 */
 205static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
 206			     int *count, int syndrome_disks)
 207{
 208	int slot = *count;
 209
 210	if (sh->ddf_layout)
 211		(*count)++;
 212	if (idx == sh->pd_idx)
 213		return syndrome_disks;
 214	if (idx == sh->qd_idx)
 215		return syndrome_disks + 1;
 216	if (!sh->ddf_layout)
 217		(*count)++;
 218	return slot;
 219}
 220
 221static void return_io(struct bio *return_bi)
 222{
 223	struct bio *bi = return_bi;
 224	while (bi) {
 225
 226		return_bi = bi->bi_next;
 227		bi->bi_next = NULL;
 228		bi->bi_iter.bi_size = 0;
 229		trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
 230					 bi, 0);
 231		bio_endio(bi, 0);
 232		bi = return_bi;
 233	}
 234}
 235
 236static void print_raid5_conf (struct r5conf *conf);
 237
 238static int stripe_operations_active(struct stripe_head *sh)
 239{
 240	return sh->check_state || sh->reconstruct_state ||
 241	       test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
 242	       test_bit(STRIPE_COMPUTE_RUN, &sh->state);
 243}
 244
 
 
 
 
 
 
 
 245static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
 246{
 247	struct r5conf *conf = sh->raid_conf;
 248	struct r5worker_group *group;
 249	int thread_cnt;
 250	int i, cpu = sh->cpu;
 251
 252	if (!cpu_online(cpu)) {
 253		cpu = cpumask_any(cpu_online_mask);
 254		sh->cpu = cpu;
 255	}
 256
 257	if (list_empty(&sh->lru)) {
 258		struct r5worker_group *group;
 259		group = conf->worker_groups + cpu_to_group(cpu);
 260		list_add_tail(&sh->lru, &group->handle_list);
 
 
 
 261		group->stripes_cnt++;
 262		sh->group = group;
 263	}
 264
 265	if (conf->worker_cnt_per_group == 0) {
 266		md_wakeup_thread(conf->mddev->thread);
 267		return;
 268	}
 269
 270	group = conf->worker_groups + cpu_to_group(sh->cpu);
 271
 272	group->workers[0].working = true;
 273	/* at least one worker should run to avoid race */
 274	queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);
 275
 276	thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1;
 277	/* wakeup more workers */
 278	for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) {
 279		if (group->workers[i].working == false) {
 280			group->workers[i].working = true;
 281			queue_work_on(sh->cpu, raid5_wq,
 282				      &group->workers[i].work);
 283			thread_cnt--;
 284		}
 285	}
 286}
 287
 288static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,
 289			      struct list_head *temp_inactive_list)
 290{
 
 
 
 291	BUG_ON(!list_empty(&sh->lru));
 292	BUG_ON(atomic_read(&conf->active_stripes)==0);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 293	if (test_bit(STRIPE_HANDLE, &sh->state)) {
 294		if (test_bit(STRIPE_DELAYED, &sh->state) &&
 295		    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
 296			list_add_tail(&sh->lru, &conf->delayed_list);
 297		else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
 298			   sh->bm_seq - conf->seq_write > 0)
 299			list_add_tail(&sh->lru, &conf->bitmap_list);
 300		else {
 301			clear_bit(STRIPE_DELAYED, &sh->state);
 302			clear_bit(STRIPE_BIT_DELAY, &sh->state);
 303			if (conf->worker_cnt_per_group == 0) {
 304				list_add_tail(&sh->lru, &conf->handle_list);
 
 
 
 
 
 305			} else {
 306				raid5_wakeup_stripe_thread(sh);
 307				return;
 308			}
 309		}
 310		md_wakeup_thread(conf->mddev->thread);
 311	} else {
 312		BUG_ON(stripe_operations_active(sh));
 313		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
 314			if (atomic_dec_return(&conf->preread_active_stripes)
 315			    < IO_THRESHOLD)
 316				md_wakeup_thread(conf->mddev->thread);
 317		atomic_dec(&conf->active_stripes);
 318		if (!test_bit(STRIPE_EXPANDING, &sh->state))
 319			list_add_tail(&sh->lru, temp_inactive_list);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 320	}
 321}
 322
 323static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,
 324			     struct list_head *temp_inactive_list)
 325{
 326	if (atomic_dec_and_test(&sh->count))
 327		do_release_stripe(conf, sh, temp_inactive_list);
 328}
 329
 330/*
 331 * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list
 332 *
 333 * Be careful: Only one task can add/delete stripes from temp_inactive_list at
 334 * given time. Adding stripes only takes device lock, while deleting stripes
 335 * only takes hash lock.
 336 */
 337static void release_inactive_stripe_list(struct r5conf *conf,
 338					 struct list_head *temp_inactive_list,
 339					 int hash)
 340{
 341	int size;
 342	bool do_wakeup = false;
 343	unsigned long flags;
 344
 345	if (hash == NR_STRIPE_HASH_LOCKS) {
 346		size = NR_STRIPE_HASH_LOCKS;
 347		hash = NR_STRIPE_HASH_LOCKS - 1;
 348	} else
 349		size = 1;
 350	while (size) {
 351		struct list_head *list = &temp_inactive_list[size - 1];
 352
 353		/*
 354		 * We don't hold any lock here yet, get_active_stripe() might
 355		 * remove stripes from the list
 356		 */
 357		if (!list_empty_careful(list)) {
 358			spin_lock_irqsave(conf->hash_locks + hash, flags);
 359			if (list_empty(conf->inactive_list + hash) &&
 360			    !list_empty(list))
 361				atomic_dec(&conf->empty_inactive_list_nr);
 362			list_splice_tail_init(list, conf->inactive_list + hash);
 363			do_wakeup = true;
 364			spin_unlock_irqrestore(conf->hash_locks + hash, flags);
 365		}
 366		size--;
 367		hash--;
 368	}
 369
 370	if (do_wakeup) {
 371		wake_up(&conf->wait_for_stripe);
 
 
 372		if (conf->retry_read_aligned)
 373			md_wakeup_thread(conf->mddev->thread);
 374	}
 375}
 376
 377/* should hold conf->device_lock already */
 378static int release_stripe_list(struct r5conf *conf,
 379			       struct list_head *temp_inactive_list)
 380{
 381	struct stripe_head *sh;
 382	int count = 0;
 383	struct llist_node *head;
 384
 385	head = llist_del_all(&conf->released_stripes);
 386	head = llist_reverse_order(head);
 387	while (head) {
 388		int hash;
 389
 390		sh = llist_entry(head, struct stripe_head, release_list);
 391		head = llist_next(head);
 392		/* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */
 393		smp_mb();
 394		clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);
 395		/*
 396		 * Don't worry the bit is set here, because if the bit is set
 397		 * again, the count is always > 1. This is true for
 398		 * STRIPE_ON_UNPLUG_LIST bit too.
 399		 */
 400		hash = sh->hash_lock_index;
 401		__release_stripe(conf, sh, &temp_inactive_list[hash]);
 402		count++;
 403	}
 404
 405	return count;
 406}
 407
 408static void release_stripe(struct stripe_head *sh)
 409{
 410	struct r5conf *conf = sh->raid_conf;
 411	unsigned long flags;
 412	struct list_head list;
 413	int hash;
 414	bool wakeup;
 415
 
 
 
 
 
 416	if (unlikely(!conf->mddev->thread) ||
 417		test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))
 418		goto slow_path;
 419	wakeup = llist_add(&sh->release_list, &conf->released_stripes);
 420	if (wakeup)
 421		md_wakeup_thread(conf->mddev->thread);
 422	return;
 423slow_path:
 424	local_irq_save(flags);
 425	/* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */
 426	if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) {
 427		INIT_LIST_HEAD(&list);
 428		hash = sh->hash_lock_index;
 429		do_release_stripe(conf, sh, &list);
 430		spin_unlock(&conf->device_lock);
 431		release_inactive_stripe_list(conf, &list, hash);
 432	}
 433	local_irq_restore(flags);
 434}
 435
 436static inline void remove_hash(struct stripe_head *sh)
 437{
 438	pr_debug("remove_hash(), stripe %llu\n",
 439		(unsigned long long)sh->sector);
 440
 441	hlist_del_init(&sh->hash);
 442}
 443
 444static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
 445{
 446	struct hlist_head *hp = stripe_hash(conf, sh->sector);
 447
 448	pr_debug("insert_hash(), stripe %llu\n",
 449		(unsigned long long)sh->sector);
 450
 451	hlist_add_head(&sh->hash, hp);
 452}
 453
 454
 455/* find an idle stripe, make sure it is unhashed, and return it. */
 456static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash)
 457{
 458	struct stripe_head *sh = NULL;
 459	struct list_head *first;
 460
 461	if (list_empty(conf->inactive_list + hash))
 462		goto out;
 463	first = (conf->inactive_list + hash)->next;
 464	sh = list_entry(first, struct stripe_head, lru);
 465	list_del_init(first);
 466	remove_hash(sh);
 467	atomic_inc(&conf->active_stripes);
 468	BUG_ON(hash != sh->hash_lock_index);
 469	if (list_empty(conf->inactive_list + hash))
 470		atomic_inc(&conf->empty_inactive_list_nr);
 471out:
 472	return sh;
 473}
 474
 475static void shrink_buffers(struct stripe_head *sh)
 476{
 477	struct page *p;
 478	int i;
 479	int num = sh->raid_conf->pool_size;
 480
 481	for (i = 0; i < num ; i++) {
 
 482		p = sh->dev[i].page;
 483		if (!p)
 484			continue;
 485		sh->dev[i].page = NULL;
 486		put_page(p);
 487	}
 488}
 489
 490static int grow_buffers(struct stripe_head *sh)
 491{
 492	int i;
 493	int num = sh->raid_conf->pool_size;
 494
 495	for (i = 0; i < num; i++) {
 496		struct page *page;
 497
 498		if (!(page = alloc_page(GFP_KERNEL))) {
 499			return 1;
 500		}
 501		sh->dev[i].page = page;
 
 502	}
 
 503	return 0;
 504}
 505
 506static void raid5_build_block(struct stripe_head *sh, int i, int previous);
 507static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
 508			    struct stripe_head *sh);
 509
 510static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
 511{
 512	struct r5conf *conf = sh->raid_conf;
 513	int i, seq;
 514
 515	BUG_ON(atomic_read(&sh->count) != 0);
 516	BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
 517	BUG_ON(stripe_operations_active(sh));
 
 518
 519	pr_debug("init_stripe called, stripe %llu\n",
 520		(unsigned long long)sh->sector);
 521
 522	remove_hash(sh);
 523retry:
 524	seq = read_seqcount_begin(&conf->gen_lock);
 525	sh->generation = conf->generation - previous;
 526	sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
 527	sh->sector = sector;
 528	stripe_set_idx(sector, conf, previous, sh);
 529	sh->state = 0;
 530
 531
 532	for (i = sh->disks; i--; ) {
 533		struct r5dev *dev = &sh->dev[i];
 534
 535		if (dev->toread || dev->read || dev->towrite || dev->written ||
 536		    test_bit(R5_LOCKED, &dev->flags)) {
 537			printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
 538			       (unsigned long long)sh->sector, i, dev->toread,
 539			       dev->read, dev->towrite, dev->written,
 540			       test_bit(R5_LOCKED, &dev->flags));
 541			WARN_ON(1);
 542		}
 543		dev->flags = 0;
 544		raid5_build_block(sh, i, previous);
 545	}
 546	if (read_seqcount_retry(&conf->gen_lock, seq))
 547		goto retry;
 
 548	insert_hash(conf, sh);
 549	sh->cpu = smp_processor_id();
 
 550}
 551
 552static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,
 553					 short generation)
 554{
 555	struct stripe_head *sh;
 556
 557	pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
 558	hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)
 559		if (sh->sector == sector && sh->generation == generation)
 560			return sh;
 561	pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
 562	return NULL;
 563}
 564
 565/*
 566 * Need to check if array has failed when deciding whether to:
 567 *  - start an array
 568 *  - remove non-faulty devices
 569 *  - add a spare
 570 *  - allow a reshape
 571 * This determination is simple when no reshape is happening.
 572 * However if there is a reshape, we need to carefully check
 573 * both the before and after sections.
 574 * This is because some failed devices may only affect one
 575 * of the two sections, and some non-in_sync devices may
 576 * be insync in the section most affected by failed devices.
 577 */
 578static int calc_degraded(struct r5conf *conf)
 579{
 580	int degraded, degraded2;
 581	int i;
 582
 583	rcu_read_lock();
 584	degraded = 0;
 585	for (i = 0; i < conf->previous_raid_disks; i++) {
 586		struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
 587		if (rdev && test_bit(Faulty, &rdev->flags))
 588			rdev = rcu_dereference(conf->disks[i].replacement);
 589		if (!rdev || test_bit(Faulty, &rdev->flags))
 590			degraded++;
 591		else if (test_bit(In_sync, &rdev->flags))
 592			;
 593		else
 594			/* not in-sync or faulty.
 595			 * If the reshape increases the number of devices,
 596			 * this is being recovered by the reshape, so
 597			 * this 'previous' section is not in_sync.
 598			 * If the number of devices is being reduced however,
 599			 * the device can only be part of the array if
 600			 * we are reverting a reshape, so this section will
 601			 * be in-sync.
 602			 */
 603			if (conf->raid_disks >= conf->previous_raid_disks)
 604				degraded++;
 605	}
 606	rcu_read_unlock();
 607	if (conf->raid_disks == conf->previous_raid_disks)
 608		return degraded;
 609	rcu_read_lock();
 610	degraded2 = 0;
 611	for (i = 0; i < conf->raid_disks; i++) {
 612		struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
 613		if (rdev && test_bit(Faulty, &rdev->flags))
 614			rdev = rcu_dereference(conf->disks[i].replacement);
 615		if (!rdev || test_bit(Faulty, &rdev->flags))
 616			degraded2++;
 617		else if (test_bit(In_sync, &rdev->flags))
 618			;
 619		else
 620			/* not in-sync or faulty.
 621			 * If reshape increases the number of devices, this
 622			 * section has already been recovered, else it
 623			 * almost certainly hasn't.
 624			 */
 625			if (conf->raid_disks <= conf->previous_raid_disks)
 626				degraded2++;
 627	}
 628	rcu_read_unlock();
 629	if (degraded2 > degraded)
 630		return degraded2;
 631	return degraded;
 632}
 633
 634static int has_failed(struct r5conf *conf)
 635{
 636	int degraded;
 637
 638	if (conf->mddev->reshape_position == MaxSector)
 639		return conf->mddev->degraded > conf->max_degraded;
 640
 641	degraded = calc_degraded(conf);
 642	if (degraded > conf->max_degraded)
 643		return 1;
 644	return 0;
 645}
 646
 647static struct stripe_head *
 648get_active_stripe(struct r5conf *conf, sector_t sector,
 649		  int previous, int noblock, int noquiesce)
 650{
 651	struct stripe_head *sh;
 652	int hash = stripe_hash_locks_hash(sector);
 
 653
 654	pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
 655
 656	spin_lock_irq(conf->hash_locks + hash);
 657
 658	do {
 659		wait_event_lock_irq(conf->wait_for_stripe,
 660				    conf->quiesce == 0 || noquiesce,
 661				    *(conf->hash_locks + hash));
 662		sh = __find_stripe(conf, sector, conf->generation - previous);
 663		if (!sh) {
 664			if (!conf->inactive_blocked)
 665				sh = get_free_stripe(conf, hash);
 
 
 
 
 
 666			if (noblock && sh == NULL)
 667				break;
 
 
 668			if (!sh) {
 669				conf->inactive_blocked = 1;
 
 
 670				wait_event_lock_irq(
 671					conf->wait_for_stripe,
 672					!list_empty(conf->inactive_list + hash) &&
 673					(atomic_read(&conf->active_stripes)
 674					 < (conf->max_nr_stripes * 3 / 4)
 675					 || !conf->inactive_blocked),
 
 676					*(conf->hash_locks + hash));
 677				conf->inactive_blocked = 0;
 
 678			} else {
 679				init_stripe(sh, sector, previous);
 680				atomic_inc(&sh->count);
 681			}
 682		} else if (!atomic_inc_not_zero(&sh->count)) {
 683			spin_lock(&conf->device_lock);
 684			if (!atomic_read(&sh->count)) {
 685				if (!test_bit(STRIPE_HANDLE, &sh->state))
 686					atomic_inc(&conf->active_stripes);
 687				BUG_ON(list_empty(&sh->lru) &&
 688				       !test_bit(STRIPE_EXPANDING, &sh->state));
 
 
 
 689				list_del_init(&sh->lru);
 
 
 690				if (sh->group) {
 691					sh->group->stripes_cnt--;
 692					sh->group = NULL;
 693				}
 694			}
 695			atomic_inc(&sh->count);
 696			spin_unlock(&conf->device_lock);
 697		}
 698	} while (sh == NULL);
 699
 700	spin_unlock_irq(conf->hash_locks + hash);
 701	return sh;
 702}
 703
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 704/* Determine if 'data_offset' or 'new_data_offset' should be used
 705 * in this stripe_head.
 706 */
 707static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
 708{
 709	sector_t progress = conf->reshape_progress;
 710	/* Need a memory barrier to make sure we see the value
 711	 * of conf->generation, or ->data_offset that was set before
 712	 * reshape_progress was updated.
 713	 */
 714	smp_rmb();
 715	if (progress == MaxSector)
 716		return 0;
 717	if (sh->generation == conf->generation - 1)
 718		return 0;
 719	/* We are in a reshape, and this is a new-generation stripe,
 720	 * so use new_data_offset.
 721	 */
 722	return 1;
 723}
 724
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 725static void
 726raid5_end_read_request(struct bio *bi, int error);
 727static void
 728raid5_end_write_request(struct bio *bi, int error);
 729
 730static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
 731{
 732	struct r5conf *conf = sh->raid_conf;
 733	int i, disks = sh->disks;
 
 
 
 734
 735	might_sleep();
 736
 
 
 
 
 
 737	for (i = disks; i--; ) {
 738		int rw;
 739		int replace_only = 0;
 740		struct bio *bi, *rbi;
 741		struct md_rdev *rdev, *rrdev = NULL;
 
 
 742		if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
 
 743			if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
 744				rw = WRITE_FUA;
 745			else
 746				rw = WRITE;
 747			if (test_bit(R5_Discard, &sh->dev[i].flags))
 748				rw |= REQ_DISCARD;
 749		} else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
 750			rw = READ;
 751		else if (test_and_clear_bit(R5_WantReplace,
 752					    &sh->dev[i].flags)) {
 753			rw = WRITE;
 754			replace_only = 1;
 755		} else
 756			continue;
 757		if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
 758			rw |= REQ_SYNC;
 759
 
 760		bi = &sh->dev[i].req;
 761		rbi = &sh->dev[i].rreq; /* For writing to replacement */
 762
 763		rcu_read_lock();
 764		rrdev = rcu_dereference(conf->disks[i].replacement);
 765		smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */
 766		rdev = rcu_dereference(conf->disks[i].rdev);
 767		if (!rdev) {
 768			rdev = rrdev;
 769			rrdev = NULL;
 770		}
 771		if (rw & WRITE) {
 772			if (replace_only)
 773				rdev = NULL;
 774			if (rdev == rrdev)
 775				/* We raced and saw duplicates */
 776				rrdev = NULL;
 777		} else {
 778			if (test_bit(R5_ReadRepl, &sh->dev[i].flags) && rrdev)
 779				rdev = rrdev;
 780			rrdev = NULL;
 781		}
 782
 783		if (rdev && test_bit(Faulty, &rdev->flags))
 784			rdev = NULL;
 785		if (rdev)
 786			atomic_inc(&rdev->nr_pending);
 787		if (rrdev && test_bit(Faulty, &rrdev->flags))
 788			rrdev = NULL;
 789		if (rrdev)
 790			atomic_inc(&rrdev->nr_pending);
 791		rcu_read_unlock();
 792
 793		/* We have already checked bad blocks for reads.  Now
 794		 * need to check for writes.  We never accept write errors
 795		 * on the replacement, so we don't to check rrdev.
 796		 */
 797		while ((rw & WRITE) && rdev &&
 798		       test_bit(WriteErrorSeen, &rdev->flags)) {
 799			sector_t first_bad;
 800			int bad_sectors;
 801			int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
 802					      &first_bad, &bad_sectors);
 803			if (!bad)
 804				break;
 805
 806			if (bad < 0) {
 807				set_bit(BlockedBadBlocks, &rdev->flags);
 808				if (!conf->mddev->external &&
 809				    conf->mddev->flags) {
 810					/* It is very unlikely, but we might
 811					 * still need to write out the
 812					 * bad block log - better give it
 813					 * a chance*/
 814					md_check_recovery(conf->mddev);
 815				}
 816				/*
 817				 * Because md_wait_for_blocked_rdev
 818				 * will dec nr_pending, we must
 819				 * increment it first.
 820				 */
 821				atomic_inc(&rdev->nr_pending);
 822				md_wait_for_blocked_rdev(rdev, conf->mddev);
 823			} else {
 824				/* Acknowledged bad block - skip the write */
 825				rdev_dec_pending(rdev, conf->mddev);
 826				rdev = NULL;
 827			}
 828		}
 829
 830		if (rdev) {
 831			if (s->syncing || s->expanding || s->expanded
 832			    || s->replacing)
 833				md_sync_acct(rdev->bdev, STRIPE_SECTORS);
 834
 835			set_bit(STRIPE_IO_STARTED, &sh->state);
 836
 837			bio_reset(bi);
 838			bi->bi_bdev = rdev->bdev;
 839			bi->bi_rw = rw;
 840			bi->bi_end_io = (rw & WRITE)
 841				? raid5_end_write_request
 842				: raid5_end_read_request;
 843			bi->bi_private = sh;
 844
 845			pr_debug("%s: for %llu schedule op %ld on disc %d\n",
 846				__func__, (unsigned long long)sh->sector,
 847				bi->bi_rw, i);
 848			atomic_inc(&sh->count);
 
 
 849			if (use_new_offset(conf, sh))
 850				bi->bi_iter.bi_sector = (sh->sector
 851						 + rdev->new_data_offset);
 852			else
 853				bi->bi_iter.bi_sector = (sh->sector
 854						 + rdev->data_offset);
 855			if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
 856				bi->bi_rw |= REQ_NOMERGE;
 857
 
 
 
 
 
 
 
 
 
 
 
 
 
 858			bi->bi_vcnt = 1;
 859			bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
 860			bi->bi_io_vec[0].bv_offset = 0;
 861			bi->bi_iter.bi_size = STRIPE_SIZE;
 
 
 
 862			/*
 863			 * If this is discard request, set bi_vcnt 0. We don't
 864			 * want to confuse SCSI because SCSI will replace payload
 865			 */
 866			if (rw & REQ_DISCARD)
 867				bi->bi_vcnt = 0;
 868			if (rrdev)
 869				set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
 870
 871			if (conf->mddev->gendisk)
 872				trace_block_bio_remap(bdev_get_queue(bi->bi_bdev),
 873						      bi, disk_devt(conf->mddev->gendisk),
 874						      sh->dev[i].sector);
 875			generic_make_request(bi);
 
 
 
 876		}
 877		if (rrdev) {
 878			if (s->syncing || s->expanding || s->expanded
 879			    || s->replacing)
 880				md_sync_acct(rrdev->bdev, STRIPE_SECTORS);
 881
 882			set_bit(STRIPE_IO_STARTED, &sh->state);
 883
 884			bio_reset(rbi);
 885			rbi->bi_bdev = rrdev->bdev;
 886			rbi->bi_rw = rw;
 887			BUG_ON(!(rw & WRITE));
 888			rbi->bi_end_io = raid5_end_write_request;
 889			rbi->bi_private = sh;
 890
 891			pr_debug("%s: for %llu schedule op %ld on "
 892				 "replacement disc %d\n",
 893				__func__, (unsigned long long)sh->sector,
 894				rbi->bi_rw, i);
 895			atomic_inc(&sh->count);
 
 
 896			if (use_new_offset(conf, sh))
 897				rbi->bi_iter.bi_sector = (sh->sector
 898						  + rrdev->new_data_offset);
 899			else
 900				rbi->bi_iter.bi_sector = (sh->sector
 901						  + rrdev->data_offset);
 
 
 
 902			rbi->bi_vcnt = 1;
 903			rbi->bi_io_vec[0].bv_len = STRIPE_SIZE;
 904			rbi->bi_io_vec[0].bv_offset = 0;
 905			rbi->bi_iter.bi_size = STRIPE_SIZE;
 
 
 906			/*
 907			 * If this is discard request, set bi_vcnt 0. We don't
 908			 * want to confuse SCSI because SCSI will replace payload
 909			 */
 910			if (rw & REQ_DISCARD)
 911				rbi->bi_vcnt = 0;
 912			if (conf->mddev->gendisk)
 913				trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev),
 914						      rbi, disk_devt(conf->mddev->gendisk),
 915						      sh->dev[i].sector);
 916			generic_make_request(rbi);
 
 
 
 917		}
 918		if (!rdev && !rrdev) {
 919			if (rw & WRITE)
 920				set_bit(STRIPE_DEGRADED, &sh->state);
 921			pr_debug("skip op %ld on disc %d for sector %llu\n",
 922				bi->bi_rw, i, (unsigned long long)sh->sector);
 923			clear_bit(R5_LOCKED, &sh->dev[i].flags);
 924			set_bit(STRIPE_HANDLE, &sh->state);
 925		}
 
 
 
 
 
 
 
 926	}
 
 
 
 927}
 928
 929static struct dma_async_tx_descriptor *
 930async_copy_data(int frombio, struct bio *bio, struct page *page,
 931	sector_t sector, struct dma_async_tx_descriptor *tx)
 
 932{
 933	struct bio_vec bvl;
 934	struct bvec_iter iter;
 935	struct page *bio_page;
 936	int page_offset;
 937	struct async_submit_ctl submit;
 938	enum async_tx_flags flags = 0;
 939
 940	if (bio->bi_iter.bi_sector >= sector)
 941		page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512;
 942	else
 943		page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512;
 944
 945	if (frombio)
 946		flags |= ASYNC_TX_FENCE;
 947	init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
 948
 949	bio_for_each_segment(bvl, bio, iter) {
 950		int len = bvl.bv_len;
 951		int clen;
 952		int b_offset = 0;
 953
 954		if (page_offset < 0) {
 955			b_offset = -page_offset;
 956			page_offset += b_offset;
 957			len -= b_offset;
 958		}
 959
 960		if (len > 0 && page_offset + len > STRIPE_SIZE)
 961			clen = STRIPE_SIZE - page_offset;
 962		else
 963			clen = len;
 964
 965		if (clen > 0) {
 966			b_offset += bvl.bv_offset;
 967			bio_page = bvl.bv_page;
 968			if (frombio)
 969				tx = async_memcpy(page, bio_page, page_offset,
 
 
 
 
 
 
 970						  b_offset, clen, &submit);
 971			else
 972				tx = async_memcpy(bio_page, page, b_offset,
 973						  page_offset, clen, &submit);
 974		}
 975		/* chain the operations */
 976		submit.depend_tx = tx;
 977
 978		if (clen < len) /* hit end of page */
 979			break;
 980		page_offset +=  len;
 981	}
 982
 983	return tx;
 984}
 985
 986static void ops_complete_biofill(void *stripe_head_ref)
 987{
 988	struct stripe_head *sh = stripe_head_ref;
 989	struct bio *return_bi = NULL;
 990	int i;
 991
 992	pr_debug("%s: stripe %llu\n", __func__,
 993		(unsigned long long)sh->sector);
 994
 995	/* clear completed biofills */
 996	for (i = sh->disks; i--; ) {
 997		struct r5dev *dev = &sh->dev[i];
 998
 999		/* acknowledge completion of a biofill operation */
1000		/* and check if we need to reply to a read request,
1001		 * new R5_Wantfill requests are held off until
1002		 * !STRIPE_BIOFILL_RUN
1003		 */
1004		if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
1005			struct bio *rbi, *rbi2;
1006
1007			BUG_ON(!dev->read);
1008			rbi = dev->read;
1009			dev->read = NULL;
1010			while (rbi && rbi->bi_iter.bi_sector <
1011				dev->sector + STRIPE_SECTORS) {
1012				rbi2 = r5_next_bio(rbi, dev->sector);
1013				if (!raid5_dec_bi_active_stripes(rbi)) {
1014					rbi->bi_next = return_bi;
1015					return_bi = rbi;
1016				}
1017				rbi = rbi2;
1018			}
1019		}
1020	}
1021	clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
1022
1023	return_io(return_bi);
1024
1025	set_bit(STRIPE_HANDLE, &sh->state);
1026	release_stripe(sh);
1027}
1028
1029static void ops_run_biofill(struct stripe_head *sh)
1030{
1031	struct dma_async_tx_descriptor *tx = NULL;
1032	struct async_submit_ctl submit;
1033	int i;
1034
 
1035	pr_debug("%s: stripe %llu\n", __func__,
1036		(unsigned long long)sh->sector);
1037
1038	for (i = sh->disks; i--; ) {
1039		struct r5dev *dev = &sh->dev[i];
1040		if (test_bit(R5_Wantfill, &dev->flags)) {
1041			struct bio *rbi;
1042			spin_lock_irq(&sh->stripe_lock);
1043			dev->read = rbi = dev->toread;
1044			dev->toread = NULL;
1045			spin_unlock_irq(&sh->stripe_lock);
1046			while (rbi && rbi->bi_iter.bi_sector <
1047				dev->sector + STRIPE_SECTORS) {
1048				tx = async_copy_data(0, rbi, dev->page,
1049					dev->sector, tx);
1050				rbi = r5_next_bio(rbi, dev->sector);
1051			}
1052		}
1053	}
1054
1055	atomic_inc(&sh->count);
1056	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
1057	async_trigger_callback(&submit);
1058}
1059
1060static void mark_target_uptodate(struct stripe_head *sh, int target)
1061{
1062	struct r5dev *tgt;
1063
1064	if (target < 0)
1065		return;
1066
1067	tgt = &sh->dev[target];
1068	set_bit(R5_UPTODATE, &tgt->flags);
1069	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1070	clear_bit(R5_Wantcompute, &tgt->flags);
1071}
1072
1073static void ops_complete_compute(void *stripe_head_ref)
1074{
1075	struct stripe_head *sh = stripe_head_ref;
1076
1077	pr_debug("%s: stripe %llu\n", __func__,
1078		(unsigned long long)sh->sector);
1079
1080	/* mark the computed target(s) as uptodate */
1081	mark_target_uptodate(sh, sh->ops.target);
1082	mark_target_uptodate(sh, sh->ops.target2);
1083
1084	clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
1085	if (sh->check_state == check_state_compute_run)
1086		sh->check_state = check_state_compute_result;
1087	set_bit(STRIPE_HANDLE, &sh->state);
1088	release_stripe(sh);
 
 
 
 
 
 
1089}
1090
1091/* return a pointer to the address conversion region of the scribble buffer */
1092static addr_conv_t *to_addr_conv(struct stripe_head *sh,
1093				 struct raid5_percpu *percpu)
1094{
1095	return percpu->scribble + sizeof(struct page *) * (sh->disks + 2);
1096}
1097
1098static struct dma_async_tx_descriptor *
1099ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
1100{
1101	int disks = sh->disks;
1102	struct page **xor_srcs = percpu->scribble;
1103	int target = sh->ops.target;
1104	struct r5dev *tgt = &sh->dev[target];
1105	struct page *xor_dest = tgt->page;
1106	int count = 0;
1107	struct dma_async_tx_descriptor *tx;
1108	struct async_submit_ctl submit;
1109	int i;
1110
 
 
1111	pr_debug("%s: stripe %llu block: %d\n",
1112		__func__, (unsigned long long)sh->sector, target);
1113	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1114
1115	for (i = disks; i--; )
1116		if (i != target)
1117			xor_srcs[count++] = sh->dev[i].page;
1118
1119	atomic_inc(&sh->count);
1120
1121	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
1122			  ops_complete_compute, sh, to_addr_conv(sh, percpu));
1123	if (unlikely(count == 1))
1124		tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
1125	else
1126		tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1127
1128	return tx;
1129}
1130
1131/* set_syndrome_sources - populate source buffers for gen_syndrome
1132 * @srcs - (struct page *) array of size sh->disks
1133 * @sh - stripe_head to parse
1134 *
1135 * Populates srcs in proper layout order for the stripe and returns the
1136 * 'count' of sources to be used in a call to async_gen_syndrome.  The P
1137 * destination buffer is recorded in srcs[count] and the Q destination
1138 * is recorded in srcs[count+1]].
1139 */
1140static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh)
 
 
1141{
1142	int disks = sh->disks;
1143	int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
1144	int d0_idx = raid6_d0(sh);
1145	int count;
1146	int i;
1147
1148	for (i = 0; i < disks; i++)
1149		srcs[i] = NULL;
1150
1151	count = 0;
1152	i = d0_idx;
1153	do {
1154		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
 
1155
1156		srcs[slot] = sh->dev[i].page;
 
 
 
 
 
 
 
 
 
 
 
 
1157		i = raid6_next_disk(i, disks);
1158	} while (i != d0_idx);
1159
1160	return syndrome_disks;
1161}
1162
1163static struct dma_async_tx_descriptor *
1164ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
1165{
1166	int disks = sh->disks;
1167	struct page **blocks = percpu->scribble;
1168	int target;
1169	int qd_idx = sh->qd_idx;
1170	struct dma_async_tx_descriptor *tx;
1171	struct async_submit_ctl submit;
1172	struct r5dev *tgt;
1173	struct page *dest;
1174	int i;
1175	int count;
1176
 
1177	if (sh->ops.target < 0)
1178		target = sh->ops.target2;
1179	else if (sh->ops.target2 < 0)
1180		target = sh->ops.target;
1181	else
1182		/* we should only have one valid target */
1183		BUG();
1184	BUG_ON(target < 0);
1185	pr_debug("%s: stripe %llu block: %d\n",
1186		__func__, (unsigned long long)sh->sector, target);
1187
1188	tgt = &sh->dev[target];
1189	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1190	dest = tgt->page;
1191
1192	atomic_inc(&sh->count);
1193
1194	if (target == qd_idx) {
1195		count = set_syndrome_sources(blocks, sh);
1196		blocks[count] = NULL; /* regenerating p is not necessary */
1197		BUG_ON(blocks[count+1] != dest); /* q should already be set */
1198		init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1199				  ops_complete_compute, sh,
1200				  to_addr_conv(sh, percpu));
1201		tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
1202	} else {
1203		/* Compute any data- or p-drive using XOR */
1204		count = 0;
1205		for (i = disks; i-- ; ) {
1206			if (i == target || i == qd_idx)
1207				continue;
1208			blocks[count++] = sh->dev[i].page;
1209		}
1210
1211		init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
1212				  NULL, ops_complete_compute, sh,
1213				  to_addr_conv(sh, percpu));
1214		tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
1215	}
1216
1217	return tx;
1218}
1219
1220static struct dma_async_tx_descriptor *
1221ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
1222{
1223	int i, count, disks = sh->disks;
1224	int syndrome_disks = sh->ddf_layout ? disks : disks-2;
1225	int d0_idx = raid6_d0(sh);
1226	int faila = -1, failb = -1;
1227	int target = sh->ops.target;
1228	int target2 = sh->ops.target2;
1229	struct r5dev *tgt = &sh->dev[target];
1230	struct r5dev *tgt2 = &sh->dev[target2];
1231	struct dma_async_tx_descriptor *tx;
1232	struct page **blocks = percpu->scribble;
1233	struct async_submit_ctl submit;
1234
 
1235	pr_debug("%s: stripe %llu block1: %d block2: %d\n",
1236		 __func__, (unsigned long long)sh->sector, target, target2);
1237	BUG_ON(target < 0 || target2 < 0);
1238	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1239	BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));
1240
1241	/* we need to open-code set_syndrome_sources to handle the
1242	 * slot number conversion for 'faila' and 'failb'
1243	 */
1244	for (i = 0; i < disks ; i++)
1245		blocks[i] = NULL;
1246	count = 0;
1247	i = d0_idx;
1248	do {
1249		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1250
1251		blocks[slot] = sh->dev[i].page;
1252
1253		if (i == target)
1254			faila = slot;
1255		if (i == target2)
1256			failb = slot;
1257		i = raid6_next_disk(i, disks);
1258	} while (i != d0_idx);
1259
1260	BUG_ON(faila == failb);
1261	if (failb < faila)
1262		swap(faila, failb);
1263	pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
1264		 __func__, (unsigned long long)sh->sector, faila, failb);
1265
1266	atomic_inc(&sh->count);
1267
1268	if (failb == syndrome_disks+1) {
1269		/* Q disk is one of the missing disks */
1270		if (faila == syndrome_disks) {
1271			/* Missing P+Q, just recompute */
1272			init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1273					  ops_complete_compute, sh,
1274					  to_addr_conv(sh, percpu));
1275			return async_gen_syndrome(blocks, 0, syndrome_disks+2,
1276						  STRIPE_SIZE, &submit);
1277		} else {
1278			struct page *dest;
1279			int data_target;
1280			int qd_idx = sh->qd_idx;
1281
1282			/* Missing D+Q: recompute D from P, then recompute Q */
1283			if (target == qd_idx)
1284				data_target = target2;
1285			else
1286				data_target = target;
1287
1288			count = 0;
1289			for (i = disks; i-- ; ) {
1290				if (i == data_target || i == qd_idx)
1291					continue;
1292				blocks[count++] = sh->dev[i].page;
1293			}
1294			dest = sh->dev[data_target].page;
1295			init_async_submit(&submit,
1296					  ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
1297					  NULL, NULL, NULL,
1298					  to_addr_conv(sh, percpu));
1299			tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
1300				       &submit);
1301
1302			count = set_syndrome_sources(blocks, sh);
1303			init_async_submit(&submit, ASYNC_TX_FENCE, tx,
1304					  ops_complete_compute, sh,
1305					  to_addr_conv(sh, percpu));
1306			return async_gen_syndrome(blocks, 0, count+2,
1307						  STRIPE_SIZE, &submit);
1308		}
1309	} else {
1310		init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1311				  ops_complete_compute, sh,
1312				  to_addr_conv(sh, percpu));
1313		if (failb == syndrome_disks) {
1314			/* We're missing D+P. */
1315			return async_raid6_datap_recov(syndrome_disks+2,
1316						       STRIPE_SIZE, faila,
1317						       blocks, &submit);
1318		} else {
1319			/* We're missing D+D. */
1320			return async_raid6_2data_recov(syndrome_disks+2,
1321						       STRIPE_SIZE, faila, failb,
1322						       blocks, &submit);
1323		}
1324	}
1325}
1326
1327
1328static void ops_complete_prexor(void *stripe_head_ref)
1329{
1330	struct stripe_head *sh = stripe_head_ref;
1331
1332	pr_debug("%s: stripe %llu\n", __func__,
1333		(unsigned long long)sh->sector);
 
 
 
 
 
 
 
1334}
1335
1336static struct dma_async_tx_descriptor *
1337ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu,
1338	       struct dma_async_tx_descriptor *tx)
1339{
1340	int disks = sh->disks;
1341	struct page **xor_srcs = percpu->scribble;
1342	int count = 0, pd_idx = sh->pd_idx, i;
1343	struct async_submit_ctl submit;
1344
1345	/* existing parity data subtracted */
1346	struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1347
 
1348	pr_debug("%s: stripe %llu\n", __func__,
1349		(unsigned long long)sh->sector);
1350
1351	for (i = disks; i--; ) {
1352		struct r5dev *dev = &sh->dev[i];
1353		/* Only process blocks that are known to be uptodate */
1354		if (test_bit(R5_Wantdrain, &dev->flags))
 
 
1355			xor_srcs[count++] = dev->page;
1356	}
1357
1358	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
1359			  ops_complete_prexor, sh, to_addr_conv(sh, percpu));
1360	tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1361
1362	return tx;
1363}
1364
1365static struct dma_async_tx_descriptor *
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1366ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
1367{
 
1368	int disks = sh->disks;
1369	int i;
 
1370
1371	pr_debug("%s: stripe %llu\n", __func__,
1372		(unsigned long long)sh->sector);
1373
1374	for (i = disks; i--; ) {
1375		struct r5dev *dev = &sh->dev[i];
1376		struct bio *chosen;
1377
1378		if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) {
 
1379			struct bio *wbi;
1380
 
 
 
 
 
 
 
1381			spin_lock_irq(&sh->stripe_lock);
1382			chosen = dev->towrite;
1383			dev->towrite = NULL;
 
1384			BUG_ON(dev->written);
1385			wbi = dev->written = chosen;
1386			spin_unlock_irq(&sh->stripe_lock);
 
1387
1388			while (wbi && wbi->bi_iter.bi_sector <
1389				dev->sector + STRIPE_SECTORS) {
1390				if (wbi->bi_rw & REQ_FUA)
1391					set_bit(R5_WantFUA, &dev->flags);
1392				if (wbi->bi_rw & REQ_SYNC)
1393					set_bit(R5_SyncIO, &dev->flags);
1394				if (wbi->bi_rw & REQ_DISCARD)
1395					set_bit(R5_Discard, &dev->flags);
1396				else
1397					tx = async_copy_data(1, wbi, dev->page,
1398						dev->sector, tx);
 
 
 
 
 
 
 
 
1399				wbi = r5_next_bio(wbi, dev->sector);
1400			}
 
 
 
 
 
 
 
 
 
1401		}
1402	}
1403
1404	return tx;
1405}
1406
1407static void ops_complete_reconstruct(void *stripe_head_ref)
1408{
1409	struct stripe_head *sh = stripe_head_ref;
1410	int disks = sh->disks;
1411	int pd_idx = sh->pd_idx;
1412	int qd_idx = sh->qd_idx;
1413	int i;
1414	bool fua = false, sync = false, discard = false;
1415
1416	pr_debug("%s: stripe %llu\n", __func__,
1417		(unsigned long long)sh->sector);
1418
1419	for (i = disks; i--; ) {
1420		fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
1421		sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
1422		discard |= test_bit(R5_Discard, &sh->dev[i].flags);
1423	}
1424
1425	for (i = disks; i--; ) {
1426		struct r5dev *dev = &sh->dev[i];
1427
1428		if (dev->written || i == pd_idx || i == qd_idx) {
1429			if (!discard)
1430				set_bit(R5_UPTODATE, &dev->flags);
 
 
 
1431			if (fua)
1432				set_bit(R5_WantFUA, &dev->flags);
1433			if (sync)
1434				set_bit(R5_SyncIO, &dev->flags);
1435		}
1436	}
1437
1438	if (sh->reconstruct_state == reconstruct_state_drain_run)
1439		sh->reconstruct_state = reconstruct_state_drain_result;
1440	else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
1441		sh->reconstruct_state = reconstruct_state_prexor_drain_result;
1442	else {
1443		BUG_ON(sh->reconstruct_state != reconstruct_state_run);
1444		sh->reconstruct_state = reconstruct_state_result;
1445	}
1446
1447	set_bit(STRIPE_HANDLE, &sh->state);
1448	release_stripe(sh);
1449}
1450
1451static void
1452ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
1453		     struct dma_async_tx_descriptor *tx)
1454{
1455	int disks = sh->disks;
1456	struct page **xor_srcs = percpu->scribble;
1457	struct async_submit_ctl submit;
1458	int count = 0, pd_idx = sh->pd_idx, i;
1459	struct page *xor_dest;
1460	int prexor = 0;
1461	unsigned long flags;
 
 
 
1462
1463	pr_debug("%s: stripe %llu\n", __func__,
1464		(unsigned long long)sh->sector);
1465
1466	for (i = 0; i < sh->disks; i++) {
1467		if (pd_idx == i)
1468			continue;
1469		if (!test_bit(R5_Discard, &sh->dev[i].flags))
1470			break;
1471	}
1472	if (i >= sh->disks) {
1473		atomic_inc(&sh->count);
1474		set_bit(R5_Discard, &sh->dev[pd_idx].flags);
1475		ops_complete_reconstruct(sh);
1476		return;
1477	}
 
 
 
1478	/* check if prexor is active which means only process blocks
1479	 * that are part of a read-modify-write (written)
1480	 */
1481	if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1482		prexor = 1;
1483		xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1484		for (i = disks; i--; ) {
1485			struct r5dev *dev = &sh->dev[i];
1486			if (dev->written)
 
1487				xor_srcs[count++] = dev->page;
1488		}
1489	} else {
1490		xor_dest = sh->dev[pd_idx].page;
1491		for (i = disks; i--; ) {
1492			struct r5dev *dev = &sh->dev[i];
1493			if (i != pd_idx)
1494				xor_srcs[count++] = dev->page;
1495		}
1496	}
1497
1498	/* 1/ if we prexor'd then the dest is reused as a source
1499	 * 2/ if we did not prexor then we are redoing the parity
1500	 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
1501	 * for the synchronous xor case
1502	 */
1503	flags = ASYNC_TX_ACK |
1504		(prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
1505
1506	atomic_inc(&sh->count);
 
 
 
 
 
 
 
 
 
 
 
1507
1508	init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh,
1509			  to_addr_conv(sh, percpu));
1510	if (unlikely(count == 1))
1511		tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
1512	else
1513		tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
 
 
 
 
 
 
1514}
1515
1516static void
1517ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
1518		     struct dma_async_tx_descriptor *tx)
1519{
1520	struct async_submit_ctl submit;
1521	struct page **blocks = percpu->scribble;
1522	int count, i;
 
 
 
 
1523
1524	pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
1525
1526	for (i = 0; i < sh->disks; i++) {
1527		if (sh->pd_idx == i || sh->qd_idx == i)
1528			continue;
1529		if (!test_bit(R5_Discard, &sh->dev[i].flags))
1530			break;
1531	}
1532	if (i >= sh->disks) {
1533		atomic_inc(&sh->count);
1534		set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
1535		set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
1536		ops_complete_reconstruct(sh);
1537		return;
1538	}
1539
1540	count = set_syndrome_sources(blocks, sh);
 
1541
1542	atomic_inc(&sh->count);
 
 
 
 
 
 
1543
1544	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct,
1545			  sh, to_addr_conv(sh, percpu));
1546	async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1547}
1548
1549static void ops_complete_check(void *stripe_head_ref)
1550{
1551	struct stripe_head *sh = stripe_head_ref;
1552
1553	pr_debug("%s: stripe %llu\n", __func__,
1554		(unsigned long long)sh->sector);
1555
1556	sh->check_state = check_state_check_result;
1557	set_bit(STRIPE_HANDLE, &sh->state);
1558	release_stripe(sh);
1559}
1560
1561static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
1562{
1563	int disks = sh->disks;
1564	int pd_idx = sh->pd_idx;
1565	int qd_idx = sh->qd_idx;
1566	struct page *xor_dest;
1567	struct page **xor_srcs = percpu->scribble;
1568	struct dma_async_tx_descriptor *tx;
1569	struct async_submit_ctl submit;
1570	int count;
1571	int i;
1572
1573	pr_debug("%s: stripe %llu\n", __func__,
1574		(unsigned long long)sh->sector);
1575
 
1576	count = 0;
1577	xor_dest = sh->dev[pd_idx].page;
1578	xor_srcs[count++] = xor_dest;
1579	for (i = disks; i--; ) {
1580		if (i == pd_idx || i == qd_idx)
1581			continue;
1582		xor_srcs[count++] = sh->dev[i].page;
1583	}
1584
1585	init_async_submit(&submit, 0, NULL, NULL, NULL,
1586			  to_addr_conv(sh, percpu));
1587	tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
1588			   &sh->ops.zero_sum_result, &submit);
1589
1590	atomic_inc(&sh->count);
1591	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
1592	tx = async_trigger_callback(&submit);
1593}
1594
1595static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
1596{
1597	struct page **srcs = percpu->scribble;
1598	struct async_submit_ctl submit;
1599	int count;
1600
1601	pr_debug("%s: stripe %llu checkp: %d\n", __func__,
1602		(unsigned long long)sh->sector, checkp);
1603
1604	count = set_syndrome_sources(srcs, sh);
 
1605	if (!checkp)
1606		srcs[count] = NULL;
1607
1608	atomic_inc(&sh->count);
1609	init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
1610			  sh, to_addr_conv(sh, percpu));
1611	async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
1612			   &sh->ops.zero_sum_result, percpu->spare_page, &submit);
1613}
1614
1615static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
1616{
1617	int overlap_clear = 0, i, disks = sh->disks;
1618	struct dma_async_tx_descriptor *tx = NULL;
1619	struct r5conf *conf = sh->raid_conf;
1620	int level = conf->level;
1621	struct raid5_percpu *percpu;
1622	unsigned long cpu;
1623
1624	cpu = get_cpu();
1625	percpu = per_cpu_ptr(conf->percpu, cpu);
1626	if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
1627		ops_run_biofill(sh);
1628		overlap_clear++;
1629	}
1630
1631	if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
1632		if (level < 6)
1633			tx = ops_run_compute5(sh, percpu);
1634		else {
1635			if (sh->ops.target2 < 0 || sh->ops.target < 0)
1636				tx = ops_run_compute6_1(sh, percpu);
1637			else
1638				tx = ops_run_compute6_2(sh, percpu);
1639		}
1640		/* terminate the chain if reconstruct is not set to be run */
1641		if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
1642			async_tx_ack(tx);
1643	}
1644
1645	if (test_bit(STRIPE_OP_PREXOR, &ops_request))
1646		tx = ops_run_prexor(sh, percpu, tx);
 
 
 
 
 
 
 
1647
1648	if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
1649		tx = ops_run_biodrain(sh, tx);
1650		overlap_clear++;
1651	}
1652
1653	if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
1654		if (level < 6)
1655			ops_run_reconstruct5(sh, percpu, tx);
1656		else
1657			ops_run_reconstruct6(sh, percpu, tx);
1658	}
1659
1660	if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
1661		if (sh->check_state == check_state_run)
1662			ops_run_check_p(sh, percpu);
1663		else if (sh->check_state == check_state_run_q)
1664			ops_run_check_pq(sh, percpu, 0);
1665		else if (sh->check_state == check_state_run_pq)
1666			ops_run_check_pq(sh, percpu, 1);
1667		else
1668			BUG();
1669	}
1670
1671	if (overlap_clear)
1672		for (i = disks; i--; ) {
1673			struct r5dev *dev = &sh->dev[i];
1674			if (test_and_clear_bit(R5_Overlap, &dev->flags))
1675				wake_up(&sh->raid_conf->wait_for_overlap);
1676		}
1677	put_cpu();
1678}
1679
1680static int grow_one_stripe(struct r5conf *conf, int hash)
 
 
 
 
 
 
 
 
1681{
1682	struct stripe_head *sh;
1683	sh = kmem_cache_zalloc(conf->slab_cache, GFP_KERNEL);
1684	if (!sh)
1685		return 0;
 
 
 
 
 
 
 
 
 
 
 
 
1686
1687	sh->raid_conf = conf;
 
 
1688
1689	spin_lock_init(&sh->stripe_lock);
 
 
 
 
 
 
 
 
 
 
 
 
1690
1691	if (grow_buffers(sh)) {
 
 
 
 
1692		shrink_buffers(sh);
1693		kmem_cache_free(conf->slab_cache, sh);
1694		return 0;
1695	}
1696	sh->hash_lock_index = hash;
 
1697	/* we just created an active stripe so... */
1698	atomic_set(&sh->count, 1);
1699	atomic_inc(&conf->active_stripes);
1700	INIT_LIST_HEAD(&sh->lru);
1701	release_stripe(sh);
 
1702	return 1;
1703}
1704
1705static int grow_stripes(struct r5conf *conf, int num)
1706{
1707	struct kmem_cache *sc;
 
1708	int devs = max(conf->raid_disks, conf->previous_raid_disks);
1709	int hash;
1710
1711	if (conf->mddev->gendisk)
1712		sprintf(conf->cache_name[0],
1713			"raid%d-%s", conf->level, mdname(conf->mddev));
1714	else
1715		sprintf(conf->cache_name[0],
1716			"raid%d-%p", conf->level, conf->mddev);
1717	sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]);
1718
1719	conf->active_name = 0;
1720	sc = kmem_cache_create(conf->cache_name[conf->active_name],
1721			       sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
1722			       0, 0, NULL);
1723	if (!sc)
1724		return 1;
1725	conf->slab_cache = sc;
1726	conf->pool_size = devs;
1727	hash = conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;
1728	while (num--) {
1729		if (!grow_one_stripe(conf, hash))
1730			return 1;
1731		conf->max_nr_stripes++;
1732		hash = (hash + 1) % NR_STRIPE_HASH_LOCKS;
1733	}
1734	return 0;
1735}
1736
1737/**
1738 * scribble_len - return the required size of the scribble region
1739 * @num - total number of disks in the array
1740 *
1741 * The size must be enough to contain:
1742 * 1/ a struct page pointer for each device in the array +2
1743 * 2/ room to convert each entry in (1) to its corresponding dma
1744 *    (dma_map_page()) or page (page_address()) address.
1745 *
1746 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
1747 * calculate over all devices (not just the data blocks), using zeros in place
1748 * of the P and Q blocks.
1749 */
1750static size_t scribble_len(int num)
 
1751{
1752	size_t len;
 
 
 
1753
1754	len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);
 
 
1755
1756	return len;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1757}
1758
1759static int resize_stripes(struct r5conf *conf, int newsize)
1760{
1761	/* Make all the stripes able to hold 'newsize' devices.
1762	 * New slots in each stripe get 'page' set to a new page.
1763	 *
1764	 * This happens in stages:
1765	 * 1/ create a new kmem_cache and allocate the required number of
1766	 *    stripe_heads.
1767	 * 2/ gather all the old stripe_heads and transfer the pages across
1768	 *    to the new stripe_heads.  This will have the side effect of
1769	 *    freezing the array as once all stripe_heads have been collected,
1770	 *    no IO will be possible.  Old stripe heads are freed once their
1771	 *    pages have been transferred over, and the old kmem_cache is
1772	 *    freed when all stripes are done.
1773	 * 3/ reallocate conf->disks to be suitable bigger.  If this fails,
1774	 *    we simple return a failre status - no need to clean anything up.
1775	 * 4/ allocate new pages for the new slots in the new stripe_heads.
1776	 *    If this fails, we don't bother trying the shrink the
1777	 *    stripe_heads down again, we just leave them as they are.
1778	 *    As each stripe_head is processed the new one is released into
1779	 *    active service.
1780	 *
1781	 * Once step2 is started, we cannot afford to wait for a write,
1782	 * so we use GFP_NOIO allocations.
1783	 */
1784	struct stripe_head *osh, *nsh;
1785	LIST_HEAD(newstripes);
1786	struct disk_info *ndisks;
1787	unsigned long cpu;
1788	int err;
1789	struct kmem_cache *sc;
1790	int i;
1791	int hash, cnt;
1792
1793	if (newsize <= conf->pool_size)
1794		return 0; /* never bother to shrink */
1795
1796	err = md_allow_write(conf->mddev);
1797	if (err)
1798		return err;
1799
1800	/* Step 1 */
1801	sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
1802			       sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
1803			       0, 0, NULL);
1804	if (!sc)
1805		return -ENOMEM;
1806
 
 
 
1807	for (i = conf->max_nr_stripes; i; i--) {
1808		nsh = kmem_cache_zalloc(sc, GFP_KERNEL);
1809		if (!nsh)
1810			break;
1811
1812		nsh->raid_conf = conf;
1813		spin_lock_init(&nsh->stripe_lock);
1814
1815		list_add(&nsh->lru, &newstripes);
1816	}
1817	if (i) {
1818		/* didn't get enough, give up */
1819		while (!list_empty(&newstripes)) {
1820			nsh = list_entry(newstripes.next, struct stripe_head, lru);
1821			list_del(&nsh->lru);
1822			kmem_cache_free(sc, nsh);
1823		}
1824		kmem_cache_destroy(sc);
 
1825		return -ENOMEM;
1826	}
1827	/* Step 2 - Must use GFP_NOIO now.
1828	 * OK, we have enough stripes, start collecting inactive
1829	 * stripes and copying them over
1830	 */
1831	hash = 0;
1832	cnt = 0;
1833	list_for_each_entry(nsh, &newstripes, lru) {
1834		lock_device_hash_lock(conf, hash);
1835		wait_event_cmd(conf->wait_for_stripe,
1836				    !list_empty(conf->inactive_list + hash),
1837				    unlock_device_hash_lock(conf, hash),
1838				    lock_device_hash_lock(conf, hash));
1839		osh = get_free_stripe(conf, hash);
1840		unlock_device_hash_lock(conf, hash);
1841		atomic_set(&nsh->count, 1);
1842		for(i=0; i<conf->pool_size; i++)
1843			nsh->dev[i].page = osh->dev[i].page;
1844		for( ; i<newsize; i++)
1845			nsh->dev[i].page = NULL;
1846		nsh->hash_lock_index = hash;
1847		kmem_cache_free(conf->slab_cache, osh);
1848		cnt++;
1849		if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS +
1850		    !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) {
1851			hash++;
1852			cnt = 0;
1853		}
1854	}
1855	kmem_cache_destroy(conf->slab_cache);
1856
1857	/* Step 3.
1858	 * At this point, we are holding all the stripes so the array
1859	 * is completely stalled, so now is a good time to resize
1860	 * conf->disks and the scribble region
1861	 */
1862	ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
1863	if (ndisks) {
1864		for (i=0; i<conf->raid_disks; i++)
1865			ndisks[i] = conf->disks[i];
1866		kfree(conf->disks);
1867		conf->disks = ndisks;
1868	} else
1869		err = -ENOMEM;
1870
1871	get_online_cpus();
1872	conf->scribble_len = scribble_len(newsize);
1873	for_each_present_cpu(cpu) {
1874		struct raid5_percpu *percpu;
1875		void *scribble;
1876
1877		percpu = per_cpu_ptr(conf->percpu, cpu);
1878		scribble = kmalloc(conf->scribble_len, GFP_NOIO);
1879
1880		if (scribble) {
1881			kfree(percpu->scribble);
1882			percpu->scribble = scribble;
 
 
1883		} else {
1884			err = -ENOMEM;
1885			break;
1886		}
1887	}
1888	put_online_cpus();
 
 
 
 
 
1889
1890	/* Step 4, return new stripes to service */
1891	while(!list_empty(&newstripes)) {
1892		nsh = list_entry(newstripes.next, struct stripe_head, lru);
1893		list_del_init(&nsh->lru);
1894
1895		for (i=conf->raid_disks; i < newsize; i++)
1896			if (nsh->dev[i].page == NULL) {
1897				struct page *p = alloc_page(GFP_NOIO);
1898				nsh->dev[i].page = p;
 
1899				if (!p)
1900					err = -ENOMEM;
1901			}
1902		release_stripe(nsh);
1903	}
1904	/* critical section pass, GFP_NOIO no longer needed */
1905
1906	conf->slab_cache = sc;
1907	conf->active_name = 1-conf->active_name;
1908	conf->pool_size = newsize;
1909	return err;
1910}
1911
1912static int drop_one_stripe(struct r5conf *conf, int hash)
1913{
1914	struct stripe_head *sh;
 
1915
1916	spin_lock_irq(conf->hash_locks + hash);
1917	sh = get_free_stripe(conf, hash);
1918	spin_unlock_irq(conf->hash_locks + hash);
1919	if (!sh)
1920		return 0;
1921	BUG_ON(atomic_read(&sh->count));
1922	shrink_buffers(sh);
1923	kmem_cache_free(conf->slab_cache, sh);
1924	atomic_dec(&conf->active_stripes);
 
1925	return 1;
1926}
1927
1928static void shrink_stripes(struct r5conf *conf)
1929{
1930	int hash;
1931	for (hash = 0; hash < NR_STRIPE_HASH_LOCKS; hash++)
1932		while (drop_one_stripe(conf, hash))
1933			;
1934
1935	if (conf->slab_cache)
1936		kmem_cache_destroy(conf->slab_cache);
1937	conf->slab_cache = NULL;
1938}
1939
1940static void raid5_end_read_request(struct bio * bi, int error)
1941{
1942	struct stripe_head *sh = bi->bi_private;
1943	struct r5conf *conf = sh->raid_conf;
1944	int disks = sh->disks, i;
1945	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
1946	char b[BDEVNAME_SIZE];
1947	struct md_rdev *rdev = NULL;
1948	sector_t s;
1949
1950	for (i=0 ; i<disks; i++)
1951		if (bi == &sh->dev[i].req)
1952			break;
1953
1954	pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n",
1955		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
1956		uptodate);
1957	if (i == disks) {
 
1958		BUG();
1959		return;
1960	}
1961	if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
1962		/* If replacement finished while this request was outstanding,
1963		 * 'replacement' might be NULL already.
1964		 * In that case it moved down to 'rdev'.
1965		 * rdev is not removed until all requests are finished.
1966		 */
1967		rdev = conf->disks[i].replacement;
1968	if (!rdev)
1969		rdev = conf->disks[i].rdev;
1970
1971	if (use_new_offset(conf, sh))
1972		s = sh->sector + rdev->new_data_offset;
1973	else
1974		s = sh->sector + rdev->data_offset;
1975	if (uptodate) {
1976		set_bit(R5_UPTODATE, &sh->dev[i].flags);
1977		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
1978			/* Note that this cannot happen on a
1979			 * replacement device.  We just fail those on
1980			 * any error
1981			 */
1982			printk_ratelimited(
1983				KERN_INFO
1984				"md/raid:%s: read error corrected"
1985				" (%lu sectors at %llu on %s)\n",
1986				mdname(conf->mddev), STRIPE_SECTORS,
1987				(unsigned long long)s,
1988				bdevname(rdev->bdev, b));
1989			atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
1990			clear_bit(R5_ReadError, &sh->dev[i].flags);
1991			clear_bit(R5_ReWrite, &sh->dev[i].flags);
1992		} else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
1993			clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
1994
 
 
 
 
 
 
 
1995		if (atomic_read(&rdev->read_errors))
1996			atomic_set(&rdev->read_errors, 0);
1997	} else {
1998		const char *bdn = bdevname(rdev->bdev, b);
1999		int retry = 0;
2000		int set_bad = 0;
2001
2002		clear_bit(R5_UPTODATE, &sh->dev[i].flags);
2003		atomic_inc(&rdev->read_errors);
 
2004		if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
2005			printk_ratelimited(
2006				KERN_WARNING
2007				"md/raid:%s: read error on replacement device "
2008				"(sector %llu on %s).\n",
2009				mdname(conf->mddev),
2010				(unsigned long long)s,
2011				bdn);
2012		else if (conf->mddev->degraded >= conf->max_degraded) {
2013			set_bad = 1;
2014			printk_ratelimited(
2015				KERN_WARNING
2016				"md/raid:%s: read error not correctable "
2017				"(sector %llu on %s).\n",
2018				mdname(conf->mddev),
2019				(unsigned long long)s,
2020				bdn);
2021		} else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
2022			/* Oh, no!!! */
2023			set_bad = 1;
2024			printk_ratelimited(
2025				KERN_WARNING
2026				"md/raid:%s: read error NOT corrected!! "
2027				"(sector %llu on %s).\n",
2028				mdname(conf->mddev),
2029				(unsigned long long)s,
2030				bdn);
2031		} else if (atomic_read(&rdev->read_errors)
2032			 > conf->max_nr_stripes)
2033			printk(KERN_WARNING
2034			       "md/raid:%s: Too many read errors, failing device %s.\n",
2035			       mdname(conf->mddev), bdn);
2036		else
 
 
 
 
 
2037			retry = 1;
2038		if (set_bad && test_bit(In_sync, &rdev->flags)
2039		    && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
2040			retry = 1;
2041		if (retry)
2042			if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {
 
 
2043				set_bit(R5_ReadError, &sh->dev[i].flags);
2044				clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2045			} else
2046				set_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2047		else {
2048			clear_bit(R5_ReadError, &sh->dev[i].flags);
2049			clear_bit(R5_ReWrite, &sh->dev[i].flags);
2050			if (!(set_bad
2051			      && test_bit(In_sync, &rdev->flags)
2052			      && rdev_set_badblocks(
2053				      rdev, sh->sector, STRIPE_SECTORS, 0)))
2054				md_error(conf->mddev, rdev);
2055		}
2056	}
2057	rdev_dec_pending(rdev, conf->mddev);
 
2058	clear_bit(R5_LOCKED, &sh->dev[i].flags);
2059	set_bit(STRIPE_HANDLE, &sh->state);
2060	release_stripe(sh);
2061}
2062
2063static void raid5_end_write_request(struct bio *bi, int error)
2064{
2065	struct stripe_head *sh = bi->bi_private;
2066	struct r5conf *conf = sh->raid_conf;
2067	int disks = sh->disks, i;
2068	struct md_rdev *uninitialized_var(rdev);
2069	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
2070	sector_t first_bad;
2071	int bad_sectors;
2072	int replacement = 0;
2073
2074	for (i = 0 ; i < disks; i++) {
2075		if (bi == &sh->dev[i].req) {
2076			rdev = conf->disks[i].rdev;
2077			break;
2078		}
2079		if (bi == &sh->dev[i].rreq) {
2080			rdev = conf->disks[i].replacement;
2081			if (rdev)
2082				replacement = 1;
2083			else
2084				/* rdev was removed and 'replacement'
2085				 * replaced it.  rdev is not removed
2086				 * until all requests are finished.
2087				 */
2088				rdev = conf->disks[i].rdev;
2089			break;
2090		}
2091	}
2092	pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n",
2093		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
2094		uptodate);
2095	if (i == disks) {
 
2096		BUG();
2097		return;
2098	}
2099
2100	if (replacement) {
2101		if (!uptodate)
2102			md_error(conf->mddev, rdev);
2103		else if (is_badblock(rdev, sh->sector,
2104				     STRIPE_SECTORS,
2105				     &first_bad, &bad_sectors))
2106			set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
2107	} else {
2108		if (!uptodate) {
2109			set_bit(STRIPE_DEGRADED, &sh->state);
2110			set_bit(WriteErrorSeen, &rdev->flags);
2111			set_bit(R5_WriteError, &sh->dev[i].flags);
2112			if (!test_and_set_bit(WantReplacement, &rdev->flags))
2113				set_bit(MD_RECOVERY_NEEDED,
2114					&rdev->mddev->recovery);
2115		} else if (is_badblock(rdev, sh->sector,
2116				       STRIPE_SECTORS,
2117				       &first_bad, &bad_sectors)) {
2118			set_bit(R5_MadeGood, &sh->dev[i].flags);
2119			if (test_bit(R5_ReadError, &sh->dev[i].flags))
2120				/* That was a successful write so make
2121				 * sure it looks like we already did
2122				 * a re-write.
2123				 */
2124				set_bit(R5_ReWrite, &sh->dev[i].flags);
2125		}
2126	}
2127	rdev_dec_pending(rdev, conf->mddev);
2128
 
 
 
 
2129	if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
2130		clear_bit(R5_LOCKED, &sh->dev[i].flags);
2131	set_bit(STRIPE_HANDLE, &sh->state);
2132	release_stripe(sh);
2133}
2134
2135static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous);
2136	
2137static void raid5_build_block(struct stripe_head *sh, int i, int previous)
2138{
2139	struct r5dev *dev = &sh->dev[i];
2140
2141	bio_init(&dev->req);
2142	dev->req.bi_io_vec = &dev->vec;
2143	dev->req.bi_vcnt++;
2144	dev->req.bi_max_vecs++;
2145	dev->req.bi_private = sh;
2146	dev->vec.bv_page = dev->page;
2147
2148	bio_init(&dev->rreq);
2149	dev->rreq.bi_io_vec = &dev->rvec;
2150	dev->rreq.bi_vcnt++;
2151	dev->rreq.bi_max_vecs++;
2152	dev->rreq.bi_private = sh;
2153	dev->rvec.bv_page = dev->page;
2154
2155	dev->flags = 0;
2156	dev->sector = compute_blocknr(sh, i, previous);
2157}
2158
2159static void error(struct mddev *mddev, struct md_rdev *rdev)
2160{
2161	char b[BDEVNAME_SIZE];
2162	struct r5conf *conf = mddev->private;
2163	unsigned long flags;
2164	pr_debug("raid456: error called\n");
2165
2166	spin_lock_irqsave(&conf->device_lock, flags);
 
 
 
 
 
 
 
 
 
 
 
 
 
2167	clear_bit(In_sync, &rdev->flags);
2168	mddev->degraded = calc_degraded(conf);
2169	spin_unlock_irqrestore(&conf->device_lock, flags);
2170	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
2171
2172	set_bit(Blocked, &rdev->flags);
2173	set_bit(Faulty, &rdev->flags);
2174	set_bit(MD_CHANGE_DEVS, &mddev->flags);
2175	printk(KERN_ALERT
2176	       "md/raid:%s: Disk failure on %s, disabling device.\n"
2177	       "md/raid:%s: Operation continuing on %d devices.\n",
2178	       mdname(mddev),
2179	       bdevname(rdev->bdev, b),
2180	       mdname(mddev),
2181	       conf->raid_disks - mddev->degraded);
2182}
2183
2184/*
2185 * Input: a 'big' sector number,
2186 * Output: index of the data and parity disk, and the sector # in them.
2187 */
2188static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
2189				     int previous, int *dd_idx,
2190				     struct stripe_head *sh)
2191{
2192	sector_t stripe, stripe2;
2193	sector_t chunk_number;
2194	unsigned int chunk_offset;
2195	int pd_idx, qd_idx;
2196	int ddf_layout = 0;
2197	sector_t new_sector;
2198	int algorithm = previous ? conf->prev_algo
2199				 : conf->algorithm;
2200	int sectors_per_chunk = previous ? conf->prev_chunk_sectors
2201					 : conf->chunk_sectors;
2202	int raid_disks = previous ? conf->previous_raid_disks
2203				  : conf->raid_disks;
2204	int data_disks = raid_disks - conf->max_degraded;
2205
2206	/* First compute the information on this sector */
2207
2208	/*
2209	 * Compute the chunk number and the sector offset inside the chunk
2210	 */
2211	chunk_offset = sector_div(r_sector, sectors_per_chunk);
2212	chunk_number = r_sector;
2213
2214	/*
2215	 * Compute the stripe number
2216	 */
2217	stripe = chunk_number;
2218	*dd_idx = sector_div(stripe, data_disks);
2219	stripe2 = stripe;
2220	/*
2221	 * Select the parity disk based on the user selected algorithm.
2222	 */
2223	pd_idx = qd_idx = -1;
2224	switch(conf->level) {
2225	case 4:
2226		pd_idx = data_disks;
2227		break;
2228	case 5:
2229		switch (algorithm) {
2230		case ALGORITHM_LEFT_ASYMMETRIC:
2231			pd_idx = data_disks - sector_div(stripe2, raid_disks);
2232			if (*dd_idx >= pd_idx)
2233				(*dd_idx)++;
2234			break;
2235		case ALGORITHM_RIGHT_ASYMMETRIC:
2236			pd_idx = sector_div(stripe2, raid_disks);
2237			if (*dd_idx >= pd_idx)
2238				(*dd_idx)++;
2239			break;
2240		case ALGORITHM_LEFT_SYMMETRIC:
2241			pd_idx = data_disks - sector_div(stripe2, raid_disks);
2242			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
2243			break;
2244		case ALGORITHM_RIGHT_SYMMETRIC:
2245			pd_idx = sector_div(stripe2, raid_disks);
2246			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
2247			break;
2248		case ALGORITHM_PARITY_0:
2249			pd_idx = 0;
2250			(*dd_idx)++;
2251			break;
2252		case ALGORITHM_PARITY_N:
2253			pd_idx = data_disks;
2254			break;
2255		default:
2256			BUG();
2257		}
2258		break;
2259	case 6:
2260
2261		switch (algorithm) {
2262		case ALGORITHM_LEFT_ASYMMETRIC:
2263			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2264			qd_idx = pd_idx + 1;
2265			if (pd_idx == raid_disks-1) {
2266				(*dd_idx)++;	/* Q D D D P */
2267				qd_idx = 0;
2268			} else if (*dd_idx >= pd_idx)
2269				(*dd_idx) += 2; /* D D P Q D */
2270			break;
2271		case ALGORITHM_RIGHT_ASYMMETRIC:
2272			pd_idx = sector_div(stripe2, raid_disks);
2273			qd_idx = pd_idx + 1;
2274			if (pd_idx == raid_disks-1) {
2275				(*dd_idx)++;	/* Q D D D P */
2276				qd_idx = 0;
2277			} else if (*dd_idx >= pd_idx)
2278				(*dd_idx) += 2; /* D D P Q D */
2279			break;
2280		case ALGORITHM_LEFT_SYMMETRIC:
2281			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2282			qd_idx = (pd_idx + 1) % raid_disks;
2283			*dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
2284			break;
2285		case ALGORITHM_RIGHT_SYMMETRIC:
2286			pd_idx = sector_div(stripe2, raid_disks);
2287			qd_idx = (pd_idx + 1) % raid_disks;
2288			*dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
2289			break;
2290
2291		case ALGORITHM_PARITY_0:
2292			pd_idx = 0;
2293			qd_idx = 1;
2294			(*dd_idx) += 2;
2295			break;
2296		case ALGORITHM_PARITY_N:
2297			pd_idx = data_disks;
2298			qd_idx = data_disks + 1;
2299			break;
2300
2301		case ALGORITHM_ROTATING_ZERO_RESTART:
2302			/* Exactly the same as RIGHT_ASYMMETRIC, but or
2303			 * of blocks for computing Q is different.
2304			 */
2305			pd_idx = sector_div(stripe2, raid_disks);
2306			qd_idx = pd_idx + 1;
2307			if (pd_idx == raid_disks-1) {
2308				(*dd_idx)++;	/* Q D D D P */
2309				qd_idx = 0;
2310			} else if (*dd_idx >= pd_idx)
2311				(*dd_idx) += 2; /* D D P Q D */
2312			ddf_layout = 1;
2313			break;
2314
2315		case ALGORITHM_ROTATING_N_RESTART:
2316			/* Same a left_asymmetric, by first stripe is
2317			 * D D D P Q  rather than
2318			 * Q D D D P
2319			 */
2320			stripe2 += 1;
2321			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2322			qd_idx = pd_idx + 1;
2323			if (pd_idx == raid_disks-1) {
2324				(*dd_idx)++;	/* Q D D D P */
2325				qd_idx = 0;
2326			} else if (*dd_idx >= pd_idx)
2327				(*dd_idx) += 2; /* D D P Q D */
2328			ddf_layout = 1;
2329			break;
2330
2331		case ALGORITHM_ROTATING_N_CONTINUE:
2332			/* Same as left_symmetric but Q is before P */
2333			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2334			qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
2335			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
2336			ddf_layout = 1;
2337			break;
2338
2339		case ALGORITHM_LEFT_ASYMMETRIC_6:
2340			/* RAID5 left_asymmetric, with Q on last device */
2341			pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
2342			if (*dd_idx >= pd_idx)
2343				(*dd_idx)++;
2344			qd_idx = raid_disks - 1;
2345			break;
2346
2347		case ALGORITHM_RIGHT_ASYMMETRIC_6:
2348			pd_idx = sector_div(stripe2, raid_disks-1);
2349			if (*dd_idx >= pd_idx)
2350				(*dd_idx)++;
2351			qd_idx = raid_disks - 1;
2352			break;
2353
2354		case ALGORITHM_LEFT_SYMMETRIC_6:
2355			pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
2356			*dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
2357			qd_idx = raid_disks - 1;
2358			break;
2359
2360		case ALGORITHM_RIGHT_SYMMETRIC_6:
2361			pd_idx = sector_div(stripe2, raid_disks-1);
2362			*dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
2363			qd_idx = raid_disks - 1;
2364			break;
2365
2366		case ALGORITHM_PARITY_0_6:
2367			pd_idx = 0;
2368			(*dd_idx)++;
2369			qd_idx = raid_disks - 1;
2370			break;
2371
2372		default:
2373			BUG();
2374		}
2375		break;
2376	}
2377
2378	if (sh) {
2379		sh->pd_idx = pd_idx;
2380		sh->qd_idx = qd_idx;
2381		sh->ddf_layout = ddf_layout;
2382	}
2383	/*
2384	 * Finally, compute the new sector number
2385	 */
2386	new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
2387	return new_sector;
2388}
2389
2390
2391static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous)
2392{
2393	struct r5conf *conf = sh->raid_conf;
2394	int raid_disks = sh->disks;
2395	int data_disks = raid_disks - conf->max_degraded;
2396	sector_t new_sector = sh->sector, check;
2397	int sectors_per_chunk = previous ? conf->prev_chunk_sectors
2398					 : conf->chunk_sectors;
2399	int algorithm = previous ? conf->prev_algo
2400				 : conf->algorithm;
2401	sector_t stripe;
2402	int chunk_offset;
2403	sector_t chunk_number;
2404	int dummy1, dd_idx = i;
2405	sector_t r_sector;
2406	struct stripe_head sh2;
2407
2408
2409	chunk_offset = sector_div(new_sector, sectors_per_chunk);
2410	stripe = new_sector;
2411
2412	if (i == sh->pd_idx)
2413		return 0;
2414	switch(conf->level) {
2415	case 4: break;
2416	case 5:
2417		switch (algorithm) {
2418		case ALGORITHM_LEFT_ASYMMETRIC:
2419		case ALGORITHM_RIGHT_ASYMMETRIC:
2420			if (i > sh->pd_idx)
2421				i--;
2422			break;
2423		case ALGORITHM_LEFT_SYMMETRIC:
2424		case ALGORITHM_RIGHT_SYMMETRIC:
2425			if (i < sh->pd_idx)
2426				i += raid_disks;
2427			i -= (sh->pd_idx + 1);
2428			break;
2429		case ALGORITHM_PARITY_0:
2430			i -= 1;
2431			break;
2432		case ALGORITHM_PARITY_N:
2433			break;
2434		default:
2435			BUG();
2436		}
2437		break;
2438	case 6:
2439		if (i == sh->qd_idx)
2440			return 0; /* It is the Q disk */
2441		switch (algorithm) {
2442		case ALGORITHM_LEFT_ASYMMETRIC:
2443		case ALGORITHM_RIGHT_ASYMMETRIC:
2444		case ALGORITHM_ROTATING_ZERO_RESTART:
2445		case ALGORITHM_ROTATING_N_RESTART:
2446			if (sh->pd_idx == raid_disks-1)
2447				i--;	/* Q D D D P */
2448			else if (i > sh->pd_idx)
2449				i -= 2; /* D D P Q D */
2450			break;
2451		case ALGORITHM_LEFT_SYMMETRIC:
2452		case ALGORITHM_RIGHT_SYMMETRIC:
2453			if (sh->pd_idx == raid_disks-1)
2454				i--; /* Q D D D P */
2455			else {
2456				/* D D P Q D */
2457				if (i < sh->pd_idx)
2458					i += raid_disks;
2459				i -= (sh->pd_idx + 2);
2460			}
2461			break;
2462		case ALGORITHM_PARITY_0:
2463			i -= 2;
2464			break;
2465		case ALGORITHM_PARITY_N:
2466			break;
2467		case ALGORITHM_ROTATING_N_CONTINUE:
2468			/* Like left_symmetric, but P is before Q */
2469			if (sh->pd_idx == 0)
2470				i--;	/* P D D D Q */
2471			else {
2472				/* D D Q P D */
2473				if (i < sh->pd_idx)
2474					i += raid_disks;
2475				i -= (sh->pd_idx + 1);
2476			}
2477			break;
2478		case ALGORITHM_LEFT_ASYMMETRIC_6:
2479		case ALGORITHM_RIGHT_ASYMMETRIC_6:
2480			if (i > sh->pd_idx)
2481				i--;
2482			break;
2483		case ALGORITHM_LEFT_SYMMETRIC_6:
2484		case ALGORITHM_RIGHT_SYMMETRIC_6:
2485			if (i < sh->pd_idx)
2486				i += data_disks + 1;
2487			i -= (sh->pd_idx + 1);
2488			break;
2489		case ALGORITHM_PARITY_0_6:
2490			i -= 1;
2491			break;
2492		default:
2493			BUG();
2494		}
2495		break;
2496	}
2497
2498	chunk_number = stripe * data_disks + i;
2499	r_sector = chunk_number * sectors_per_chunk + chunk_offset;
2500
2501	check = raid5_compute_sector(conf, r_sector,
2502				     previous, &dummy1, &sh2);
2503	if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
2504		|| sh2.qd_idx != sh->qd_idx) {
2505		printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n",
2506		       mdname(conf->mddev));
2507		return 0;
2508	}
2509	return r_sector;
2510}
2511
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2512
2513static void
2514schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
2515			 int rcw, int expand)
2516{
2517	int i, pd_idx = sh->pd_idx, disks = sh->disks;
2518	struct r5conf *conf = sh->raid_conf;
2519	int level = conf->level;
2520
2521	if (rcw) {
 
 
 
 
 
 
 
2522
2523		for (i = disks; i--; ) {
2524			struct r5dev *dev = &sh->dev[i];
2525
2526			if (dev->towrite) {
2527				set_bit(R5_LOCKED, &dev->flags);
2528				set_bit(R5_Wantdrain, &dev->flags);
2529				if (!expand)
2530					clear_bit(R5_UPTODATE, &dev->flags);
2531				s->locked++;
 
 
 
2532			}
2533		}
2534		/* if we are not expanding this is a proper write request, and
2535		 * there will be bios with new data to be drained into the
2536		 * stripe cache
2537		 */
2538		if (!expand) {
2539			if (!s->locked)
2540				/* False alarm, nothing to do */
2541				return;
2542			sh->reconstruct_state = reconstruct_state_drain_run;
2543			set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2544		} else
2545			sh->reconstruct_state = reconstruct_state_run;
2546
2547		set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2548
2549		if (s->locked + conf->max_degraded == disks)
2550			if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
2551				atomic_inc(&conf->pending_full_writes);
2552	} else {
2553		BUG_ON(level == 6);
2554		BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
2555			test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
 
 
 
2556
2557		for (i = disks; i--; ) {
2558			struct r5dev *dev = &sh->dev[i];
2559			if (i == pd_idx)
2560				continue;
2561
2562			if (dev->towrite &&
2563			    (test_bit(R5_UPTODATE, &dev->flags) ||
2564			     test_bit(R5_Wantcompute, &dev->flags))) {
2565				set_bit(R5_Wantdrain, &dev->flags);
2566				set_bit(R5_LOCKED, &dev->flags);
2567				clear_bit(R5_UPTODATE, &dev->flags);
2568				s->locked++;
 
 
 
2569			}
2570		}
2571		if (!s->locked)
2572			/* False alarm - nothing to do */
2573			return;
2574		sh->reconstruct_state = reconstruct_state_prexor_drain_run;
2575		set_bit(STRIPE_OP_PREXOR, &s->ops_request);
2576		set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2577		set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2578	}
2579
2580	/* keep the parity disk(s) locked while asynchronous operations
2581	 * are in flight
2582	 */
2583	set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
2584	clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
2585	s->locked++;
2586
2587	if (level == 6) {
2588		int qd_idx = sh->qd_idx;
2589		struct r5dev *dev = &sh->dev[qd_idx];
2590
2591		set_bit(R5_LOCKED, &dev->flags);
2592		clear_bit(R5_UPTODATE, &dev->flags);
2593		s->locked++;
2594	}
2595
 
 
 
 
 
 
2596	pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
2597		__func__, (unsigned long long)sh->sector,
2598		s->locked, s->ops_request);
2599}
2600
2601/*
2602 * Each stripe/dev can have one or more bion attached.
2603 * toread/towrite point to the first in a chain.
2604 * The bi_next chain must be in order.
2605 */
2606static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite)
 
2607{
2608	struct bio **bip;
2609	struct r5conf *conf = sh->raid_conf;
2610	int firstwrite=0;
2611
2612	pr_debug("adding bi b#%llu to stripe s#%llu\n",
2613		(unsigned long long)bi->bi_iter.bi_sector,
2614		(unsigned long long)sh->sector);
2615
2616	/*
2617	 * If several bio share a stripe. The bio bi_phys_segments acts as a
2618	 * reference count to avoid race. The reference count should already be
2619	 * increased before this function is called (for example, in
2620	 * make_request()), so other bio sharing this stripe will not free the
2621	 * stripe. If a stripe is owned by one stripe, the stripe lock will
2622	 * protect it.
2623	 */
2624	spin_lock_irq(&sh->stripe_lock);
 
 
 
 
2625	if (forwrite) {
2626		bip = &sh->dev[dd_idx].towrite;
2627		if (*bip == NULL)
2628			firstwrite = 1;
2629	} else
2630		bip = &sh->dev[dd_idx].toread;
2631	while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) {
2632		if (bio_end_sector(*bip) > bi->bi_iter.bi_sector)
2633			goto overlap;
2634		bip = & (*bip)->bi_next;
2635	}
2636	if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi))
2637		goto overlap;
2638
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2639	BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
2640	if (*bip)
2641		bi->bi_next = *bip;
2642	*bip = bi;
2643	raid5_inc_bi_active_stripes(bi);
 
2644
2645	if (forwrite) {
2646		/* check if page is covered */
2647		sector_t sector = sh->dev[dd_idx].sector;
2648		for (bi=sh->dev[dd_idx].towrite;
2649		     sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
2650			     bi && bi->bi_iter.bi_sector <= sector;
2651		     bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
2652			if (bio_end_sector(bi) >= sector)
2653				sector = bio_end_sector(bi);
2654		}
2655		if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
2656			set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags);
 
2657	}
2658
2659	pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
2660		(unsigned long long)(*bip)->bi_iter.bi_sector,
2661		(unsigned long long)sh->sector, dd_idx);
2662	spin_unlock_irq(&sh->stripe_lock);
2663
2664	if (conf->mddev->bitmap && firstwrite) {
2665		bitmap_startwrite(conf->mddev->bitmap, sh->sector,
2666				  STRIPE_SECTORS, 0);
2667		sh->bm_seq = conf->seq_flush+1;
2668		set_bit(STRIPE_BIT_DELAY, &sh->state);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2669	}
 
 
 
 
2670	return 1;
2671
2672 overlap:
2673	set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
2674	spin_unlock_irq(&sh->stripe_lock);
2675	return 0;
2676}
2677
2678static void end_reshape(struct r5conf *conf);
2679
2680static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
2681			    struct stripe_head *sh)
2682{
2683	int sectors_per_chunk =
2684		previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
2685	int dd_idx;
2686	int chunk_offset = sector_div(stripe, sectors_per_chunk);
2687	int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
2688
2689	raid5_compute_sector(conf,
2690			     stripe * (disks - conf->max_degraded)
2691			     *sectors_per_chunk + chunk_offset,
2692			     previous,
2693			     &dd_idx, sh);
2694}
2695
2696static void
2697handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
2698				struct stripe_head_state *s, int disks,
2699				struct bio **return_bi)
2700{
2701	int i;
 
2702	for (i = disks; i--; ) {
2703		struct bio *bi;
2704		int bitmap_end = 0;
2705
2706		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
2707			struct md_rdev *rdev;
2708			rcu_read_lock();
2709			rdev = rcu_dereference(conf->disks[i].rdev);
2710			if (rdev && test_bit(In_sync, &rdev->flags))
 
2711				atomic_inc(&rdev->nr_pending);
2712			else
2713				rdev = NULL;
2714			rcu_read_unlock();
2715			if (rdev) {
2716				if (!rdev_set_badblocks(
2717					    rdev,
2718					    sh->sector,
2719					    STRIPE_SECTORS, 0))
2720					md_error(conf->mddev, rdev);
2721				rdev_dec_pending(rdev, conf->mddev);
2722			}
2723		}
2724		spin_lock_irq(&sh->stripe_lock);
2725		/* fail all writes first */
2726		bi = sh->dev[i].towrite;
2727		sh->dev[i].towrite = NULL;
 
2728		spin_unlock_irq(&sh->stripe_lock);
2729		if (bi)
2730			bitmap_end = 1;
2731
 
 
2732		if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
2733			wake_up(&conf->wait_for_overlap);
2734
2735		while (bi && bi->bi_iter.bi_sector <
2736			sh->dev[i].sector + STRIPE_SECTORS) {
2737			struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
2738			clear_bit(BIO_UPTODATE, &bi->bi_flags);
2739			if (!raid5_dec_bi_active_stripes(bi)) {
2740				md_write_end(conf->mddev);
2741				bi->bi_next = *return_bi;
2742				*return_bi = bi;
2743			}
2744			bi = nextbi;
2745		}
2746		if (bitmap_end)
2747			bitmap_endwrite(conf->mddev->bitmap, sh->sector,
2748				STRIPE_SECTORS, 0, 0);
2749		bitmap_end = 0;
2750		/* and fail all 'written' */
2751		bi = sh->dev[i].written;
2752		sh->dev[i].written = NULL;
 
 
 
 
 
2753		if (bi) bitmap_end = 1;
2754		while (bi && bi->bi_iter.bi_sector <
2755		       sh->dev[i].sector + STRIPE_SECTORS) {
2756			struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
2757			clear_bit(BIO_UPTODATE, &bi->bi_flags);
2758			if (!raid5_dec_bi_active_stripes(bi)) {
2759				md_write_end(conf->mddev);
2760				bi->bi_next = *return_bi;
2761				*return_bi = bi;
2762			}
2763			bi = bi2;
2764		}
2765
2766		/* fail any reads if this device is non-operational and
2767		 * the data has not reached the cache yet.
2768		 */
2769		if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
 
2770		    (!test_bit(R5_Insync, &sh->dev[i].flags) ||
2771		      test_bit(R5_ReadError, &sh->dev[i].flags))) {
2772			spin_lock_irq(&sh->stripe_lock);
2773			bi = sh->dev[i].toread;
2774			sh->dev[i].toread = NULL;
2775			spin_unlock_irq(&sh->stripe_lock);
2776			if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
2777				wake_up(&conf->wait_for_overlap);
 
 
2778			while (bi && bi->bi_iter.bi_sector <
2779			       sh->dev[i].sector + STRIPE_SECTORS) {
2780				struct bio *nextbi =
2781					r5_next_bio(bi, sh->dev[i].sector);
2782				clear_bit(BIO_UPTODATE, &bi->bi_flags);
2783				if (!raid5_dec_bi_active_stripes(bi)) {
2784					bi->bi_next = *return_bi;
2785					*return_bi = bi;
2786				}
2787				bi = nextbi;
2788			}
2789		}
2790		if (bitmap_end)
2791			bitmap_endwrite(conf->mddev->bitmap, sh->sector,
2792					STRIPE_SECTORS, 0, 0);
2793		/* If we were in the middle of a write the parity block might
2794		 * still be locked - so just clear all R5_LOCKED flags
2795		 */
2796		clear_bit(R5_LOCKED, &sh->dev[i].flags);
2797	}
 
 
2798
2799	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
2800		if (atomic_dec_and_test(&conf->pending_full_writes))
2801			md_wakeup_thread(conf->mddev->thread);
2802}
2803
2804static void
2805handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
2806		   struct stripe_head_state *s)
2807{
2808	int abort = 0;
2809	int i;
2810
 
2811	clear_bit(STRIPE_SYNCING, &sh->state);
2812	if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
2813		wake_up(&conf->wait_for_overlap);
2814	s->syncing = 0;
2815	s->replacing = 0;
2816	/* There is nothing more to do for sync/check/repair.
2817	 * Don't even need to abort as that is handled elsewhere
2818	 * if needed, and not always wanted e.g. if there is a known
2819	 * bad block here.
2820	 * For recover/replace we need to record a bad block on all
2821	 * non-sync devices, or abort the recovery
2822	 */
2823	if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) {
2824		/* During recovery devices cannot be removed, so
2825		 * locking and refcounting of rdevs is not needed
2826		 */
 
2827		for (i = 0; i < conf->raid_disks; i++) {
2828			struct md_rdev *rdev = conf->disks[i].rdev;
2829			if (rdev
2830			    && !test_bit(Faulty, &rdev->flags)
2831			    && !test_bit(In_sync, &rdev->flags)
2832			    && !rdev_set_badblocks(rdev, sh->sector,
2833						   STRIPE_SECTORS, 0))
2834				abort = 1;
2835			rdev = conf->disks[i].replacement;
2836			if (rdev
2837			    && !test_bit(Faulty, &rdev->flags)
2838			    && !test_bit(In_sync, &rdev->flags)
2839			    && !rdev_set_badblocks(rdev, sh->sector,
2840						   STRIPE_SECTORS, 0))
2841				abort = 1;
2842		}
 
2843		if (abort)
2844			conf->recovery_disabled =
2845				conf->mddev->recovery_disabled;
2846	}
2847	md_done_sync(conf->mddev, STRIPE_SECTORS, !abort);
2848}
2849
2850static int want_replace(struct stripe_head *sh, int disk_idx)
2851{
2852	struct md_rdev *rdev;
2853	int rv = 0;
2854	/* Doing recovery so rcu locking not required */
2855	rdev = sh->raid_conf->disks[disk_idx].replacement;
 
2856	if (rdev
2857	    && !test_bit(Faulty, &rdev->flags)
2858	    && !test_bit(In_sync, &rdev->flags)
2859	    && (rdev->recovery_offset <= sh->sector
2860		|| rdev->mddev->recovery_cp <= sh->sector))
2861		rv = 1;
2862
2863	return rv;
2864}
2865
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2866/* fetch_block - checks the given member device to see if its data needs
2867 * to be read or computed to satisfy a request.
2868 *
2869 * Returns 1 when no more member devices need to be checked, otherwise returns
2870 * 0 to tell the loop in handle_stripe_fill to continue
2871 */
2872static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
2873		       int disk_idx, int disks)
2874{
2875	struct r5dev *dev = &sh->dev[disk_idx];
2876	struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
2877				  &sh->dev[s->failed_num[1]] };
2878
2879	/* is the data in this block needed, and can we get it? */
2880	if (!test_bit(R5_LOCKED, &dev->flags) &&
2881	    !test_bit(R5_UPTODATE, &dev->flags) &&
2882	    (dev->toread ||
2883	     (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) ||
2884	     s->syncing || s->expanding ||
2885	     (s->replacing && want_replace(sh, disk_idx)) ||
2886	     (s->failed >= 1 && fdev[0]->toread) ||
2887	     (s->failed >= 2 && fdev[1]->toread) ||
2888	     (sh->raid_conf->level <= 5 && s->failed && fdev[0]->towrite &&
2889	      !test_bit(R5_OVERWRITE, &fdev[0]->flags)) ||
2890	     (sh->raid_conf->level == 6 && s->failed && s->to_write))) {
2891		/* we would like to get this block, possibly by computing it,
2892		 * otherwise read it if the backing disk is insync
2893		 */
2894		BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
2895		BUG_ON(test_bit(R5_Wantread, &dev->flags));
 
 
 
 
 
 
 
 
 
 
 
2896		if ((s->uptodate == disks - 1) &&
 
2897		    (s->failed && (disk_idx == s->failed_num[0] ||
2898				   disk_idx == s->failed_num[1]))) {
2899			/* have disk failed, and we're requested to fetch it;
2900			 * do compute it
2901			 */
2902			pr_debug("Computing stripe %llu block %d\n",
2903			       (unsigned long long)sh->sector, disk_idx);
2904			set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2905			set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2906			set_bit(R5_Wantcompute, &dev->flags);
2907			sh->ops.target = disk_idx;
2908			sh->ops.target2 = -1; /* no 2nd target */
2909			s->req_compute = 1;
2910			/* Careful: from this point on 'uptodate' is in the eye
2911			 * of raid_run_ops which services 'compute' operations
2912			 * before writes. R5_Wantcompute flags a block that will
2913			 * be R5_UPTODATE by the time it is needed for a
2914			 * subsequent operation.
2915			 */
2916			s->uptodate++;
2917			return 1;
2918		} else if (s->uptodate == disks-2 && s->failed >= 2) {
2919			/* Computing 2-failure is *very* expensive; only
2920			 * do it if failed >= 2
2921			 */
2922			int other;
2923			for (other = disks; other--; ) {
2924				if (other == disk_idx)
2925					continue;
2926				if (!test_bit(R5_UPTODATE,
2927				      &sh->dev[other].flags))
2928					break;
2929			}
2930			BUG_ON(other < 0);
2931			pr_debug("Computing stripe %llu blocks %d,%d\n",
2932			       (unsigned long long)sh->sector,
2933			       disk_idx, other);
2934			set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2935			set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2936			set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
2937			set_bit(R5_Wantcompute, &sh->dev[other].flags);
2938			sh->ops.target = disk_idx;
2939			sh->ops.target2 = other;
2940			s->uptodate += 2;
2941			s->req_compute = 1;
2942			return 1;
2943		} else if (test_bit(R5_Insync, &dev->flags)) {
2944			set_bit(R5_LOCKED, &dev->flags);
2945			set_bit(R5_Wantread, &dev->flags);
2946			s->locked++;
2947			pr_debug("Reading block %d (sync=%d)\n",
2948				disk_idx, s->syncing);
2949		}
2950	}
2951
2952	return 0;
2953}
2954
2955/**
2956 * handle_stripe_fill - read or compute data to satisfy pending requests.
2957 */
2958static void handle_stripe_fill(struct stripe_head *sh,
2959			       struct stripe_head_state *s,
2960			       int disks)
2961{
2962	int i;
2963
2964	/* look for blocks to read/compute, skip this if a compute
2965	 * is already in flight, or if the stripe contents are in the
2966	 * midst of changing due to a write
2967	 */
2968	if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
2969	    !sh->reconstruct_state)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2970		for (i = disks; i--; )
2971			if (fetch_block(sh, s, i, disks))
2972				break;
 
 
2973	set_bit(STRIPE_HANDLE, &sh->state);
2974}
2975
2976
 
2977/* handle_stripe_clean_event
2978 * any written block on an uptodate or failed drive can be returned.
2979 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
2980 * never LOCKED, so we don't need to test 'failed' directly.
2981 */
2982static void handle_stripe_clean_event(struct r5conf *conf,
2983	struct stripe_head *sh, int disks, struct bio **return_bi)
2984{
2985	int i;
2986	struct r5dev *dev;
2987	int discard_pending = 0;
 
 
2988
2989	for (i = disks; i--; )
2990		if (sh->dev[i].written) {
2991			dev = &sh->dev[i];
2992			if (!test_bit(R5_LOCKED, &dev->flags) &&
2993			    (test_bit(R5_UPTODATE, &dev->flags) ||
2994			     test_bit(R5_Discard, &dev->flags))) {
 
2995				/* We can return any write requests */
2996				struct bio *wbi, *wbi2;
2997				pr_debug("Return write for disc %d\n", i);
2998				if (test_and_clear_bit(R5_Discard, &dev->flags))
2999					clear_bit(R5_UPTODATE, &dev->flags);
 
 
 
 
 
 
 
3000				wbi = dev->written;
3001				dev->written = NULL;
3002				while (wbi && wbi->bi_iter.bi_sector <
3003					dev->sector + STRIPE_SECTORS) {
3004					wbi2 = r5_next_bio(wbi, dev->sector);
3005					if (!raid5_dec_bi_active_stripes(wbi)) {
3006						md_write_end(conf->mddev);
3007						wbi->bi_next = *return_bi;
3008						*return_bi = wbi;
3009					}
3010					wbi = wbi2;
3011				}
3012				bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3013						STRIPE_SECTORS,
3014					 !test_bit(STRIPE_DEGRADED, &sh->state),
3015						0);
 
 
 
 
 
 
 
 
 
 
 
3016			} else if (test_bit(R5_Discard, &dev->flags))
3017				discard_pending = 1;
3018		}
 
 
 
3019	if (!discard_pending &&
3020	    test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {
 
3021		clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
3022		clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
3023		if (sh->qd_idx >= 0) {
3024			clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
3025			clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);
3026		}
3027		/* now that discard is done we can proceed with any sync */
3028		clear_bit(STRIPE_DISCARD, &sh->state);
3029		/*
3030		 * SCSI discard will change some bio fields and the stripe has
3031		 * no updated data, so remove it from hash list and the stripe
3032		 * will be reinitialized
3033		 */
3034		spin_lock_irq(&conf->device_lock);
 
 
3035		remove_hash(sh);
3036		spin_unlock_irq(&conf->device_lock);
 
 
 
 
 
 
 
 
3037		if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
3038			set_bit(STRIPE_HANDLE, &sh->state);
3039
3040	}
3041
3042	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
3043		if (atomic_dec_and_test(&conf->pending_full_writes))
3044			md_wakeup_thread(conf->mddev->thread);
 
 
 
3045}
3046
3047static void handle_stripe_dirtying(struct r5conf *conf,
3048				   struct stripe_head *sh,
3049				   struct stripe_head_state *s,
3050				   int disks)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3051{
3052	int rmw = 0, rcw = 0, i;
3053	sector_t recovery_cp = conf->mddev->recovery_cp;
3054
3055	/* RAID6 requires 'rcw' in current implementation.
3056	 * Otherwise, check whether resync is now happening or should start.
3057	 * If yes, then the array is dirty (after unclean shutdown or
3058	 * initial creation), so parity in some stripes might be inconsistent.
3059	 * In this case, we need to always do reconstruct-write, to ensure
3060	 * that in case of drive failure or read-error correction, we
3061	 * generate correct data from the parity.
3062	 */
3063	if (conf->max_degraded == 2 ||
3064	    (recovery_cp < MaxSector && sh->sector >= recovery_cp)) {
 
3065		/* Calculate the real rcw later - for now make it
3066		 * look like rcw is cheaper
3067		 */
3068		rcw = 1; rmw = 2;
3069		pr_debug("force RCW max_degraded=%u, recovery_cp=%llu sh->sector=%llu\n",
3070			 conf->max_degraded, (unsigned long long)recovery_cp,
3071			 (unsigned long long)sh->sector);
3072	} else for (i = disks; i--; ) {
3073		/* would I have to read this buffer for read_modify_write */
3074		struct r5dev *dev = &sh->dev[i];
3075		if ((dev->towrite || i == sh->pd_idx) &&
 
 
3076		    !test_bit(R5_LOCKED, &dev->flags) &&
3077		    !(test_bit(R5_UPTODATE, &dev->flags) ||
3078		      test_bit(R5_Wantcompute, &dev->flags))) {
3079			if (test_bit(R5_Insync, &dev->flags))
3080				rmw++;
3081			else
3082				rmw += 2*disks;  /* cannot read it */
3083		}
3084		/* Would I have to read this buffer for reconstruct_write */
3085		if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx &&
 
3086		    !test_bit(R5_LOCKED, &dev->flags) &&
3087		    !(test_bit(R5_UPTODATE, &dev->flags) ||
3088		    test_bit(R5_Wantcompute, &dev->flags))) {
3089			if (test_bit(R5_Insync, &dev->flags)) rcw++;
 
3090			else
3091				rcw += 2*disks;
3092		}
3093	}
3094	pr_debug("for sector %llu, rmw=%d rcw=%d\n",
3095		(unsigned long long)sh->sector, rmw, rcw);
 
3096	set_bit(STRIPE_HANDLE, &sh->state);
3097	if (rmw < rcw && rmw > 0) {
3098		/* prefer read-modify-write, but need to get some data */
3099		if (conf->mddev->queue)
3100			blk_add_trace_msg(conf->mddev->queue,
3101					  "raid5 rmw %llu %d",
3102					  (unsigned long long)sh->sector, rmw);
3103		for (i = disks; i--; ) {
3104			struct r5dev *dev = &sh->dev[i];
3105			if ((dev->towrite || i == sh->pd_idx) &&
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3106			    !test_bit(R5_LOCKED, &dev->flags) &&
3107			    !(test_bit(R5_UPTODATE, &dev->flags) ||
3108			    test_bit(R5_Wantcompute, &dev->flags)) &&
3109			    test_bit(R5_Insync, &dev->flags)) {
3110				if (
3111				  test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
3112					pr_debug("Read_old block "
3113						 "%d for r-m-w\n", i);
3114					set_bit(R5_LOCKED, &dev->flags);
3115					set_bit(R5_Wantread, &dev->flags);
3116					s->locked++;
3117				} else {
3118					set_bit(STRIPE_DELAYED, &sh->state);
3119					set_bit(STRIPE_HANDLE, &sh->state);
3120				}
3121			}
3122		}
3123	}
3124	if (rcw <= rmw && rcw > 0) {
3125		/* want reconstruct write, but need to get some data */
3126		int qread =0;
3127		rcw = 0;
3128		for (i = disks; i--; ) {
3129			struct r5dev *dev = &sh->dev[i];
3130			if (!test_bit(R5_OVERWRITE, &dev->flags) &&
3131			    i != sh->pd_idx && i != sh->qd_idx &&
3132			    !test_bit(R5_LOCKED, &dev->flags) &&
3133			    !(test_bit(R5_UPTODATE, &dev->flags) ||
3134			      test_bit(R5_Wantcompute, &dev->flags))) {
3135				rcw++;
3136				if (!test_bit(R5_Insync, &dev->flags))
3137					continue; /* it's a failed drive */
3138				if (
3139				  test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
3140					pr_debug("Read_old block "
3141						"%d for Reconstruct\n", i);
3142					set_bit(R5_LOCKED, &dev->flags);
3143					set_bit(R5_Wantread, &dev->flags);
3144					s->locked++;
3145					qread++;
3146				} else {
3147					set_bit(STRIPE_DELAYED, &sh->state);
3148					set_bit(STRIPE_HANDLE, &sh->state);
3149				}
3150			}
3151		}
3152		if (rcw && conf->mddev->queue)
3153			blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d",
3154					  (unsigned long long)sh->sector,
3155					  rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));
3156	}
 
 
 
 
 
3157	/* now if nothing is locked, and if we have enough data,
3158	 * we can start a write request
3159	 */
3160	/* since handle_stripe can be called at any time we need to handle the
3161	 * case where a compute block operation has been submitted and then a
3162	 * subsequent call wants to start a write request.  raid_run_ops only
3163	 * handles the case where compute block and reconstruct are requested
3164	 * simultaneously.  If this is not the case then new writes need to be
3165	 * held off until the compute completes.
3166	 */
3167	if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
3168	    (s->locked == 0 && (rcw == 0 || rmw == 0) &&
3169	    !test_bit(STRIPE_BIT_DELAY, &sh->state)))
3170		schedule_reconstruction(sh, s, rcw == 0, 0);
 
3171}
3172
3173static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
3174				struct stripe_head_state *s, int disks)
3175{
3176	struct r5dev *dev = NULL;
3177
 
3178	set_bit(STRIPE_HANDLE, &sh->state);
3179
3180	switch (sh->check_state) {
3181	case check_state_idle:
3182		/* start a new check operation if there are no failures */
3183		if (s->failed == 0) {
3184			BUG_ON(s->uptodate != disks);
3185			sh->check_state = check_state_run;
3186			set_bit(STRIPE_OP_CHECK, &s->ops_request);
3187			clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
3188			s->uptodate--;
3189			break;
3190		}
3191		dev = &sh->dev[s->failed_num[0]];
3192		/* fall through */
3193	case check_state_compute_result:
3194		sh->check_state = check_state_idle;
3195		if (!dev)
3196			dev = &sh->dev[sh->pd_idx];
3197
3198		/* check that a write has not made the stripe insync */
3199		if (test_bit(STRIPE_INSYNC, &sh->state))
3200			break;
3201
3202		/* either failed parity check, or recovery is happening */
3203		BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
3204		BUG_ON(s->uptodate != disks);
3205
3206		set_bit(R5_LOCKED, &dev->flags);
3207		s->locked++;
3208		set_bit(R5_Wantwrite, &dev->flags);
3209
3210		clear_bit(STRIPE_DEGRADED, &sh->state);
3211		set_bit(STRIPE_INSYNC, &sh->state);
3212		break;
3213	case check_state_run:
3214		break; /* we will be called again upon completion */
3215	case check_state_check_result:
3216		sh->check_state = check_state_idle;
3217
3218		/* if a failure occurred during the check operation, leave
3219		 * STRIPE_INSYNC not set and let the stripe be handled again
3220		 */
3221		if (s->failed)
3222			break;
3223
3224		/* handle a successful check operation, if parity is correct
3225		 * we are done.  Otherwise update the mismatch count and repair
3226		 * parity if !MD_RECOVERY_CHECK
3227		 */
3228		if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
3229			/* parity is correct (on disc,
3230			 * not in buffer any more)
3231			 */
3232			set_bit(STRIPE_INSYNC, &sh->state);
3233		else {
3234			atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
3235			if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
3236				/* don't try to repair!! */
3237				set_bit(STRIPE_INSYNC, &sh->state);
3238			else {
 
 
 
 
 
3239				sh->check_state = check_state_compute_run;
3240				set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3241				set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
3242				set_bit(R5_Wantcompute,
3243					&sh->dev[sh->pd_idx].flags);
3244				sh->ops.target = sh->pd_idx;
3245				sh->ops.target2 = -1;
3246				s->uptodate++;
3247			}
3248		}
3249		break;
3250	case check_state_compute_run:
3251		break;
3252	default:
3253		printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
3254		       __func__, sh->check_state,
3255		       (unsigned long long) sh->sector);
3256		BUG();
3257	}
3258}
3259
3260
3261static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
3262				  struct stripe_head_state *s,
3263				  int disks)
3264{
3265	int pd_idx = sh->pd_idx;
3266	int qd_idx = sh->qd_idx;
3267	struct r5dev *dev;
3268
 
3269	set_bit(STRIPE_HANDLE, &sh->state);
3270
3271	BUG_ON(s->failed > 2);
3272
3273	/* Want to check and possibly repair P and Q.
3274	 * However there could be one 'failed' device, in which
3275	 * case we can only check one of them, possibly using the
3276	 * other to generate missing data
3277	 */
3278
3279	switch (sh->check_state) {
3280	case check_state_idle:
3281		/* start a new check operation if there are < 2 failures */
3282		if (s->failed == s->q_failed) {
3283			/* The only possible failed device holds Q, so it
3284			 * makes sense to check P (If anything else were failed,
3285			 * we would have used P to recreate it).
3286			 */
3287			sh->check_state = check_state_run;
3288		}
3289		if (!s->q_failed && s->failed < 2) {
3290			/* Q is not failed, and we didn't use it to generate
3291			 * anything, so it makes sense to check it
3292			 */
3293			if (sh->check_state == check_state_run)
3294				sh->check_state = check_state_run_pq;
3295			else
3296				sh->check_state = check_state_run_q;
3297		}
3298
3299		/* discard potentially stale zero_sum_result */
3300		sh->ops.zero_sum_result = 0;
3301
3302		if (sh->check_state == check_state_run) {
3303			/* async_xor_zero_sum destroys the contents of P */
3304			clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
3305			s->uptodate--;
3306		}
3307		if (sh->check_state >= check_state_run &&
3308		    sh->check_state <= check_state_run_pq) {
3309			/* async_syndrome_zero_sum preserves P and Q, so
3310			 * no need to mark them !uptodate here
3311			 */
3312			set_bit(STRIPE_OP_CHECK, &s->ops_request);
3313			break;
3314		}
3315
3316		/* we have 2-disk failure */
3317		BUG_ON(s->failed != 2);
3318		/* fall through */
3319	case check_state_compute_result:
3320		sh->check_state = check_state_idle;
3321
3322		/* check that a write has not made the stripe insync */
3323		if (test_bit(STRIPE_INSYNC, &sh->state))
3324			break;
3325
3326		/* now write out any block on a failed drive,
3327		 * or P or Q if they were recomputed
3328		 */
3329		BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */
3330		if (s->failed == 2) {
3331			dev = &sh->dev[s->failed_num[1]];
3332			s->locked++;
3333			set_bit(R5_LOCKED, &dev->flags);
3334			set_bit(R5_Wantwrite, &dev->flags);
3335		}
3336		if (s->failed >= 1) {
3337			dev = &sh->dev[s->failed_num[0]];
3338			s->locked++;
3339			set_bit(R5_LOCKED, &dev->flags);
3340			set_bit(R5_Wantwrite, &dev->flags);
3341		}
3342		if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
3343			dev = &sh->dev[pd_idx];
3344			s->locked++;
3345			set_bit(R5_LOCKED, &dev->flags);
3346			set_bit(R5_Wantwrite, &dev->flags);
3347		}
3348		if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
3349			dev = &sh->dev[qd_idx];
3350			s->locked++;
3351			set_bit(R5_LOCKED, &dev->flags);
3352			set_bit(R5_Wantwrite, &dev->flags);
3353		}
 
 
 
 
 
 
 
 
3354		clear_bit(STRIPE_DEGRADED, &sh->state);
3355
3356		set_bit(STRIPE_INSYNC, &sh->state);
3357		break;
3358	case check_state_run:
3359	case check_state_run_q:
3360	case check_state_run_pq:
3361		break; /* we will be called again upon completion */
3362	case check_state_check_result:
3363		sh->check_state = check_state_idle;
3364
3365		/* handle a successful check operation, if parity is correct
3366		 * we are done.  Otherwise update the mismatch count and repair
3367		 * parity if !MD_RECOVERY_CHECK
3368		 */
3369		if (sh->ops.zero_sum_result == 0) {
3370			/* both parities are correct */
3371			if (!s->failed)
3372				set_bit(STRIPE_INSYNC, &sh->state);
3373			else {
3374				/* in contrast to the raid5 case we can validate
3375				 * parity, but still have a failure to write
3376				 * back
3377				 */
3378				sh->check_state = check_state_compute_result;
3379				/* Returning at this point means that we may go
3380				 * off and bring p and/or q uptodate again so
3381				 * we make sure to check zero_sum_result again
3382				 * to verify if p or q need writeback
3383				 */
3384			}
3385		} else {
3386			atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
3387			if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
3388				/* don't try to repair!! */
3389				set_bit(STRIPE_INSYNC, &sh->state);
3390			else {
 
 
 
 
 
3391				int *target = &sh->ops.target;
3392
3393				sh->ops.target = -1;
3394				sh->ops.target2 = -1;
3395				sh->check_state = check_state_compute_run;
3396				set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3397				set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
3398				if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
3399					set_bit(R5_Wantcompute,
3400						&sh->dev[pd_idx].flags);
3401					*target = pd_idx;
3402					target = &sh->ops.target2;
3403					s->uptodate++;
3404				}
3405				if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
3406					set_bit(R5_Wantcompute,
3407						&sh->dev[qd_idx].flags);
3408					*target = qd_idx;
3409					s->uptodate++;
3410				}
3411			}
3412		}
3413		break;
3414	case check_state_compute_run:
3415		break;
3416	default:
3417		printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
3418		       __func__, sh->check_state,
3419		       (unsigned long long) sh->sector);
3420		BUG();
3421	}
3422}
3423
3424static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
3425{
3426	int i;
3427
3428	/* We have read all the blocks in this stripe and now we need to
3429	 * copy some of them into a target stripe for expand.
3430	 */
3431	struct dma_async_tx_descriptor *tx = NULL;
 
3432	clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
3433	for (i = 0; i < sh->disks; i++)
3434		if (i != sh->pd_idx && i != sh->qd_idx) {
3435			int dd_idx, j;
3436			struct stripe_head *sh2;
3437			struct async_submit_ctl submit;
3438
3439			sector_t bn = compute_blocknr(sh, i, 1);
3440			sector_t s = raid5_compute_sector(conf, bn, 0,
3441							  &dd_idx, NULL);
3442			sh2 = get_active_stripe(conf, s, 0, 1, 1);
3443			if (sh2 == NULL)
3444				/* so far only the early blocks of this stripe
3445				 * have been requested.  When later blocks
3446				 * get requested, we will try again
3447				 */
3448				continue;
3449			if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
3450			   test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
3451				/* must have already done this block */
3452				release_stripe(sh2);
3453				continue;
3454			}
3455
3456			/* place all the copies on one channel */
3457			init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
3458			tx = async_memcpy(sh2->dev[dd_idx].page,
3459					  sh->dev[i].page, 0, 0, STRIPE_SIZE,
3460					  &submit);
3461
3462			set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
3463			set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
3464			for (j = 0; j < conf->raid_disks; j++)
3465				if (j != sh2->pd_idx &&
3466				    j != sh2->qd_idx &&
3467				    !test_bit(R5_Expanded, &sh2->dev[j].flags))
3468					break;
3469			if (j == conf->raid_disks) {
3470				set_bit(STRIPE_EXPAND_READY, &sh2->state);
3471				set_bit(STRIPE_HANDLE, &sh2->state);
3472			}
3473			release_stripe(sh2);
3474
3475		}
3476	/* done submitting copies, wait for them to complete */
3477	async_tx_quiesce(&tx);
3478}
3479
3480/*
3481 * handle_stripe - do things to a stripe.
3482 *
3483 * We lock the stripe by setting STRIPE_ACTIVE and then examine the
3484 * state of various bits to see what needs to be done.
3485 * Possible results:
3486 *    return some read requests which now have data
3487 *    return some write requests which are safely on storage
3488 *    schedule a read on some buffers
3489 *    schedule a write of some buffers
3490 *    return confirmation of parity correctness
3491 *
3492 */
3493
3494static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
3495{
3496	struct r5conf *conf = sh->raid_conf;
3497	int disks = sh->disks;
3498	struct r5dev *dev;
3499	int i;
3500	int do_recovery = 0;
3501
3502	memset(s, 0, sizeof(*s));
3503
3504	s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state);
3505	s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state);
3506	s->failed_num[0] = -1;
3507	s->failed_num[1] = -1;
 
3508
3509	/* Now to look around and see what can be done */
3510	rcu_read_lock();
3511	for (i=disks; i--; ) {
3512		struct md_rdev *rdev;
3513		sector_t first_bad;
3514		int bad_sectors;
3515		int is_bad = 0;
3516
3517		dev = &sh->dev[i];
3518
3519		pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
3520			 i, dev->flags,
3521			 dev->toread, dev->towrite, dev->written);
3522		/* maybe we can reply to a read
3523		 *
3524		 * new wantfill requests are only permitted while
3525		 * ops_complete_biofill is guaranteed to be inactive
3526		 */
3527		if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
3528		    !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
3529			set_bit(R5_Wantfill, &dev->flags);
3530
3531		/* now count some things */
3532		if (test_bit(R5_LOCKED, &dev->flags))
3533			s->locked++;
3534		if (test_bit(R5_UPTODATE, &dev->flags))
3535			s->uptodate++;
3536		if (test_bit(R5_Wantcompute, &dev->flags)) {
3537			s->compute++;
3538			BUG_ON(s->compute > 2);
3539		}
3540
3541		if (test_bit(R5_Wantfill, &dev->flags))
3542			s->to_fill++;
3543		else if (dev->toread)
3544			s->to_read++;
3545		if (dev->towrite) {
3546			s->to_write++;
3547			if (!test_bit(R5_OVERWRITE, &dev->flags))
3548				s->non_overwrite++;
3549		}
3550		if (dev->written)
3551			s->written++;
3552		/* Prefer to use the replacement for reads, but only
3553		 * if it is recovered enough and has no bad blocks.
3554		 */
3555		rdev = rcu_dereference(conf->disks[i].replacement);
3556		if (rdev && !test_bit(Faulty, &rdev->flags) &&
3557		    rdev->recovery_offset >= sh->sector + STRIPE_SECTORS &&
3558		    !is_badblock(rdev, sh->sector, STRIPE_SECTORS,
3559				 &first_bad, &bad_sectors))
3560			set_bit(R5_ReadRepl, &dev->flags);
3561		else {
3562			if (rdev)
3563				set_bit(R5_NeedReplace, &dev->flags);
 
 
3564			rdev = rcu_dereference(conf->disks[i].rdev);
3565			clear_bit(R5_ReadRepl, &dev->flags);
3566		}
3567		if (rdev && test_bit(Faulty, &rdev->flags))
3568			rdev = NULL;
3569		if (rdev) {
3570			is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
3571					     &first_bad, &bad_sectors);
3572			if (s->blocked_rdev == NULL
3573			    && (test_bit(Blocked, &rdev->flags)
3574				|| is_bad < 0)) {
3575				if (is_bad < 0)
3576					set_bit(BlockedBadBlocks,
3577						&rdev->flags);
3578				s->blocked_rdev = rdev;
3579				atomic_inc(&rdev->nr_pending);
3580			}
3581		}
3582		clear_bit(R5_Insync, &dev->flags);
3583		if (!rdev)
3584			/* Not in-sync */;
3585		else if (is_bad) {
3586			/* also not in-sync */
3587			if (!test_bit(WriteErrorSeen, &rdev->flags) &&
3588			    test_bit(R5_UPTODATE, &dev->flags)) {
3589				/* treat as in-sync, but with a read error
3590				 * which we can now try to correct
3591				 */
3592				set_bit(R5_Insync, &dev->flags);
3593				set_bit(R5_ReadError, &dev->flags);
3594			}
3595		} else if (test_bit(In_sync, &rdev->flags))
3596			set_bit(R5_Insync, &dev->flags);
3597		else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
3598			/* in sync if before recovery_offset */
3599			set_bit(R5_Insync, &dev->flags);
3600		else if (test_bit(R5_UPTODATE, &dev->flags) &&
3601			 test_bit(R5_Expanded, &dev->flags))
3602			/* If we've reshaped into here, we assume it is Insync.
3603			 * We will shortly update recovery_offset to make
3604			 * it official.
3605			 */
3606			set_bit(R5_Insync, &dev->flags);
3607
3608		if (test_bit(R5_WriteError, &dev->flags)) {
3609			/* This flag does not apply to '.replacement'
3610			 * only to .rdev, so make sure to check that*/
3611			struct md_rdev *rdev2 = rcu_dereference(
3612				conf->disks[i].rdev);
3613			if (rdev2 == rdev)
3614				clear_bit(R5_Insync, &dev->flags);
3615			if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
3616				s->handle_bad_blocks = 1;
3617				atomic_inc(&rdev2->nr_pending);
3618			} else
3619				clear_bit(R5_WriteError, &dev->flags);
3620		}
3621		if (test_bit(R5_MadeGood, &dev->flags)) {
3622			/* This flag does not apply to '.replacement'
3623			 * only to .rdev, so make sure to check that*/
3624			struct md_rdev *rdev2 = rcu_dereference(
3625				conf->disks[i].rdev);
3626			if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
3627				s->handle_bad_blocks = 1;
3628				atomic_inc(&rdev2->nr_pending);
3629			} else
3630				clear_bit(R5_MadeGood, &dev->flags);
3631		}
3632		if (test_bit(R5_MadeGoodRepl, &dev->flags)) {
3633			struct md_rdev *rdev2 = rcu_dereference(
3634				conf->disks[i].replacement);
3635			if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
3636				s->handle_bad_blocks = 1;
3637				atomic_inc(&rdev2->nr_pending);
3638			} else
3639				clear_bit(R5_MadeGoodRepl, &dev->flags);
3640		}
3641		if (!test_bit(R5_Insync, &dev->flags)) {
3642			/* The ReadError flag will just be confusing now */
3643			clear_bit(R5_ReadError, &dev->flags);
3644			clear_bit(R5_ReWrite, &dev->flags);
3645		}
3646		if (test_bit(R5_ReadError, &dev->flags))
3647			clear_bit(R5_Insync, &dev->flags);
3648		if (!test_bit(R5_Insync, &dev->flags)) {
3649			if (s->failed < 2)
3650				s->failed_num[s->failed] = i;
3651			s->failed++;
3652			if (rdev && !test_bit(Faulty, &rdev->flags))
3653				do_recovery = 1;
 
 
 
 
 
 
3654		}
 
 
 
 
 
3655	}
3656	if (test_bit(STRIPE_SYNCING, &sh->state)) {
3657		/* If there is a failed device being replaced,
3658		 *     we must be recovering.
3659		 * else if we are after recovery_cp, we must be syncing
3660		 * else if MD_RECOVERY_REQUESTED is set, we also are syncing.
3661		 * else we can only be replacing
3662		 * sync and recovery both need to read all devices, and so
3663		 * use the same flag.
3664		 */
3665		if (do_recovery ||
3666		    sh->sector >= conf->mddev->recovery_cp ||
3667		    test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery)))
3668			s->syncing = 1;
3669		else
3670			s->replacing = 1;
3671	}
3672	rcu_read_unlock();
3673}
3674
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3675static void handle_stripe(struct stripe_head *sh)
3676{
3677	struct stripe_head_state s;
3678	struct r5conf *conf = sh->raid_conf;
3679	int i;
3680	int prexor;
3681	int disks = sh->disks;
3682	struct r5dev *pdev, *qdev;
3683
3684	clear_bit(STRIPE_HANDLE, &sh->state);
3685	if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
3686		/* already being handled, ensure it gets handled
3687		 * again when current action finishes */
3688		set_bit(STRIPE_HANDLE, &sh->state);
3689		return;
3690	}
3691
3692	if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
 
 
 
 
 
 
 
 
3693		spin_lock(&sh->stripe_lock);
3694		/* Cannot process 'sync' concurrently with 'discard' */
3695		if (!test_bit(STRIPE_DISCARD, &sh->state) &&
 
 
 
 
 
3696		    test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
3697			set_bit(STRIPE_SYNCING, &sh->state);
3698			clear_bit(STRIPE_INSYNC, &sh->state);
3699			clear_bit(STRIPE_REPLACED, &sh->state);
3700		}
3701		spin_unlock(&sh->stripe_lock);
3702	}
3703	clear_bit(STRIPE_DELAYED, &sh->state);
3704
3705	pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
3706		"pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
3707	       (unsigned long long)sh->sector, sh->state,
3708	       atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
3709	       sh->check_state, sh->reconstruct_state);
3710
3711	analyse_stripe(sh, &s);
3712
3713	if (s.handle_bad_blocks) {
 
 
 
 
3714		set_bit(STRIPE_HANDLE, &sh->state);
3715		goto finish;
3716	}
3717
3718	if (unlikely(s.blocked_rdev)) {
3719		if (s.syncing || s.expanding || s.expanded ||
3720		    s.replacing || s.to_write || s.written) {
3721			set_bit(STRIPE_HANDLE, &sh->state);
3722			goto finish;
3723		}
3724		/* There is nothing for the blocked_rdev to block */
3725		rdev_dec_pending(s.blocked_rdev, conf->mddev);
3726		s.blocked_rdev = NULL;
3727	}
3728
3729	if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
3730		set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
3731		set_bit(STRIPE_BIOFILL_RUN, &sh->state);
3732	}
3733
3734	pr_debug("locked=%d uptodate=%d to_read=%d"
3735	       " to_write=%d failed=%d failed_num=%d,%d\n",
3736	       s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
3737	       s.failed_num[0], s.failed_num[1]);
3738	/* check if the array has lost more than max_degraded devices and,
 
3739	 * if so, some requests might need to be failed.
 
 
 
3740	 */
3741	if (s.failed > conf->max_degraded) {
 
3742		sh->check_state = 0;
3743		sh->reconstruct_state = 0;
 
3744		if (s.to_read+s.to_write+s.written)
3745			handle_failed_stripe(conf, sh, &s, disks, &s.return_bi);
3746		if (s.syncing + s.replacing)
3747			handle_failed_sync(conf, sh, &s);
3748	}
3749
3750	/* Now we check to see if any write operations have recently
3751	 * completed
3752	 */
3753	prexor = 0;
3754	if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
3755		prexor = 1;
3756	if (sh->reconstruct_state == reconstruct_state_drain_result ||
3757	    sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
3758		sh->reconstruct_state = reconstruct_state_idle;
3759
3760		/* All the 'written' buffers and the parity block are ready to
3761		 * be written back to disk
3762		 */
3763		BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&
3764		       !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));
3765		BUG_ON(sh->qd_idx >= 0 &&
3766		       !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&
3767		       !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));
3768		for (i = disks; i--; ) {
3769			struct r5dev *dev = &sh->dev[i];
3770			if (test_bit(R5_LOCKED, &dev->flags) &&
3771				(i == sh->pd_idx || i == sh->qd_idx ||
3772				 dev->written)) {
 
3773				pr_debug("Writing block %d\n", i);
3774				set_bit(R5_Wantwrite, &dev->flags);
3775				if (prexor)
3776					continue;
 
 
3777				if (!test_bit(R5_Insync, &dev->flags) ||
3778				    ((i == sh->pd_idx || i == sh->qd_idx)  &&
3779				     s.failed == 0))
3780					set_bit(STRIPE_INSYNC, &sh->state);
3781			}
3782		}
3783		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
3784			s.dec_preread_active = 1;
3785	}
3786
3787	/*
3788	 * might be able to return some write requests if the parity blocks
3789	 * are safe, or on a failed drive
3790	 */
3791	pdev = &sh->dev[sh->pd_idx];
3792	s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
3793		|| (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
3794	qdev = &sh->dev[sh->qd_idx];
3795	s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
3796		|| (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
3797		|| conf->level < 6;
3798
3799	if (s.written &&
3800	    (s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
3801			     && !test_bit(R5_LOCKED, &pdev->flags)
3802			     && (test_bit(R5_UPTODATE, &pdev->flags) ||
3803				 test_bit(R5_Discard, &pdev->flags))))) &&
3804	    (s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
3805			     && !test_bit(R5_LOCKED, &qdev->flags)
3806			     && (test_bit(R5_UPTODATE, &qdev->flags) ||
3807				 test_bit(R5_Discard, &qdev->flags))))))
3808		handle_stripe_clean_event(conf, sh, disks, &s.return_bi);
 
 
 
 
3809
3810	/* Now we might consider reading some blocks, either to check/generate
3811	 * parity, or to satisfy requests
3812	 * or to load a block that is being partially written.
3813	 */
3814	if (s.to_read || s.non_overwrite
3815	    || (conf->level == 6 && s.to_write && s.failed)
3816	    || (s.syncing && (s.uptodate + s.compute < disks))
3817	    || s.replacing
3818	    || s.expanding)
3819		handle_stripe_fill(sh, &s, disks);
3820
3821	/* Now to consider new write requests and what else, if anything
3822	 * should be read.  We do not handle new writes when:
 
 
 
 
 
 
 
 
3823	 * 1/ A 'write' operation (copy+xor) is already in flight.
3824	 * 2/ A 'check' operation is in flight, as it may clobber the parity
3825	 *    block.
 
3826	 */
3827	if (s.to_write && !sh->reconstruct_state && !sh->check_state)
3828		handle_stripe_dirtying(conf, sh, &s, disks);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3829
3830	/* maybe we need to check and possibly fix the parity for this stripe
3831	 * Any reads will already have been scheduled, so we just see if enough
3832	 * data is available.  The parity check is held off while parity
3833	 * dependent operations are in flight.
3834	 */
3835	if (sh->check_state ||
3836	    (s.syncing && s.locked == 0 &&
3837	     !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
3838	     !test_bit(STRIPE_INSYNC, &sh->state))) {
3839		if (conf->level == 6)
3840			handle_parity_checks6(conf, sh, &s, disks);
3841		else
3842			handle_parity_checks5(conf, sh, &s, disks);
3843	}
3844
3845	if ((s.replacing || s.syncing) && s.locked == 0
3846	    && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)
3847	    && !test_bit(STRIPE_REPLACED, &sh->state)) {
3848		/* Write out to replacement devices where possible */
3849		for (i = 0; i < conf->raid_disks; i++)
3850			if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {
3851				WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));
3852				set_bit(R5_WantReplace, &sh->dev[i].flags);
3853				set_bit(R5_LOCKED, &sh->dev[i].flags);
3854				s.locked++;
3855			}
3856		if (s.replacing)
3857			set_bit(STRIPE_INSYNC, &sh->state);
3858		set_bit(STRIPE_REPLACED, &sh->state);
3859	}
3860	if ((s.syncing || s.replacing) && s.locked == 0 &&
3861	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
3862	    test_bit(STRIPE_INSYNC, &sh->state)) {
3863		md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
3864		clear_bit(STRIPE_SYNCING, &sh->state);
3865		if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
3866			wake_up(&conf->wait_for_overlap);
3867	}
3868
3869	/* If the failed drives are just a ReadError, then we might need
3870	 * to progress the repair/check process
3871	 */
3872	if (s.failed <= conf->max_degraded && !conf->mddev->ro)
3873		for (i = 0; i < s.failed; i++) {
3874			struct r5dev *dev = &sh->dev[s.failed_num[i]];
3875			if (test_bit(R5_ReadError, &dev->flags)
3876			    && !test_bit(R5_LOCKED, &dev->flags)
3877			    && test_bit(R5_UPTODATE, &dev->flags)
3878				) {
3879				if (!test_bit(R5_ReWrite, &dev->flags)) {
3880					set_bit(R5_Wantwrite, &dev->flags);
3881					set_bit(R5_ReWrite, &dev->flags);
3882					set_bit(R5_LOCKED, &dev->flags);
3883					s.locked++;
3884				} else {
3885					/* let's read it back */
3886					set_bit(R5_Wantread, &dev->flags);
3887					set_bit(R5_LOCKED, &dev->flags);
3888					s.locked++;
3889				}
3890			}
3891		}
3892
3893
3894	/* Finish reconstruct operations initiated by the expansion process */
3895	if (sh->reconstruct_state == reconstruct_state_result) {
3896		struct stripe_head *sh_src
3897			= get_active_stripe(conf, sh->sector, 1, 1, 1);
3898		if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) {
3899			/* sh cannot be written until sh_src has been read.
3900			 * so arrange for sh to be delayed a little
3901			 */
3902			set_bit(STRIPE_DELAYED, &sh->state);
3903			set_bit(STRIPE_HANDLE, &sh->state);
3904			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
3905					      &sh_src->state))
3906				atomic_inc(&conf->preread_active_stripes);
3907			release_stripe(sh_src);
3908			goto finish;
3909		}
3910		if (sh_src)
3911			release_stripe(sh_src);
3912
3913		sh->reconstruct_state = reconstruct_state_idle;
3914		clear_bit(STRIPE_EXPANDING, &sh->state);
3915		for (i = conf->raid_disks; i--; ) {
3916			set_bit(R5_Wantwrite, &sh->dev[i].flags);
3917			set_bit(R5_LOCKED, &sh->dev[i].flags);
3918			s.locked++;
3919		}
3920	}
3921
3922	if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
3923	    !sh->reconstruct_state) {
3924		/* Need to write out all blocks after computing parity */
3925		sh->disks = conf->raid_disks;
3926		stripe_set_idx(sh->sector, conf, 0, sh);
3927		schedule_reconstruction(sh, &s, 1, 1);
3928	} else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
3929		clear_bit(STRIPE_EXPAND_READY, &sh->state);
3930		atomic_dec(&conf->reshape_stripes);
3931		wake_up(&conf->wait_for_overlap);
3932		md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
3933	}
3934
3935	if (s.expanding && s.locked == 0 &&
3936	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
3937		handle_stripe_expansion(conf, sh);
3938
3939finish:
3940	/* wait for this device to become unblocked */
3941	if (unlikely(s.blocked_rdev)) {
3942		if (conf->mddev->external)
3943			md_wait_for_blocked_rdev(s.blocked_rdev,
3944						 conf->mddev);
3945		else
3946			/* Internal metadata will immediately
3947			 * be written by raid5d, so we don't
3948			 * need to wait here.
3949			 */
3950			rdev_dec_pending(s.blocked_rdev,
3951					 conf->mddev);
3952	}
3953
3954	if (s.handle_bad_blocks)
3955		for (i = disks; i--; ) {
3956			struct md_rdev *rdev;
3957			struct r5dev *dev = &sh->dev[i];
3958			if (test_and_clear_bit(R5_WriteError, &dev->flags)) {
3959				/* We own a safe reference to the rdev */
3960				rdev = conf->disks[i].rdev;
3961				if (!rdev_set_badblocks(rdev, sh->sector,
3962							STRIPE_SECTORS, 0))
3963					md_error(conf->mddev, rdev);
3964				rdev_dec_pending(rdev, conf->mddev);
3965			}
3966			if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
3967				rdev = conf->disks[i].rdev;
3968				rdev_clear_badblocks(rdev, sh->sector,
3969						     STRIPE_SECTORS, 0);
3970				rdev_dec_pending(rdev, conf->mddev);
3971			}
3972			if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
3973				rdev = conf->disks[i].replacement;
3974				if (!rdev)
3975					/* rdev have been moved down */
3976					rdev = conf->disks[i].rdev;
3977				rdev_clear_badblocks(rdev, sh->sector,
3978						     STRIPE_SECTORS, 0);
3979				rdev_dec_pending(rdev, conf->mddev);
3980			}
3981		}
3982
3983	if (s.ops_request)
3984		raid_run_ops(sh, s.ops_request);
3985
3986	ops_run_io(sh, &s);
3987
3988	if (s.dec_preread_active) {
3989		/* We delay this until after ops_run_io so that if make_request
3990		 * is waiting on a flush, it won't continue until the writes
3991		 * have actually been submitted.
3992		 */
3993		atomic_dec(&conf->preread_active_stripes);
3994		if (atomic_read(&conf->preread_active_stripes) <
3995		    IO_THRESHOLD)
3996			md_wakeup_thread(conf->mddev->thread);
3997	}
3998
3999	return_io(s.return_bi);
4000
4001	clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
4002}
4003
4004static void raid5_activate_delayed(struct r5conf *conf)
4005{
4006	if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
4007		while (!list_empty(&conf->delayed_list)) {
4008			struct list_head *l = conf->delayed_list.next;
4009			struct stripe_head *sh;
4010			sh = list_entry(l, struct stripe_head, lru);
4011			list_del_init(l);
4012			clear_bit(STRIPE_DELAYED, &sh->state);
4013			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
4014				atomic_inc(&conf->preread_active_stripes);
4015			list_add_tail(&sh->lru, &conf->hold_list);
4016			raid5_wakeup_stripe_thread(sh);
4017		}
4018	}
4019}
4020
4021static void activate_bit_delay(struct r5conf *conf,
4022	struct list_head *temp_inactive_list)
4023{
4024	/* device_lock is held */
4025	struct list_head head;
4026	list_add(&head, &conf->bitmap_list);
4027	list_del_init(&conf->bitmap_list);
4028	while (!list_empty(&head)) {
4029		struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
4030		int hash;
4031		list_del_init(&sh->lru);
4032		atomic_inc(&sh->count);
4033		hash = sh->hash_lock_index;
4034		__release_stripe(conf, sh, &temp_inactive_list[hash]);
4035	}
4036}
4037
4038int md_raid5_congested(struct mddev *mddev, int bits)
4039{
4040	struct r5conf *conf = mddev->private;
4041
4042	/* No difference between reads and writes.  Just check
4043	 * how busy the stripe_cache is
4044	 */
4045
4046	if (conf->inactive_blocked)
 
 
 
 
4047		return 1;
4048	if (conf->quiesce)
4049		return 1;
4050	if (atomic_read(&conf->empty_inactive_list_nr))
4051		return 1;
4052
4053	return 0;
4054}
4055EXPORT_SYMBOL_GPL(md_raid5_congested);
4056
4057static int raid5_congested(void *data, int bits)
4058{
4059	struct mddev *mddev = data;
4060
4061	return mddev_congested(mddev, bits) ||
4062		md_raid5_congested(mddev, bits);
4063}
4064
4065/* We want read requests to align with chunks where possible,
4066 * but write requests don't need to.
4067 */
4068static int raid5_mergeable_bvec(struct request_queue *q,
4069				struct bvec_merge_data *bvm,
4070				struct bio_vec *biovec)
4071{
4072	struct mddev *mddev = q->queuedata;
4073	sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
4074	int max;
4075	unsigned int chunk_sectors = mddev->chunk_sectors;
4076	unsigned int bio_sectors = bvm->bi_size >> 9;
4077
4078	if ((bvm->bi_rw & 1) == WRITE)
4079		return biovec->bv_len; /* always allow writes to be mergeable */
4080
4081	if (mddev->new_chunk_sectors < mddev->chunk_sectors)
4082		chunk_sectors = mddev->new_chunk_sectors;
4083	max =  (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
4084	if (max < 0) max = 0;
4085	if (max <= biovec->bv_len && bio_sectors == 0)
4086		return biovec->bv_len;
4087	else
4088		return max;
4089}
4090
4091
4092static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)
4093{
4094	sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev);
4095	unsigned int chunk_sectors = mddev->chunk_sectors;
 
4096	unsigned int bio_sectors = bio_sectors(bio);
4097
4098	if (mddev->new_chunk_sectors < mddev->chunk_sectors)
4099		chunk_sectors = mddev->new_chunk_sectors;
 
4100	return  chunk_sectors >=
4101		((sector & (chunk_sectors - 1)) + bio_sectors);
4102}
4103
4104/*
4105 *  add bio to the retry LIFO  ( in O(1) ... we are in interrupt )
4106 *  later sampled by raid5d.
4107 */
4108static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)
4109{
4110	unsigned long flags;
4111
4112	spin_lock_irqsave(&conf->device_lock, flags);
4113
4114	bi->bi_next = conf->retry_read_aligned_list;
4115	conf->retry_read_aligned_list = bi;
4116
4117	spin_unlock_irqrestore(&conf->device_lock, flags);
4118	md_wakeup_thread(conf->mddev->thread);
4119}
4120
4121
4122static struct bio *remove_bio_from_retry(struct r5conf *conf)
4123{
4124	struct bio *bi;
4125
4126	bi = conf->retry_read_aligned;
4127	if (bi) {
 
4128		conf->retry_read_aligned = NULL;
4129		return bi;
4130	}
4131	bi = conf->retry_read_aligned_list;
4132	if(bi) {
4133		conf->retry_read_aligned_list = bi->bi_next;
4134		bi->bi_next = NULL;
4135		/*
4136		 * this sets the active strip count to 1 and the processed
4137		 * strip count to zero (upper 8 bits)
4138		 */
4139		raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */
4140	}
4141
4142	return bi;
4143}
4144
4145
4146/*
4147 *  The "raid5_align_endio" should check if the read succeeded and if it
4148 *  did, call bio_endio on the original bio (having bio_put the new bio
4149 *  first).
4150 *  If the read failed..
4151 */
4152static void raid5_align_endio(struct bio *bi, int error)
4153{
4154	struct bio* raid_bi  = bi->bi_private;
4155	struct mddev *mddev;
4156	struct r5conf *conf;
4157	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
4158	struct md_rdev *rdev;
 
4159
4160	bio_put(bi);
4161
4162	rdev = (void*)raid_bi->bi_next;
4163	raid_bi->bi_next = NULL;
4164	mddev = rdev->mddev;
4165	conf = mddev->private;
4166
4167	rdev_dec_pending(rdev, conf->mddev);
4168
4169	if (!error && uptodate) {
4170		trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev),
4171					 raid_bi, 0);
4172		bio_endio(raid_bi, 0);
4173		if (atomic_dec_and_test(&conf->active_aligned_reads))
4174			wake_up(&conf->wait_for_stripe);
4175		return;
4176	}
4177
4178
4179	pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
4180
4181	add_bio_to_retry(raid_bi, conf);
4182}
4183
4184static int bio_fits_rdev(struct bio *bi)
4185{
4186	struct request_queue *q = bdev_get_queue(bi->bi_bdev);
4187
4188	if (bio_sectors(bi) > queue_max_sectors(q))
4189		return 0;
4190	blk_recount_segments(q, bi);
4191	if (bi->bi_phys_segments > queue_max_segments(q))
4192		return 0;
4193
4194	if (q->merge_bvec_fn)
4195		/* it's too hard to apply the merge_bvec_fn at this stage,
4196		 * just just give up
4197		 */
4198		return 0;
4199
4200	return 1;
4201}
4202
4203
4204static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio)
4205{
4206	struct r5conf *conf = mddev->private;
4207	int dd_idx;
4208	struct bio* align_bi;
4209	struct md_rdev *rdev;
4210	sector_t end_sector;
4211
4212	if (!in_chunk_boundary(mddev, raid_bio)) {
4213		pr_debug("chunk_aligned_read : non aligned\n");
4214		return 0;
4215	}
4216	/*
4217	 * use bio_clone_mddev to make a copy of the bio
4218	 */
4219	align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev);
4220	if (!align_bi)
4221		return 0;
4222	/*
4223	 *   set bi_end_io to a new function, and set bi_private to the
4224	 *     original bio.
4225	 */
4226	align_bi->bi_end_io  = raid5_align_endio;
4227	align_bi->bi_private = raid_bio;
4228	/*
4229	 *	compute position
4230	 */
4231	align_bi->bi_iter.bi_sector =
4232		raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector,
4233				     0, &dd_idx, NULL);
4234
4235	end_sector = bio_end_sector(align_bi);
4236	rcu_read_lock();
4237	rdev = rcu_dereference(conf->disks[dd_idx].replacement);
4238	if (!rdev || test_bit(Faulty, &rdev->flags) ||
4239	    rdev->recovery_offset < end_sector) {
4240		rdev = rcu_dereference(conf->disks[dd_idx].rdev);
4241		if (rdev &&
4242		    (test_bit(Faulty, &rdev->flags) ||
4243		    !(test_bit(In_sync, &rdev->flags) ||
4244		      rdev->recovery_offset >= end_sector)))
4245			rdev = NULL;
4246	}
 
 
 
 
 
 
 
4247	if (rdev) {
4248		sector_t first_bad;
4249		int bad_sectors;
4250
4251		atomic_inc(&rdev->nr_pending);
4252		rcu_read_unlock();
4253		raid_bio->bi_next = (void*)rdev;
4254		align_bi->bi_bdev =  rdev->bdev;
4255		align_bi->bi_flags &= ~(1 << BIO_SEG_VALID);
4256
4257		if (!bio_fits_rdev(align_bi) ||
4258		    is_badblock(rdev, align_bi->bi_iter.bi_sector,
4259				bio_sectors(align_bi),
4260				&first_bad, &bad_sectors)) {
4261			/* too big in some way, or has a known bad block */
4262			bio_put(align_bi);
4263			rdev_dec_pending(rdev, mddev);
4264			return 0;
4265		}
4266
4267		/* No reshape active, so we can trust rdev->data_offset */
4268		align_bi->bi_iter.bi_sector += rdev->data_offset;
4269
4270		spin_lock_irq(&conf->device_lock);
4271		wait_event_lock_irq(conf->wait_for_stripe,
4272				    conf->quiesce == 0,
4273				    conf->device_lock);
4274		atomic_inc(&conf->active_aligned_reads);
4275		spin_unlock_irq(&conf->device_lock);
4276
4277		if (mddev->gendisk)
4278			trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev),
4279					      align_bi, disk_devt(mddev->gendisk),
4280					      raid_bio->bi_iter.bi_sector);
4281		generic_make_request(align_bi);
4282		return 1;
4283	} else {
4284		rcu_read_unlock();
4285		bio_put(align_bi);
4286		return 0;
4287	}
4288}
4289
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4290/* __get_priority_stripe - get the next stripe to process
4291 *
4292 * Full stripe writes are allowed to pass preread active stripes up until
4293 * the bypass_threshold is exceeded.  In general the bypass_count
4294 * increments when the handle_list is handled before the hold_list; however, it
4295 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
4296 * stripe with in flight i/o.  The bypass_count will be reset when the
4297 * head of the hold_list has changed, i.e. the head was promoted to the
4298 * handle_list.
4299 */
4300static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group)
4301{
4302	struct stripe_head *sh = NULL, *tmp;
4303	struct list_head *handle_list = NULL;
4304	struct r5worker_group *wg = NULL;
4305
 
 
 
 
 
 
 
4306	if (conf->worker_cnt_per_group == 0) {
4307		handle_list = &conf->handle_list;
 
4308	} else if (group != ANY_GROUP) {
4309		handle_list = &conf->worker_groups[group].handle_list;
 
4310		wg = &conf->worker_groups[group];
4311	} else {
4312		int i;
4313		for (i = 0; i < conf->group_cnt; i++) {
4314			handle_list = &conf->worker_groups[i].handle_list;
 
4315			wg = &conf->worker_groups[i];
4316			if (!list_empty(handle_list))
4317				break;
4318		}
4319	}
4320
4321	pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
4322		  __func__,
4323		  list_empty(handle_list) ? "empty" : "busy",
4324		  list_empty(&conf->hold_list) ? "empty" : "busy",
4325		  atomic_read(&conf->pending_full_writes), conf->bypass_count);
4326
4327	if (!list_empty(handle_list)) {
4328		sh = list_entry(handle_list->next, typeof(*sh), lru);
4329
4330		if (list_empty(&conf->hold_list))
4331			conf->bypass_count = 0;
4332		else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
4333			if (conf->hold_list.next == conf->last_hold)
4334				conf->bypass_count++;
4335			else {
4336				conf->last_hold = conf->hold_list.next;
4337				conf->bypass_count -= conf->bypass_threshold;
4338				if (conf->bypass_count < 0)
4339					conf->bypass_count = 0;
4340			}
4341		}
4342	} else if (!list_empty(&conf->hold_list) &&
4343		   ((conf->bypass_threshold &&
4344		     conf->bypass_count > conf->bypass_threshold) ||
4345		    atomic_read(&conf->pending_full_writes) == 0)) {
4346
4347		list_for_each_entry(tmp, &conf->hold_list,  lru) {
4348			if (conf->worker_cnt_per_group == 0 ||
4349			    group == ANY_GROUP ||
4350			    !cpu_online(tmp->cpu) ||
4351			    cpu_to_group(tmp->cpu) == group) {
4352				sh = tmp;
4353				break;
4354			}
4355		}
4356
4357		if (sh) {
4358			conf->bypass_count -= conf->bypass_threshold;
4359			if (conf->bypass_count < 0)
4360				conf->bypass_count = 0;
4361		}
4362		wg = NULL;
4363	}
4364
4365	if (!sh)
4366		return NULL;
 
 
 
 
 
4367
4368	if (wg) {
4369		wg->stripes_cnt--;
4370		sh->group = NULL;
4371	}
4372	list_del_init(&sh->lru);
4373	BUG_ON(atomic_inc_return(&sh->count) != 1);
4374	return sh;
4375}
4376
4377struct raid5_plug_cb {
4378	struct blk_plug_cb	cb;
4379	struct list_head	list;
4380	struct list_head	temp_inactive_list[NR_STRIPE_HASH_LOCKS];
4381};
4382
4383static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule)
4384{
4385	struct raid5_plug_cb *cb = container_of(
4386		blk_cb, struct raid5_plug_cb, cb);
4387	struct stripe_head *sh;
4388	struct mddev *mddev = cb->cb.data;
4389	struct r5conf *conf = mddev->private;
4390	int cnt = 0;
4391	int hash;
4392
4393	if (cb->list.next && !list_empty(&cb->list)) {
4394		spin_lock_irq(&conf->device_lock);
4395		while (!list_empty(&cb->list)) {
4396			sh = list_first_entry(&cb->list, struct stripe_head, lru);
4397			list_del_init(&sh->lru);
4398			/*
4399			 * avoid race release_stripe_plug() sees
4400			 * STRIPE_ON_UNPLUG_LIST clear but the stripe
4401			 * is still in our list
4402			 */
4403			smp_mb__before_clear_bit();
4404			clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);
4405			/*
4406			 * STRIPE_ON_RELEASE_LIST could be set here. In that
4407			 * case, the count is always > 1 here
4408			 */
4409			hash = sh->hash_lock_index;
4410			__release_stripe(conf, sh, &cb->temp_inactive_list[hash]);
4411			cnt++;
4412		}
4413		spin_unlock_irq(&conf->device_lock);
4414	}
4415	release_inactive_stripe_list(conf, cb->temp_inactive_list,
4416				     NR_STRIPE_HASH_LOCKS);
4417	if (mddev->queue)
4418		trace_block_unplug(mddev->queue, cnt, !from_schedule);
4419	kfree(cb);
4420}
4421
4422static void release_stripe_plug(struct mddev *mddev,
4423				struct stripe_head *sh)
4424{
4425	struct blk_plug_cb *blk_cb = blk_check_plugged(
4426		raid5_unplug, mddev,
4427		sizeof(struct raid5_plug_cb));
4428	struct raid5_plug_cb *cb;
4429
4430	if (!blk_cb) {
4431		release_stripe(sh);
4432		return;
4433	}
4434
4435	cb = container_of(blk_cb, struct raid5_plug_cb, cb);
4436
4437	if (cb->list.next == NULL) {
4438		int i;
4439		INIT_LIST_HEAD(&cb->list);
4440		for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
4441			INIT_LIST_HEAD(cb->temp_inactive_list + i);
4442	}
4443
4444	if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))
4445		list_add_tail(&sh->lru, &cb->list);
4446	else
4447		release_stripe(sh);
4448}
4449
4450static void make_discard_request(struct mddev *mddev, struct bio *bi)
4451{
4452	struct r5conf *conf = mddev->private;
4453	sector_t logical_sector, last_sector;
4454	struct stripe_head *sh;
4455	int remaining;
4456	int stripe_sectors;
4457
4458	if (mddev->reshape_position != MaxSector)
4459		/* Skip discard while reshape is happening */
4460		return;
4461
4462	logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
4463	last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9);
4464
4465	bi->bi_next = NULL;
4466	bi->bi_phys_segments = 1; /* over-loaded to count active stripes */
4467
4468	stripe_sectors = conf->chunk_sectors *
4469		(conf->raid_disks - conf->max_degraded);
4470	logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector,
4471					       stripe_sectors);
4472	sector_div(last_sector, stripe_sectors);
4473
4474	logical_sector *= conf->chunk_sectors;
4475	last_sector *= conf->chunk_sectors;
4476
4477	for (; logical_sector < last_sector;
4478	     logical_sector += STRIPE_SECTORS) {
4479		DEFINE_WAIT(w);
4480		int d;
4481	again:
4482		sh = get_active_stripe(conf, logical_sector, 0, 0, 0);
4483		prepare_to_wait(&conf->wait_for_overlap, &w,
4484				TASK_UNINTERRUPTIBLE);
4485		set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
4486		if (test_bit(STRIPE_SYNCING, &sh->state)) {
4487			release_stripe(sh);
4488			schedule();
4489			goto again;
4490		}
4491		clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
4492		spin_lock_irq(&sh->stripe_lock);
4493		for (d = 0; d < conf->raid_disks; d++) {
4494			if (d == sh->pd_idx || d == sh->qd_idx)
4495				continue;
4496			if (sh->dev[d].towrite || sh->dev[d].toread) {
4497				set_bit(R5_Overlap, &sh->dev[d].flags);
4498				spin_unlock_irq(&sh->stripe_lock);
4499				release_stripe(sh);
4500				schedule();
4501				goto again;
4502			}
4503		}
4504		set_bit(STRIPE_DISCARD, &sh->state);
4505		finish_wait(&conf->wait_for_overlap, &w);
 
4506		for (d = 0; d < conf->raid_disks; d++) {
4507			if (d == sh->pd_idx || d == sh->qd_idx)
4508				continue;
4509			sh->dev[d].towrite = bi;
4510			set_bit(R5_OVERWRITE, &sh->dev[d].flags);
4511			raid5_inc_bi_active_stripes(bi);
 
 
4512		}
4513		spin_unlock_irq(&sh->stripe_lock);
4514		if (conf->mddev->bitmap) {
4515			for (d = 0;
4516			     d < conf->raid_disks - conf->max_degraded;
4517			     d++)
4518				bitmap_startwrite(mddev->bitmap,
4519						  sh->sector,
4520						  STRIPE_SECTORS,
4521						  0);
4522			sh->bm_seq = conf->seq_flush + 1;
4523			set_bit(STRIPE_BIT_DELAY, &sh->state);
4524		}
4525
4526		set_bit(STRIPE_HANDLE, &sh->state);
4527		clear_bit(STRIPE_DELAYED, &sh->state);
4528		if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
4529			atomic_inc(&conf->preread_active_stripes);
4530		release_stripe_plug(mddev, sh);
4531	}
4532
4533	remaining = raid5_dec_bi_active_stripes(bi);
4534	if (remaining == 0) {
4535		md_write_end(mddev);
4536		bio_endio(bi, 0);
4537	}
4538}
4539
4540static void make_request(struct mddev *mddev, struct bio * bi)
4541{
4542	struct r5conf *conf = mddev->private;
4543	int dd_idx;
4544	sector_t new_sector;
4545	sector_t logical_sector, last_sector;
4546	struct stripe_head *sh;
4547	const int rw = bio_data_dir(bi);
4548	int remaining;
4549	DEFINE_WAIT(w);
4550	bool do_prepare;
 
4551
4552	if (unlikely(bi->bi_rw & REQ_FLUSH)) {
4553		md_flush_request(mddev, bi);
4554		return;
4555	}
4556
4557	md_write_start(mddev, bi);
 
 
 
 
 
 
 
 
 
 
 
 
4558
4559	if (rw == READ &&
4560	     mddev->reshape_position == MaxSector &&
4561	     chunk_aligned_read(mddev,bi))
4562		return;
 
 
 
 
 
 
 
 
 
4563
4564	if (unlikely(bi->bi_rw & REQ_DISCARD)) {
4565		make_discard_request(mddev, bi);
4566		return;
 
4567	}
4568
4569	logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
4570	last_sector = bio_end_sector(bi);
4571	bi->bi_next = NULL;
4572	bi->bi_phys_segments = 1;	/* over-loaded to count active stripes */
4573
4574	prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
4575	for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
4576		int previous;
4577		int seq;
4578
4579		do_prepare = false;
4580	retry:
4581		seq = read_seqcount_begin(&conf->gen_lock);
4582		previous = 0;
4583		if (do_prepare)
4584			prepare_to_wait(&conf->wait_for_overlap, &w,
4585				TASK_UNINTERRUPTIBLE);
4586		if (unlikely(conf->reshape_progress != MaxSector)) {
4587			/* spinlock is needed as reshape_progress may be
4588			 * 64bit on a 32bit platform, and so it might be
4589			 * possible to see a half-updated value
4590			 * Of course reshape_progress could change after
4591			 * the lock is dropped, so once we get a reference
4592			 * to the stripe that we think it is, we will have
4593			 * to check again.
4594			 */
4595			spin_lock_irq(&conf->device_lock);
4596			if (mddev->reshape_backwards
4597			    ? logical_sector < conf->reshape_progress
4598			    : logical_sector >= conf->reshape_progress) {
4599				previous = 1;
4600			} else {
4601				if (mddev->reshape_backwards
4602				    ? logical_sector < conf->reshape_safe
4603				    : logical_sector >= conf->reshape_safe) {
4604					spin_unlock_irq(&conf->device_lock);
4605					schedule();
4606					do_prepare = true;
4607					goto retry;
4608				}
4609			}
4610			spin_unlock_irq(&conf->device_lock);
4611		}
4612
4613		new_sector = raid5_compute_sector(conf, logical_sector,
4614						  previous,
4615						  &dd_idx, NULL);
4616		pr_debug("raid456: make_request, sector %llu logical %llu\n",
4617			(unsigned long long)new_sector,
4618			(unsigned long long)logical_sector);
4619
4620		sh = get_active_stripe(conf, new_sector, previous,
4621				       (bi->bi_rw&RWA_MASK), 0);
4622		if (sh) {
4623			if (unlikely(previous)) {
4624				/* expansion might have moved on while waiting for a
4625				 * stripe, so we must do the range check again.
4626				 * Expansion could still move past after this
4627				 * test, but as we are holding a reference to
4628				 * 'sh', we know that if that happens,
4629				 *  STRIPE_EXPANDING will get set and the expansion
4630				 * won't proceed until we finish with the stripe.
4631				 */
4632				int must_retry = 0;
4633				spin_lock_irq(&conf->device_lock);
4634				if (mddev->reshape_backwards
4635				    ? logical_sector >= conf->reshape_progress
4636				    : logical_sector < conf->reshape_progress)
4637					/* mismatch, need to try again */
4638					must_retry = 1;
4639				spin_unlock_irq(&conf->device_lock);
4640				if (must_retry) {
4641					release_stripe(sh);
4642					schedule();
4643					do_prepare = true;
4644					goto retry;
4645				}
4646			}
4647			if (read_seqcount_retry(&conf->gen_lock, seq)) {
4648				/* Might have got the wrong stripe_head
4649				 * by accident
4650				 */
4651				release_stripe(sh);
4652				goto retry;
4653			}
4654
4655			if (rw == WRITE &&
4656			    logical_sector >= mddev->suspend_lo &&
4657			    logical_sector < mddev->suspend_hi) {
4658				release_stripe(sh);
4659				/* As the suspend_* range is controlled by
4660				 * userspace, we want an interruptible
4661				 * wait.
4662				 */
4663				flush_signals(current);
4664				prepare_to_wait(&conf->wait_for_overlap,
4665						&w, TASK_INTERRUPTIBLE);
4666				if (logical_sector >= mddev->suspend_lo &&
4667				    logical_sector < mddev->suspend_hi) {
4668					schedule();
4669					do_prepare = true;
4670				}
4671				goto retry;
4672			}
4673
4674			if (test_bit(STRIPE_EXPANDING, &sh->state) ||
4675			    !add_stripe_bio(sh, bi, dd_idx, rw)) {
4676				/* Stripe is busy expanding or
4677				 * add failed due to overlap.  Flush everything
4678				 * and wait a while
4679				 */
4680				md_wakeup_thread(mddev->thread);
4681				release_stripe(sh);
4682				schedule();
4683				do_prepare = true;
4684				goto retry;
4685			}
4686			set_bit(STRIPE_HANDLE, &sh->state);
 
 
 
 
 
 
 
4687			clear_bit(STRIPE_DELAYED, &sh->state);
4688			if ((bi->bi_rw & REQ_SYNC) &&
 
4689			    !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
4690				atomic_inc(&conf->preread_active_stripes);
4691			release_stripe_plug(mddev, sh);
4692		} else {
4693			/* cannot get stripe for read-ahead, just give-up */
4694			clear_bit(BIO_UPTODATE, &bi->bi_flags);
4695			break;
4696		}
4697	}
4698	finish_wait(&conf->wait_for_overlap, &w);
4699
4700	remaining = raid5_dec_bi_active_stripes(bi);
4701	if (remaining == 0) {
4702
4703		if ( rw == WRITE )
4704			md_write_end(mddev);
4705
4706		trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
4707					 bi, 0);
4708		bio_endio(bi, 0);
4709	}
4710}
4711
4712static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks);
4713
4714static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
4715{
4716	/* reshaping is quite different to recovery/resync so it is
4717	 * handled quite separately ... here.
4718	 *
4719	 * On each call to sync_request, we gather one chunk worth of
4720	 * destination stripes and flag them as expanding.
4721	 * Then we find all the source stripes and request reads.
4722	 * As the reads complete, handle_stripe will copy the data
4723	 * into the destination stripe and release that stripe.
4724	 */
4725	struct r5conf *conf = mddev->private;
4726	struct stripe_head *sh;
 
4727	sector_t first_sector, last_sector;
4728	int raid_disks = conf->previous_raid_disks;
4729	int data_disks = raid_disks - conf->max_degraded;
4730	int new_data_disks = conf->raid_disks - conf->max_degraded;
4731	int i;
4732	int dd_idx;
4733	sector_t writepos, readpos, safepos;
4734	sector_t stripe_addr;
4735	int reshape_sectors;
4736	struct list_head stripes;
 
4737
4738	if (sector_nr == 0) {
4739		/* If restarting in the middle, skip the initial sectors */
4740		if (mddev->reshape_backwards &&
4741		    conf->reshape_progress < raid5_size(mddev, 0, 0)) {
4742			sector_nr = raid5_size(mddev, 0, 0)
4743				- conf->reshape_progress;
 
 
 
 
4744		} else if (!mddev->reshape_backwards &&
4745			   conf->reshape_progress > 0)
4746			sector_nr = conf->reshape_progress;
4747		sector_div(sector_nr, new_data_disks);
4748		if (sector_nr) {
4749			mddev->curr_resync_completed = sector_nr;
4750			sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4751			*skipped = 1;
4752			return sector_nr;
 
4753		}
4754	}
4755
4756	/* We need to process a full chunk at a time.
4757	 * If old and new chunk sizes differ, we need to process the
4758	 * largest of these
4759	 */
4760	if (mddev->new_chunk_sectors > mddev->chunk_sectors)
4761		reshape_sectors = mddev->new_chunk_sectors;
4762	else
4763		reshape_sectors = mddev->chunk_sectors;
4764
4765	/* We update the metadata at least every 10 seconds, or when
4766	 * the data about to be copied would over-write the source of
4767	 * the data at the front of the range.  i.e. one new_stripe
4768	 * along from reshape_progress new_maps to after where
4769	 * reshape_safe old_maps to
4770	 */
4771	writepos = conf->reshape_progress;
4772	sector_div(writepos, new_data_disks);
4773	readpos = conf->reshape_progress;
4774	sector_div(readpos, data_disks);
4775	safepos = conf->reshape_safe;
4776	sector_div(safepos, data_disks);
4777	if (mddev->reshape_backwards) {
4778		writepos -= min_t(sector_t, reshape_sectors, writepos);
 
4779		readpos += reshape_sectors;
4780		safepos += reshape_sectors;
4781	} else {
4782		writepos += reshape_sectors;
 
 
 
 
4783		readpos -= min_t(sector_t, reshape_sectors, readpos);
4784		safepos -= min_t(sector_t, reshape_sectors, safepos);
4785	}
4786
4787	/* Having calculated the 'writepos' possibly use it
4788	 * to set 'stripe_addr' which is where we will write to.
4789	 */
4790	if (mddev->reshape_backwards) {
4791		BUG_ON(conf->reshape_progress == 0);
4792		stripe_addr = writepos;
4793		BUG_ON((mddev->dev_sectors &
4794			~((sector_t)reshape_sectors - 1))
4795		       - reshape_sectors - stripe_addr
4796		       != sector_nr);
4797	} else {
4798		BUG_ON(writepos != sector_nr + reshape_sectors);
4799		stripe_addr = sector_nr;
4800	}
4801
4802	/* 'writepos' is the most advanced device address we might write.
4803	 * 'readpos' is the least advanced device address we might read.
4804	 * 'safepos' is the least address recorded in the metadata as having
4805	 *     been reshaped.
4806	 * If there is a min_offset_diff, these are adjusted either by
4807	 * increasing the safepos/readpos if diff is negative, or
4808	 * increasing writepos if diff is positive.
4809	 * If 'readpos' is then behind 'writepos', there is no way that we can
4810	 * ensure safety in the face of a crash - that must be done by userspace
4811	 * making a backup of the data.  So in that case there is no particular
4812	 * rush to update metadata.
4813	 * Otherwise if 'safepos' is behind 'writepos', then we really need to
4814	 * update the metadata to advance 'safepos' to match 'readpos' so that
4815	 * we can be safe in the event of a crash.
4816	 * So we insist on updating metadata if safepos is behind writepos and
4817	 * readpos is beyond writepos.
4818	 * In any case, update the metadata every 10 seconds.
4819	 * Maybe that number should be configurable, but I'm not sure it is
4820	 * worth it.... maybe it could be a multiple of safemode_delay???
4821	 */
4822	if (conf->min_offset_diff < 0) {
4823		safepos += -conf->min_offset_diff;
4824		readpos += -conf->min_offset_diff;
4825	} else
4826		writepos += conf->min_offset_diff;
4827
4828	if ((mddev->reshape_backwards
4829	     ? (safepos > writepos && readpos < writepos)
4830	     : (safepos < writepos && readpos > writepos)) ||
4831	    time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
4832		/* Cannot proceed until we've updated the superblock... */
4833		wait_event(conf->wait_for_overlap,
4834			   atomic_read(&conf->reshape_stripes)==0
4835			   || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
4836		if (atomic_read(&conf->reshape_stripes) != 0)
4837			return 0;
4838		mddev->reshape_position = conf->reshape_progress;
4839		mddev->curr_resync_completed = sector_nr;
 
 
 
 
 
 
 
 
 
4840		conf->reshape_checkpoint = jiffies;
4841		set_bit(MD_CHANGE_DEVS, &mddev->flags);
4842		md_wakeup_thread(mddev->thread);
4843		wait_event(mddev->sb_wait, mddev->flags == 0 ||
4844			   test_bit(MD_RECOVERY_INTR, &mddev->recovery));
4845		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
4846			return 0;
4847		spin_lock_irq(&conf->device_lock);
4848		conf->reshape_safe = mddev->reshape_position;
4849		spin_unlock_irq(&conf->device_lock);
4850		wake_up(&conf->wait_for_overlap);
4851		sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4852	}
4853
4854	INIT_LIST_HEAD(&stripes);
4855	for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
4856		int j;
4857		int skipped_disk = 0;
4858		sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
4859		set_bit(STRIPE_EXPANDING, &sh->state);
4860		atomic_inc(&conf->reshape_stripes);
4861		/* If any of this stripe is beyond the end of the old
4862		 * array, then we need to zero those blocks
4863		 */
4864		for (j=sh->disks; j--;) {
4865			sector_t s;
4866			if (j == sh->pd_idx)
4867				continue;
4868			if (conf->level == 6 &&
4869			    j == sh->qd_idx)
4870				continue;
4871			s = compute_blocknr(sh, j, 0);
4872			if (s < raid5_size(mddev, 0, 0)) {
4873				skipped_disk = 1;
4874				continue;
4875			}
4876			memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
4877			set_bit(R5_Expanded, &sh->dev[j].flags);
4878			set_bit(R5_UPTODATE, &sh->dev[j].flags);
4879		}
4880		if (!skipped_disk) {
4881			set_bit(STRIPE_EXPAND_READY, &sh->state);
4882			set_bit(STRIPE_HANDLE, &sh->state);
4883		}
4884		list_add(&sh->lru, &stripes);
4885	}
4886	spin_lock_irq(&conf->device_lock);
4887	if (mddev->reshape_backwards)
4888		conf->reshape_progress -= reshape_sectors * new_data_disks;
4889	else
4890		conf->reshape_progress += reshape_sectors * new_data_disks;
4891	spin_unlock_irq(&conf->device_lock);
4892	/* Ok, those stripe are ready. We can start scheduling
4893	 * reads on the source stripes.
4894	 * The source stripes are determined by mapping the first and last
4895	 * block on the destination stripes.
4896	 */
4897	first_sector =
4898		raid5_compute_sector(conf, stripe_addr*(new_data_disks),
4899				     1, &dd_idx, NULL);
4900	last_sector =
4901		raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
4902					    * new_data_disks - 1),
4903				     1, &dd_idx, NULL);
4904	if (last_sector >= mddev->dev_sectors)
4905		last_sector = mddev->dev_sectors - 1;
4906	while (first_sector <= last_sector) {
4907		sh = get_active_stripe(conf, first_sector, 1, 0, 1);
4908		set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
4909		set_bit(STRIPE_HANDLE, &sh->state);
4910		release_stripe(sh);
4911		first_sector += STRIPE_SECTORS;
4912	}
4913	/* Now that the sources are clearly marked, we can release
4914	 * the destination stripes
4915	 */
4916	while (!list_empty(&stripes)) {
4917		sh = list_entry(stripes.next, struct stripe_head, lru);
4918		list_del_init(&sh->lru);
4919		release_stripe(sh);
4920	}
4921	/* If this takes us to the resync_max point where we have to pause,
4922	 * then we need to write out the superblock.
4923	 */
4924	sector_nr += reshape_sectors;
4925	if ((sector_nr - mddev->curr_resync_completed) * 2
 
 
 
4926	    >= mddev->resync_max - mddev->curr_resync_completed) {
4927		/* Cannot proceed until we've updated the superblock... */
4928		wait_event(conf->wait_for_overlap,
4929			   atomic_read(&conf->reshape_stripes) == 0
4930			   || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
4931		if (atomic_read(&conf->reshape_stripes) != 0)
4932			goto ret;
4933		mddev->reshape_position = conf->reshape_progress;
4934		mddev->curr_resync_completed = sector_nr;
 
 
 
 
 
 
 
 
4935		conf->reshape_checkpoint = jiffies;
4936		set_bit(MD_CHANGE_DEVS, &mddev->flags);
4937		md_wakeup_thread(mddev->thread);
4938		wait_event(mddev->sb_wait,
4939			   !test_bit(MD_CHANGE_DEVS, &mddev->flags)
4940			   || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
4941		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
4942			goto ret;
4943		spin_lock_irq(&conf->device_lock);
4944		conf->reshape_safe = mddev->reshape_position;
4945		spin_unlock_irq(&conf->device_lock);
4946		wake_up(&conf->wait_for_overlap);
4947		sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4948	}
4949ret:
4950	return reshape_sectors;
4951}
4952
4953/* FIXME go_faster isn't used */
4954static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped, int go_faster)
4955{
4956	struct r5conf *conf = mddev->private;
4957	struct stripe_head *sh;
4958	sector_t max_sector = mddev->dev_sectors;
4959	sector_t sync_blocks;
4960	int still_degraded = 0;
4961	int i;
4962
4963	if (sector_nr >= max_sector) {
4964		/* just being told to finish up .. nothing much to do */
4965
4966		if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
4967			end_reshape(conf);
4968			return 0;
4969		}
4970
4971		if (mddev->curr_resync < max_sector) /* aborted */
4972			bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
4973					&sync_blocks, 1);
4974		else /* completed sync */
4975			conf->fullsync = 0;
4976		bitmap_close_sync(mddev->bitmap);
4977
4978		return 0;
4979	}
4980
4981	/* Allow raid5_quiesce to complete */
4982	wait_event(conf->wait_for_overlap, conf->quiesce != 2);
4983
4984	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
4985		return reshape_request(mddev, sector_nr, skipped);
4986
4987	/* No need to check resync_max as we never do more than one
4988	 * stripe, and as resync_max will always be on a chunk boundary,
4989	 * if the check in md_do_sync didn't fire, there is no chance
4990	 * of overstepping resync_max here
4991	 */
4992
4993	/* if there is too many failed drives and we are trying
4994	 * to resync, then assert that we are finished, because there is
4995	 * nothing we can do.
4996	 */
4997	if (mddev->degraded >= conf->max_degraded &&
4998	    test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
4999		sector_t rv = mddev->dev_sectors - sector_nr;
5000		*skipped = 1;
5001		return rv;
5002	}
5003	if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
5004	    !conf->fullsync &&
5005	    !bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
5006	    sync_blocks >= STRIPE_SECTORS) {
5007		/* we can skip this block, and probably more */
5008		sync_blocks /= STRIPE_SECTORS;
5009		*skipped = 1;
5010		return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
5011	}
5012
5013	bitmap_cond_end_sync(mddev->bitmap, sector_nr);
5014
5015	sh = get_active_stripe(conf, sector_nr, 0, 1, 0);
5016	if (sh == NULL) {
5017		sh = get_active_stripe(conf, sector_nr, 0, 0, 0);
5018		/* make sure we don't swamp the stripe cache if someone else
5019		 * is trying to get access
5020		 */
5021		schedule_timeout_uninterruptible(1);
5022	}
5023	/* Need to check if array will still be degraded after recovery/resync
5024	 * We don't need to check the 'failed' flag as when that gets set,
5025	 * recovery aborts.
5026	 */
5027	for (i = 0; i < conf->raid_disks; i++)
5028		if (conf->disks[i].rdev == NULL)
 
 
 
5029			still_degraded = 1;
 
 
5030
5031	bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);
5032
5033	set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
 
5034
5035	handle_stripe(sh);
5036	release_stripe(sh);
5037
5038	return STRIPE_SECTORS;
5039}
5040
5041static int  retry_aligned_read(struct r5conf *conf, struct bio *raid_bio)
 
5042{
5043	/* We may not be able to submit a whole bio at once as there
5044	 * may not be enough stripe_heads available.
5045	 * We cannot pre-allocate enough stripe_heads as we may need
5046	 * more than exist in the cache (if we allow ever large chunks).
5047	 * So we do one stripe head at a time and record in
5048	 * ->bi_hw_segments how many have been done.
5049	 *
5050	 * We *know* that this entire raid_bio is in one chunk, so
5051	 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
5052	 */
5053	struct stripe_head *sh;
5054	int dd_idx;
5055	sector_t sector, logical_sector, last_sector;
5056	int scnt = 0;
5057	int remaining;
5058	int handled = 0;
5059
5060	logical_sector = raid_bio->bi_iter.bi_sector &
5061		~((sector_t)STRIPE_SECTORS-1);
5062	sector = raid5_compute_sector(conf, logical_sector,
5063				      0, &dd_idx, NULL);
5064	last_sector = bio_end_sector(raid_bio);
5065
5066	for (; logical_sector < last_sector;
5067	     logical_sector += STRIPE_SECTORS,
5068		     sector += STRIPE_SECTORS,
5069		     scnt++) {
5070
5071		if (scnt < raid5_bi_processed_stripes(raid_bio))
5072			/* already done this stripe */
5073			continue;
5074
5075		sh = get_active_stripe(conf, sector, 0, 1, 0);
5076
5077		if (!sh) {
5078			/* failed to get a stripe - must wait */
5079			raid5_set_bi_processed_stripes(raid_bio, scnt);
5080			conf->retry_read_aligned = raid_bio;
 
5081			return handled;
5082		}
5083
5084		if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) {
5085			release_stripe(sh);
5086			raid5_set_bi_processed_stripes(raid_bio, scnt);
5087			conf->retry_read_aligned = raid_bio;
 
5088			return handled;
5089		}
5090
5091		set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);
5092		handle_stripe(sh);
5093		release_stripe(sh);
5094		handled++;
5095	}
5096	remaining = raid5_dec_bi_active_stripes(raid_bio);
5097	if (remaining == 0) {
5098		trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev),
5099					 raid_bio, 0);
5100		bio_endio(raid_bio, 0);
5101	}
5102	if (atomic_dec_and_test(&conf->active_aligned_reads))
5103		wake_up(&conf->wait_for_stripe);
5104	return handled;
5105}
5106
5107static int handle_active_stripes(struct r5conf *conf, int group,
5108				 struct r5worker *worker,
5109				 struct list_head *temp_inactive_list)
 
 
5110{
5111	struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;
5112	int i, batch_size = 0, hash;
5113	bool release_inactive = false;
5114
5115	while (batch_size < MAX_STRIPE_BATCH &&
5116			(sh = __get_priority_stripe(conf, group)) != NULL)
5117		batch[batch_size++] = sh;
5118
5119	if (batch_size == 0) {
5120		for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
5121			if (!list_empty(temp_inactive_list + i))
5122				break;
5123		if (i == NR_STRIPE_HASH_LOCKS)
 
 
 
5124			return batch_size;
 
5125		release_inactive = true;
5126	}
5127	spin_unlock_irq(&conf->device_lock);
5128
5129	release_inactive_stripe_list(conf, temp_inactive_list,
5130				     NR_STRIPE_HASH_LOCKS);
5131
 
5132	if (release_inactive) {
5133		spin_lock_irq(&conf->device_lock);
5134		return 0;
5135	}
5136
5137	for (i = 0; i < batch_size; i++)
5138		handle_stripe(batch[i]);
 
5139
5140	cond_resched();
5141
5142	spin_lock_irq(&conf->device_lock);
5143	for (i = 0; i < batch_size; i++) {
5144		hash = batch[i]->hash_lock_index;
5145		__release_stripe(conf, batch[i], &temp_inactive_list[hash]);
5146	}
5147	return batch_size;
5148}
5149
5150static void raid5_do_work(struct work_struct *work)
5151{
5152	struct r5worker *worker = container_of(work, struct r5worker, work);
5153	struct r5worker_group *group = worker->group;
5154	struct r5conf *conf = group->conf;
 
5155	int group_id = group - conf->worker_groups;
5156	int handled;
5157	struct blk_plug plug;
5158
5159	pr_debug("+++ raid5worker active\n");
5160
5161	blk_start_plug(&plug);
5162	handled = 0;
5163	spin_lock_irq(&conf->device_lock);
5164	while (1) {
5165		int batch_size, released;
5166
5167		released = release_stripe_list(conf, worker->temp_inactive_list);
5168
5169		batch_size = handle_active_stripes(conf, group_id, worker,
5170						   worker->temp_inactive_list);
5171		worker->working = false;
5172		if (!batch_size && !released)
5173			break;
5174		handled += batch_size;
 
 
 
5175	}
5176	pr_debug("%d stripes handled\n", handled);
5177
5178	spin_unlock_irq(&conf->device_lock);
 
 
 
 
 
 
5179	blk_finish_plug(&plug);
5180
5181	pr_debug("--- raid5worker inactive\n");
5182}
5183
5184/*
5185 * This is our raid5 kernel thread.
5186 *
5187 * We scan the hash table for stripes which can be handled now.
5188 * During the scan, completed stripes are saved for us by the interrupt
5189 * handler, so that they will not have to wait for our next wakeup.
5190 */
5191static void raid5d(struct md_thread *thread)
5192{
5193	struct mddev *mddev = thread->mddev;
5194	struct r5conf *conf = mddev->private;
5195	int handled;
5196	struct blk_plug plug;
5197
5198	pr_debug("+++ raid5d active\n");
5199
5200	md_check_recovery(mddev);
5201
5202	blk_start_plug(&plug);
5203	handled = 0;
5204	spin_lock_irq(&conf->device_lock);
5205	while (1) {
5206		struct bio *bio;
5207		int batch_size, released;
 
5208
5209		released = release_stripe_list(conf, conf->temp_inactive_list);
 
 
5210
5211		if (
5212		    !list_empty(&conf->bitmap_list)) {
5213			/* Now is a good time to flush some bitmap updates */
5214			conf->seq_flush++;
5215			spin_unlock_irq(&conf->device_lock);
5216			bitmap_unplug(mddev->bitmap);
5217			spin_lock_irq(&conf->device_lock);
5218			conf->seq_write = conf->seq_flush;
5219			activate_bit_delay(conf, conf->temp_inactive_list);
5220		}
5221		raid5_activate_delayed(conf);
5222
5223		while ((bio = remove_bio_from_retry(conf))) {
5224			int ok;
5225			spin_unlock_irq(&conf->device_lock);
5226			ok = retry_aligned_read(conf, bio);
5227			spin_lock_irq(&conf->device_lock);
5228			if (!ok)
5229				break;
5230			handled++;
5231		}
5232
5233		batch_size = handle_active_stripes(conf, ANY_GROUP, NULL,
5234						   conf->temp_inactive_list);
5235		if (!batch_size && !released)
5236			break;
5237		handled += batch_size;
5238
5239		if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) {
5240			spin_unlock_irq(&conf->device_lock);
5241			md_check_recovery(mddev);
5242			spin_lock_irq(&conf->device_lock);
5243		}
5244	}
5245	pr_debug("%d stripes handled\n", handled);
5246
5247	spin_unlock_irq(&conf->device_lock);
 
 
 
 
 
 
 
 
 
 
 
 
 
5248
5249	async_tx_issue_pending_all();
5250	blk_finish_plug(&plug);
5251
5252	pr_debug("--- raid5d inactive\n");
5253}
5254
5255static ssize_t
5256raid5_show_stripe_cache_size(struct mddev *mddev, char *page)
5257{
5258	struct r5conf *conf = mddev->private;
 
 
 
5259	if (conf)
5260		return sprintf(page, "%d\n", conf->max_nr_stripes);
5261	else
5262		return 0;
5263}
5264
5265int
5266raid5_set_cache_size(struct mddev *mddev, int size)
5267{
 
5268	struct r5conf *conf = mddev->private;
5269	int err;
5270	int hash;
5271
5272	if (size <= 16 || size > 32768)
5273		return -EINVAL;
5274	hash = (conf->max_nr_stripes - 1) % NR_STRIPE_HASH_LOCKS;
5275	while (size < conf->max_nr_stripes) {
5276		if (drop_one_stripe(conf, hash))
5277			conf->max_nr_stripes--;
5278		else
 
 
 
 
 
 
 
 
 
 
5279			break;
5280		hash--;
5281		if (hash < 0)
5282			hash = NR_STRIPE_HASH_LOCKS - 1;
5283	}
5284	err = md_allow_write(mddev);
5285	if (err)
5286		return err;
5287	hash = conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;
5288	while (size > conf->max_nr_stripes) {
5289		if (grow_one_stripe(conf, hash))
5290			conf->max_nr_stripes++;
5291		else break;
5292		hash = (hash + 1) % NR_STRIPE_HASH_LOCKS;
5293	}
5294	return 0;
5295}
5296EXPORT_SYMBOL(raid5_set_cache_size);
5297
5298static ssize_t
5299raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len)
5300{
5301	struct r5conf *conf = mddev->private;
5302	unsigned long new;
5303	int err;
5304
5305	if (len >= PAGE_SIZE)
5306		return -EINVAL;
5307	if (!conf)
5308		return -ENODEV;
5309
5310	if (kstrtoul(page, 10, &new))
5311		return -EINVAL;
5312	err = raid5_set_cache_size(mddev, new);
5313	if (err)
5314		return err;
5315	return len;
 
 
 
 
 
 
 
5316}
5317
5318static struct md_sysfs_entry
5319raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
5320				raid5_show_stripe_cache_size,
5321				raid5_store_stripe_cache_size);
5322
5323static ssize_t
5324raid5_show_preread_threshold(struct mddev *mddev, char *page)
5325{
5326	struct r5conf *conf = mddev->private;
5327	if (conf)
5328		return sprintf(page, "%d\n", conf->bypass_threshold);
5329	else
5330		return 0;
5331}
5332
5333static ssize_t
5334raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len)
5335{
5336	struct r5conf *conf = mddev->private;
5337	unsigned long new;
5338	if (len >= PAGE_SIZE)
5339		return -EINVAL;
5340	if (!conf)
5341		return -ENODEV;
5342
 
 
 
5343	if (kstrtoul(page, 10, &new))
5344		return -EINVAL;
5345	if (new > conf->max_nr_stripes)
 
5346		return -EINVAL;
5347	conf->bypass_threshold = new;
 
 
 
 
 
 
5348	return len;
5349}
5350
5351static struct md_sysfs_entry
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5352raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
5353					S_IRUGO | S_IWUSR,
5354					raid5_show_preread_threshold,
5355					raid5_store_preread_threshold);
5356
5357static ssize_t
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5358stripe_cache_active_show(struct mddev *mddev, char *page)
5359{
5360	struct r5conf *conf = mddev->private;
5361	if (conf)
5362		return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
5363	else
5364		return 0;
5365}
5366
5367static struct md_sysfs_entry
5368raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
5369
5370static ssize_t
5371raid5_show_group_thread_cnt(struct mddev *mddev, char *page)
5372{
5373	struct r5conf *conf = mddev->private;
 
 
 
5374	if (conf)
5375		return sprintf(page, "%d\n", conf->worker_cnt_per_group);
5376	else
5377		return 0;
5378}
5379
5380static int alloc_thread_groups(struct r5conf *conf, int cnt,
5381			       int *group_cnt,
5382			       int *worker_cnt_per_group,
5383			       struct r5worker_group **worker_groups);
5384static ssize_t
5385raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len)
5386{
5387	struct r5conf *conf = mddev->private;
5388	unsigned long new;
5389	int err;
5390	struct r5worker_group *new_groups, *old_groups;
5391	int group_cnt, worker_cnt_per_group;
5392
5393	if (len >= PAGE_SIZE)
5394		return -EINVAL;
5395	if (!conf)
5396		return -ENODEV;
5397
5398	if (kstrtoul(page, 10, &new))
5399		return -EINVAL;
5400
5401	if (new == conf->worker_cnt_per_group)
5402		return len;
 
 
 
 
 
 
5403
5404	mddev_suspend(mddev);
 
 
5405
5406	old_groups = conf->worker_groups;
5407	if (old_groups)
5408		flush_workqueue(raid5_wq);
5409
5410	err = alloc_thread_groups(conf, new,
5411				  &group_cnt, &worker_cnt_per_group,
5412				  &new_groups);
5413	if (!err) {
5414		spin_lock_irq(&conf->device_lock);
5415		conf->group_cnt = group_cnt;
5416		conf->worker_cnt_per_group = worker_cnt_per_group;
5417		conf->worker_groups = new_groups;
5418		spin_unlock_irq(&conf->device_lock);
5419
5420		if (old_groups)
5421			kfree(old_groups[0].workers);
5422		kfree(old_groups);
 
 
5423	}
 
5424
5425	mddev_resume(mddev);
5426
5427	if (err)
5428		return err;
5429	return len;
5430}
5431
5432static struct md_sysfs_entry
5433raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR,
5434				raid5_show_group_thread_cnt,
5435				raid5_store_group_thread_cnt);
5436
5437static struct attribute *raid5_attrs[] =  {
5438	&raid5_stripecache_size.attr,
5439	&raid5_stripecache_active.attr,
5440	&raid5_preread_bypass_threshold.attr,
5441	&raid5_group_thread_cnt.attr,
 
 
 
 
5442	NULL,
5443};
5444static struct attribute_group raid5_attrs_group = {
5445	.name = NULL,
5446	.attrs = raid5_attrs,
5447};
5448
5449static int alloc_thread_groups(struct r5conf *conf, int cnt,
5450			       int *group_cnt,
5451			       int *worker_cnt_per_group,
5452			       struct r5worker_group **worker_groups)
5453{
5454	int i, j, k;
5455	ssize_t size;
5456	struct r5worker *workers;
5457
5458	*worker_cnt_per_group = cnt;
5459	if (cnt == 0) {
5460		*group_cnt = 0;
5461		*worker_groups = NULL;
5462		return 0;
5463	}
5464	*group_cnt = num_possible_nodes();
5465	size = sizeof(struct r5worker) * cnt;
5466	workers = kzalloc(size * *group_cnt, GFP_NOIO);
5467	*worker_groups = kzalloc(sizeof(struct r5worker_group) *
5468				*group_cnt, GFP_NOIO);
5469	if (!*worker_groups || !workers) {
5470		kfree(workers);
5471		kfree(*worker_groups);
5472		return -ENOMEM;
5473	}
5474
5475	for (i = 0; i < *group_cnt; i++) {
5476		struct r5worker_group *group;
5477
5478		group = &(*worker_groups)[i];
5479		INIT_LIST_HEAD(&group->handle_list);
 
5480		group->conf = conf;
5481		group->workers = workers + i * cnt;
5482
5483		for (j = 0; j < cnt; j++) {
5484			struct r5worker *worker = group->workers + j;
5485			worker->group = group;
5486			INIT_WORK(&worker->work, raid5_do_work);
5487
5488			for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++)
5489				INIT_LIST_HEAD(worker->temp_inactive_list + k);
5490		}
5491	}
5492
5493	return 0;
5494}
5495
5496static void free_thread_groups(struct r5conf *conf)
5497{
5498	if (conf->worker_groups)
5499		kfree(conf->worker_groups[0].workers);
5500	kfree(conf->worker_groups);
5501	conf->worker_groups = NULL;
5502}
5503
5504static sector_t
5505raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks)
5506{
5507	struct r5conf *conf = mddev->private;
5508
5509	if (!sectors)
5510		sectors = mddev->dev_sectors;
5511	if (!raid_disks)
5512		/* size is defined by the smallest of previous and new size */
5513		raid_disks = min(conf->raid_disks, conf->previous_raid_disks);
5514
5515	sectors &= ~((sector_t)mddev->chunk_sectors - 1);
5516	sectors &= ~((sector_t)mddev->new_chunk_sectors - 1);
5517	return sectors * (raid_disks - conf->max_degraded);
5518}
5519
5520static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
5521{
5522	safe_put_page(percpu->spare_page);
5523	kfree(percpu->scribble);
5524	percpu->spare_page = NULL;
 
5525	percpu->scribble = NULL;
5526}
5527
5528static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
5529{
5530	if (conf->level == 6 && !percpu->spare_page)
5531		percpu->spare_page = alloc_page(GFP_KERNEL);
5532	if (!percpu->scribble)
5533		percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL);
 
5534
5535	if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) {
 
 
 
 
 
 
5536		free_scratch_buffer(conf, percpu);
5537		return -ENOMEM;
5538	}
5539
5540	return 0;
5541}
5542
5543static void raid5_free_percpu(struct r5conf *conf)
5544{
5545	unsigned long cpu;
 
 
 
 
5546
 
 
5547	if (!conf->percpu)
5548		return;
5549
5550#ifdef CONFIG_HOTPLUG_CPU
5551	unregister_cpu_notifier(&conf->cpu_notify);
5552#endif
5553
5554	get_online_cpus();
5555	for_each_possible_cpu(cpu)
5556		free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
5557	put_online_cpus();
5558
5559	free_percpu(conf->percpu);
5560}
5561
5562static void free_conf(struct r5conf *conf)
5563{
 
 
 
 
 
5564	free_thread_groups(conf);
5565	shrink_stripes(conf);
5566	raid5_free_percpu(conf);
 
 
 
5567	kfree(conf->disks);
 
5568	kfree(conf->stripe_hashtbl);
 
5569	kfree(conf);
5570}
5571
5572#ifdef CONFIG_HOTPLUG_CPU
5573static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action,
5574			      void *hcpu)
5575{
5576	struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify);
5577	long cpu = (long)hcpu;
5578	struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);
5579
5580	switch (action) {
5581	case CPU_UP_PREPARE:
5582	case CPU_UP_PREPARE_FROZEN:
5583		if (alloc_scratch_buffer(conf, percpu)) {
5584			pr_err("%s: failed memory allocation for cpu%ld\n",
5585			       __func__, cpu);
5586			return notifier_from_errno(-ENOMEM);
5587		}
5588		break;
5589	case CPU_DEAD:
5590	case CPU_DEAD_FROZEN:
5591		free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
5592		break;
5593	default:
5594		break;
5595	}
5596	return NOTIFY_OK;
5597}
5598#endif
5599
5600static int raid5_alloc_percpu(struct r5conf *conf)
5601{
5602	unsigned long cpu;
5603	int err = 0;
5604
5605	conf->percpu = alloc_percpu(struct raid5_percpu);
5606	if (!conf->percpu)
5607		return -ENOMEM;
5608
5609#ifdef CONFIG_HOTPLUG_CPU
5610	conf->cpu_notify.notifier_call = raid456_cpu_notify;
5611	conf->cpu_notify.priority = 0;
5612	err = register_cpu_notifier(&conf->cpu_notify);
5613	if (err)
5614		return err;
5615#endif
 
 
5616
5617	get_online_cpus();
5618	for_each_present_cpu(cpu) {
5619		err = alloc_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
5620		if (err) {
5621			pr_err("%s: failed memory allocation for cpu%ld\n",
5622			       __func__, cpu);
5623			break;
 
 
 
 
 
 
 
 
5624		}
 
5625	}
5626	put_online_cpus();
 
5627
5628	return err;
 
 
 
 
 
 
 
 
5629}
5630
5631static struct r5conf *setup_conf(struct mddev *mddev)
5632{
5633	struct r5conf *conf;
5634	int raid_disk, memory, max_disks;
5635	struct md_rdev *rdev;
5636	struct disk_info *disk;
5637	char pers_name[6];
5638	int i;
5639	int group_cnt, worker_cnt_per_group;
5640	struct r5worker_group *new_group;
 
5641
5642	if (mddev->new_level != 5
5643	    && mddev->new_level != 4
5644	    && mddev->new_level != 6) {
5645		printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n",
5646		       mdname(mddev), mddev->new_level);
5647		return ERR_PTR(-EIO);
5648	}
5649	if ((mddev->new_level == 5
5650	     && !algorithm_valid_raid5(mddev->new_layout)) ||
5651	    (mddev->new_level == 6
5652	     && !algorithm_valid_raid6(mddev->new_layout))) {
5653		printk(KERN_ERR "md/raid:%s: layout %d not supported\n",
5654		       mdname(mddev), mddev->new_layout);
5655		return ERR_PTR(-EIO);
5656	}
5657	if (mddev->new_level == 6 && mddev->raid_disks < 4) {
5658		printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n",
5659		       mdname(mddev), mddev->raid_disks);
5660		return ERR_PTR(-EINVAL);
5661	}
5662
5663	if (!mddev->new_chunk_sectors ||
5664	    (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
5665	    !is_power_of_2(mddev->new_chunk_sectors)) {
5666		printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n",
5667		       mdname(mddev), mddev->new_chunk_sectors << 9);
5668		return ERR_PTR(-EINVAL);
5669	}
5670
5671	conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
5672	if (conf == NULL)
5673		goto abort;
 
 
 
 
 
 
 
 
 
5674	/* Don't enable multi-threading by default*/
5675	if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group,
5676				 &new_group)) {
5677		conf->group_cnt = group_cnt;
5678		conf->worker_cnt_per_group = worker_cnt_per_group;
5679		conf->worker_groups = new_group;
5680	} else
5681		goto abort;
5682	spin_lock_init(&conf->device_lock);
5683	seqcount_init(&conf->gen_lock);
 
 
5684	init_waitqueue_head(&conf->wait_for_stripe);
5685	init_waitqueue_head(&conf->wait_for_overlap);
5686	INIT_LIST_HEAD(&conf->handle_list);
 
5687	INIT_LIST_HEAD(&conf->hold_list);
5688	INIT_LIST_HEAD(&conf->delayed_list);
5689	INIT_LIST_HEAD(&conf->bitmap_list);
5690	init_llist_head(&conf->released_stripes);
5691	atomic_set(&conf->active_stripes, 0);
5692	atomic_set(&conf->preread_active_stripes, 0);
5693	atomic_set(&conf->active_aligned_reads, 0);
 
 
 
 
 
 
 
 
 
 
 
5694	conf->bypass_threshold = BYPASS_THRESHOLD;
5695	conf->recovery_disabled = mddev->recovery_disabled - 1;
5696
5697	conf->raid_disks = mddev->raid_disks;
5698	if (mddev->reshape_position == MaxSector)
5699		conf->previous_raid_disks = mddev->raid_disks;
5700	else
5701		conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
5702	max_disks = max(conf->raid_disks, conf->previous_raid_disks);
5703	conf->scribble_len = scribble_len(max_disks);
5704
5705	conf->disks = kzalloc(max_disks * sizeof(struct disk_info),
5706			      GFP_KERNEL);
 
5707	if (!conf->disks)
5708		goto abort;
5709
 
 
 
 
 
 
 
 
 
5710	conf->mddev = mddev;
5711
5712	if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
5713		goto abort;
5714
5715	/* We init hash_locks[0] separately to that it can be used
5716	 * as the reference lock in the spin_lock_nest_lock() call
5717	 * in lock_all_device_hash_locks_irq in order to convince
5718	 * lockdep that we know what we are doing.
5719	 */
5720	spin_lock_init(conf->hash_locks);
5721	for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
5722		spin_lock_init(conf->hash_locks + i);
5723
5724	for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
5725		INIT_LIST_HEAD(conf->inactive_list + i);
5726
5727	for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
5728		INIT_LIST_HEAD(conf->temp_inactive_list + i);
5729
 
 
 
 
 
 
 
5730	conf->level = mddev->new_level;
 
5731	if (raid5_alloc_percpu(conf) != 0)
5732		goto abort;
5733
5734	pr_debug("raid456: run(%s) called.\n", mdname(mddev));
5735
5736	rdev_for_each(rdev, mddev) {
5737		raid_disk = rdev->raid_disk;
5738		if (raid_disk >= max_disks
5739		    || raid_disk < 0)
5740			continue;
5741		disk = conf->disks + raid_disk;
5742
5743		if (test_bit(Replacement, &rdev->flags)) {
5744			if (disk->replacement)
5745				goto abort;
5746			disk->replacement = rdev;
5747		} else {
5748			if (disk->rdev)
5749				goto abort;
5750			disk->rdev = rdev;
5751		}
5752
5753		if (test_bit(In_sync, &rdev->flags)) {
5754			char b[BDEVNAME_SIZE];
5755			printk(KERN_INFO "md/raid:%s: device %s operational as raid"
5756			       " disk %d\n",
5757			       mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
5758		} else if (rdev->saved_raid_disk != raid_disk)
5759			/* Cannot rely on bitmap to complete recovery */
5760			conf->fullsync = 1;
5761	}
5762
5763	conf->chunk_sectors = mddev->new_chunk_sectors;
5764	conf->level = mddev->new_level;
5765	if (conf->level == 6)
5766		conf->max_degraded = 2;
5767	else
 
 
 
 
5768		conf->max_degraded = 1;
 
 
5769	conf->algorithm = mddev->new_layout;
5770	conf->reshape_progress = mddev->reshape_position;
5771	if (conf->reshape_progress != MaxSector) {
5772		conf->prev_chunk_sectors = mddev->chunk_sectors;
5773		conf->prev_algo = mddev->layout;
 
 
 
5774	}
5775
5776	memory = conf->max_nr_stripes * (sizeof(struct stripe_head) +
 
 
 
 
 
 
 
 
 
 
5777		 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
5778	atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS);
5779	if (grow_stripes(conf, NR_STRIPES)) {
5780		printk(KERN_ERR
5781		       "md/raid:%s: couldn't allocate %dkB for buffers\n",
5782		       mdname(mddev), memory);
5783		goto abort;
5784	} else
5785		printk(KERN_INFO "md/raid:%s: allocated %dkB\n",
5786		       mdname(mddev), memory);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5787
5788	sprintf(pers_name, "raid%d", mddev->new_level);
5789	conf->thread = md_register_thread(raid5d, mddev, pers_name);
5790	if (!conf->thread) {
5791		printk(KERN_ERR
5792		       "md/raid:%s: couldn't allocate thread.\n",
5793		       mdname(mddev));
5794		goto abort;
5795	}
5796
5797	return conf;
5798
5799 abort:
5800	if (conf) {
5801		free_conf(conf);
5802		return ERR_PTR(-EIO);
5803	} else
5804		return ERR_PTR(-ENOMEM);
5805}
5806
5807
5808static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded)
5809{
5810	switch (algo) {
5811	case ALGORITHM_PARITY_0:
5812		if (raid_disk < max_degraded)
5813			return 1;
5814		break;
5815	case ALGORITHM_PARITY_N:
5816		if (raid_disk >= raid_disks - max_degraded)
5817			return 1;
5818		break;
5819	case ALGORITHM_PARITY_0_6:
5820		if (raid_disk == 0 || 
5821		    raid_disk == raid_disks - 1)
5822			return 1;
5823		break;
5824	case ALGORITHM_LEFT_ASYMMETRIC_6:
5825	case ALGORITHM_RIGHT_ASYMMETRIC_6:
5826	case ALGORITHM_LEFT_SYMMETRIC_6:
5827	case ALGORITHM_RIGHT_SYMMETRIC_6:
5828		if (raid_disk == raid_disks - 1)
5829			return 1;
5830	}
5831	return 0;
5832}
5833
5834static int run(struct mddev *mddev)
5835{
5836	struct r5conf *conf;
5837	int working_disks = 0;
5838	int dirty_parity_disks = 0;
5839	struct md_rdev *rdev;
 
5840	sector_t reshape_offset = 0;
5841	int i;
5842	long long min_offset_diff = 0;
5843	int first = 1;
5844
 
 
 
5845	if (mddev->recovery_cp != MaxSector)
5846		printk(KERN_NOTICE "md/raid:%s: not clean"
5847		       " -- starting background reconstruction\n",
5848		       mdname(mddev));
5849
5850	rdev_for_each(rdev, mddev) {
5851		long long diff;
 
 
 
 
 
5852		if (rdev->raid_disk < 0)
5853			continue;
5854		diff = (rdev->new_data_offset - rdev->data_offset);
5855		if (first) {
5856			min_offset_diff = diff;
5857			first = 0;
5858		} else if (mddev->reshape_backwards &&
5859			 diff < min_offset_diff)
5860			min_offset_diff = diff;
5861		else if (!mddev->reshape_backwards &&
5862			 diff > min_offset_diff)
5863			min_offset_diff = diff;
5864	}
5865
 
 
 
 
 
 
 
5866	if (mddev->reshape_position != MaxSector) {
5867		/* Check that we can continue the reshape.
5868		 * Difficulties arise if the stripe we would write to
5869		 * next is at or after the stripe we would read from next.
5870		 * For a reshape that changes the number of devices, this
5871		 * is only possible for a very short time, and mdadm makes
5872		 * sure that time appears to have past before assembling
5873		 * the array.  So we fail if that time hasn't passed.
5874		 * For a reshape that keeps the number of devices the same
5875		 * mdadm must be monitoring the reshape can keeping the
5876		 * critical areas read-only and backed up.  It will start
5877		 * the array in read-only mode, so we check for that.
5878		 */
5879		sector_t here_new, here_old;
5880		int old_disks;
5881		int max_degraded = (mddev->level == 6 ? 2 : 1);
 
 
 
 
 
 
 
 
5882
5883		if (mddev->new_level != mddev->level) {
5884			printk(KERN_ERR "md/raid:%s: unsupported reshape "
5885			       "required - aborting.\n",
5886			       mdname(mddev));
5887			return -EINVAL;
5888		}
5889		old_disks = mddev->raid_disks - mddev->delta_disks;
5890		/* reshape_position must be on a new-stripe boundary, and one
5891		 * further up in new geometry must map after here in old
5892		 * geometry.
 
 
 
5893		 */
5894		here_new = mddev->reshape_position;
5895		if (sector_div(here_new, mddev->new_chunk_sectors *
5896			       (mddev->raid_disks - max_degraded))) {
5897			printk(KERN_ERR "md/raid:%s: reshape_position not "
5898			       "on a stripe boundary\n", mdname(mddev));
 
5899			return -EINVAL;
5900		}
5901		reshape_offset = here_new * mddev->new_chunk_sectors;
5902		/* here_new is the stripe we will write to */
5903		here_old = mddev->reshape_position;
5904		sector_div(here_old, mddev->chunk_sectors *
5905			   (old_disks-max_degraded));
5906		/* here_old is the first stripe that we might need to read
5907		 * from */
5908		if (mddev->delta_disks == 0) {
5909			if ((here_new * mddev->new_chunk_sectors !=
5910			     here_old * mddev->chunk_sectors)) {
5911				printk(KERN_ERR "md/raid:%s: reshape position is"
5912				       " confused - aborting\n", mdname(mddev));
5913				return -EINVAL;
5914			}
5915			/* We cannot be sure it is safe to start an in-place
5916			 * reshape.  It is only safe if user-space is monitoring
5917			 * and taking constant backups.
5918			 * mdadm always starts a situation like this in
5919			 * readonly mode so it can take control before
5920			 * allowing any writes.  So just check for that.
5921			 */
5922			if (abs(min_offset_diff) >= mddev->chunk_sectors &&
5923			    abs(min_offset_diff) >= mddev->new_chunk_sectors)
5924				/* not really in-place - so OK */;
5925			else if (mddev->ro == 0) {
5926				printk(KERN_ERR "md/raid:%s: in-place reshape "
5927				       "must be started in read-only mode "
5928				       "- aborting\n",
5929				       mdname(mddev));
5930				return -EINVAL;
5931			}
5932		} else if (mddev->reshape_backwards
5933		    ? (here_new * mddev->new_chunk_sectors + min_offset_diff <=
5934		       here_old * mddev->chunk_sectors)
5935		    : (here_new * mddev->new_chunk_sectors >=
5936		       here_old * mddev->chunk_sectors + (-min_offset_diff))) {
5937			/* Reading from the same stripe as writing to - bad */
5938			printk(KERN_ERR "md/raid:%s: reshape_position too early for "
5939			       "auto-recovery - aborting.\n",
5940			       mdname(mddev));
5941			return -EINVAL;
5942		}
5943		printk(KERN_INFO "md/raid:%s: reshape will continue\n",
5944		       mdname(mddev));
5945		/* OK, we should be able to continue; */
5946	} else {
5947		BUG_ON(mddev->level != mddev->new_level);
5948		BUG_ON(mddev->layout != mddev->new_layout);
5949		BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
5950		BUG_ON(mddev->delta_disks != 0);
5951	}
5952
 
 
 
 
 
 
 
 
5953	if (mddev->private == NULL)
5954		conf = setup_conf(mddev);
5955	else
5956		conf = mddev->private;
5957
5958	if (IS_ERR(conf))
5959		return PTR_ERR(conf);
5960
 
 
 
 
 
 
 
 
 
 
5961	conf->min_offset_diff = min_offset_diff;
5962	mddev->thread = conf->thread;
5963	conf->thread = NULL;
5964	mddev->private = conf;
5965
5966	for (i = 0; i < conf->raid_disks && conf->previous_raid_disks;
5967	     i++) {
5968		rdev = conf->disks[i].rdev;
5969		if (!rdev && conf->disks[i].replacement) {
5970			/* The replacement is all we have yet */
5971			rdev = conf->disks[i].replacement;
5972			conf->disks[i].replacement = NULL;
5973			clear_bit(Replacement, &rdev->flags);
5974			conf->disks[i].rdev = rdev;
5975		}
5976		if (!rdev)
5977			continue;
5978		if (conf->disks[i].replacement &&
5979		    conf->reshape_progress != MaxSector) {
5980			/* replacements and reshape simply do not mix. */
5981			printk(KERN_ERR "md: cannot handle concurrent "
5982			       "replacement and reshape.\n");
5983			goto abort;
5984		}
5985		if (test_bit(In_sync, &rdev->flags)) {
5986			working_disks++;
5987			continue;
5988		}
5989		/* This disc is not fully in-sync.  However if it
5990		 * just stored parity (beyond the recovery_offset),
5991		 * when we don't need to be concerned about the
5992		 * array being dirty.
5993		 * When reshape goes 'backwards', we never have
5994		 * partially completed devices, so we only need
5995		 * to worry about reshape going forwards.
5996		 */
5997		/* Hack because v0.91 doesn't store recovery_offset properly. */
5998		if (mddev->major_version == 0 &&
5999		    mddev->minor_version > 90)
6000			rdev->recovery_offset = reshape_offset;
6001
6002		if (rdev->recovery_offset < reshape_offset) {
6003			/* We need to check old and new layout */
6004			if (!only_parity(rdev->raid_disk,
6005					 conf->algorithm,
6006					 conf->raid_disks,
6007					 conf->max_degraded))
6008				continue;
6009		}
6010		if (!only_parity(rdev->raid_disk,
6011				 conf->prev_algo,
6012				 conf->previous_raid_disks,
6013				 conf->max_degraded))
6014			continue;
6015		dirty_parity_disks++;
6016	}
6017
6018	/*
6019	 * 0 for a fully functional array, 1 or 2 for a degraded array.
6020	 */
6021	mddev->degraded = calc_degraded(conf);
6022
6023	if (has_failed(conf)) {
6024		printk(KERN_ERR "md/raid:%s: not enough operational devices"
6025			" (%d/%d failed)\n",
6026			mdname(mddev), mddev->degraded, conf->raid_disks);
6027		goto abort;
6028	}
6029
6030	/* device size must be a multiple of chunk size */
6031	mddev->dev_sectors &= ~(mddev->chunk_sectors - 1);
6032	mddev->resync_max_sectors = mddev->dev_sectors;
6033
6034	if (mddev->degraded > dirty_parity_disks &&
6035	    mddev->recovery_cp != MaxSector) {
6036		if (mddev->ok_start_degraded)
6037			printk(KERN_WARNING
6038			       "md/raid:%s: starting dirty degraded array"
6039			       " - data corruption possible.\n",
6040			       mdname(mddev));
 
6041		else {
6042			printk(KERN_ERR
6043			       "md/raid:%s: cannot start dirty degraded array.\n",
6044			       mdname(mddev));
6045			goto abort;
6046		}
6047	}
6048
6049	if (mddev->degraded == 0)
6050		printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d"
6051		       " devices, algorithm %d\n", mdname(mddev), conf->level,
6052		       mddev->raid_disks-mddev->degraded, mddev->raid_disks,
6053		       mddev->new_layout);
6054	else
6055		printk(KERN_ALERT "md/raid:%s: raid level %d active with %d"
6056		       " out of %d devices, algorithm %d\n",
6057		       mdname(mddev), conf->level,
6058		       mddev->raid_disks - mddev->degraded,
6059		       mddev->raid_disks, mddev->new_layout);
6060
6061	print_raid5_conf(conf);
6062
6063	if (conf->reshape_progress != MaxSector) {
6064		conf->reshape_safe = conf->reshape_progress;
6065		atomic_set(&conf->reshape_stripes, 0);
6066		clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
6067		clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
6068		set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
6069		set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
6070		mddev->sync_thread = md_register_thread(md_do_sync, mddev,
6071							"reshape");
 
 
6072	}
6073
6074
6075	/* Ok, everything is just fine now */
6076	if (mddev->to_remove == &raid5_attrs_group)
6077		mddev->to_remove = NULL;
6078	else if (mddev->kobj.sd &&
6079	    sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
6080		printk(KERN_WARNING
6081		       "raid5: failed to create sysfs attributes for %s\n",
6082		       mdname(mddev));
6083	md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
6084
6085	if (mddev->queue) {
6086		int chunk_size;
6087		bool discard_supported = true;
6088		/* read-ahead size must cover two whole stripes, which
6089		 * is 2 * (datadisks) * chunksize where 'n' is the
6090		 * number of raid devices
6091		 */
6092		int data_disks = conf->previous_raid_disks - conf->max_degraded;
6093		int stripe = data_disks *
6094			((mddev->chunk_sectors << 9) / PAGE_SIZE);
6095		if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
6096			mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
6097
6098		blk_queue_merge_bvec(mddev->queue, raid5_mergeable_bvec);
6099
6100		mddev->queue->backing_dev_info.congested_data = mddev;
6101		mddev->queue->backing_dev_info.congested_fn = raid5_congested;
6102
6103		chunk_size = mddev->chunk_sectors << 9;
6104		blk_queue_io_min(mddev->queue, chunk_size);
6105		blk_queue_io_opt(mddev->queue, chunk_size *
6106				 (conf->raid_disks - conf->max_degraded));
6107		mddev->queue->limits.raid_partial_stripes_expensive = 1;
6108		/*
6109		 * We can only discard a whole stripe. It doesn't make sense to
6110		 * discard data disk but write parity disk
6111		 */
6112		stripe = stripe * PAGE_SIZE;
6113		/* Round up to power of 2, as discard handling
6114		 * currently assumes that */
6115		while ((stripe-1) & stripe)
6116			stripe = (stripe | (stripe-1)) + 1;
6117		mddev->queue->limits.discard_alignment = stripe;
6118		mddev->queue->limits.discard_granularity = stripe;
6119		/*
6120		 * unaligned part of discard request will be ignored, so can't
6121		 * guarantee discard_zerors_data
6122		 */
6123		mddev->queue->limits.discard_zeroes_data = 0;
6124
6125		blk_queue_max_write_same_sectors(mddev->queue, 0);
 
6126
6127		rdev_for_each(rdev, mddev) {
6128			disk_stack_limits(mddev->gendisk, rdev->bdev,
6129					  rdev->data_offset << 9);
6130			disk_stack_limits(mddev->gendisk, rdev->bdev,
6131					  rdev->new_data_offset << 9);
6132			/*
6133			 * discard_zeroes_data is required, otherwise data
6134			 * could be lost. Consider a scenario: discard a stripe
6135			 * (the stripe could be inconsistent if
6136			 * discard_zeroes_data is 0); write one disk of the
6137			 * stripe (the stripe could be inconsistent again
6138			 * depending on which disks are used to calculate
6139			 * parity); the disk is broken; The stripe data of this
6140			 * disk is lost.
6141			 */
6142			if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) ||
6143			    !bdev_get_queue(rdev->bdev)->
6144						limits.discard_zeroes_data)
6145				discard_supported = false;
6146		}
6147
6148		if (discard_supported &&
6149		   mddev->queue->limits.max_discard_sectors >= stripe &&
6150		   mddev->queue->limits.discard_granularity >= stripe)
6151			queue_flag_set_unlocked(QUEUE_FLAG_DISCARD,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6152						mddev->queue);
6153		else
6154			queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
6155						mddev->queue);
 
 
6156	}
6157
 
 
 
6158	return 0;
6159abort:
6160	md_unregister_thread(&mddev->thread);
6161	print_raid5_conf(conf);
6162	free_conf(conf);
6163	mddev->private = NULL;
6164	printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev));
6165	return -EIO;
6166}
6167
6168static int stop(struct mddev *mddev)
6169{
6170	struct r5conf *conf = mddev->private;
6171
6172	md_unregister_thread(&mddev->thread);
6173	if (mddev->queue)
6174		mddev->queue->backing_dev_info.congested_fn = NULL;
6175	free_conf(conf);
6176	mddev->private = NULL;
6177	mddev->to_remove = &raid5_attrs_group;
6178	return 0;
6179}
6180
6181static void status(struct seq_file *seq, struct mddev *mddev)
6182{
6183	struct r5conf *conf = mddev->private;
6184	int i;
6185
6186	seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
6187		mddev->chunk_sectors / 2, mddev->layout);
6188	seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
6189	for (i = 0; i < conf->raid_disks; i++)
6190		seq_printf (seq, "%s",
6191			       conf->disks[i].rdev &&
6192			       test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_");
 
 
6193	seq_printf (seq, "]");
6194}
6195
6196static void print_raid5_conf (struct r5conf *conf)
6197{
6198	int i;
6199	struct disk_info *tmp;
6200
6201	printk(KERN_DEBUG "RAID conf printout:\n");
6202	if (!conf) {
6203		printk("(conf==NULL)\n");
6204		return;
6205	}
6206	printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level,
6207	       conf->raid_disks,
6208	       conf->raid_disks - conf->mddev->degraded);
6209
6210	for (i = 0; i < conf->raid_disks; i++) {
6211		char b[BDEVNAME_SIZE];
6212		tmp = conf->disks + i;
6213		if (tmp->rdev)
6214			printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n",
6215			       i, !test_bit(Faulty, &tmp->rdev->flags),
6216			       bdevname(tmp->rdev->bdev, b));
6217	}
6218}
6219
6220static int raid5_spare_active(struct mddev *mddev)
6221{
6222	int i;
6223	struct r5conf *conf = mddev->private;
6224	struct disk_info *tmp;
6225	int count = 0;
6226	unsigned long flags;
6227
6228	for (i = 0; i < conf->raid_disks; i++) {
6229		tmp = conf->disks + i;
6230		if (tmp->replacement
6231		    && tmp->replacement->recovery_offset == MaxSector
6232		    && !test_bit(Faulty, &tmp->replacement->flags)
6233		    && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
6234			/* Replacement has just become active. */
6235			if (!tmp->rdev
6236			    || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
6237				count++;
6238			if (tmp->rdev) {
6239				/* Replaced device not technically faulty,
6240				 * but we need to be sure it gets removed
6241				 * and never re-added.
6242				 */
6243				set_bit(Faulty, &tmp->rdev->flags);
6244				sysfs_notify_dirent_safe(
6245					tmp->rdev->sysfs_state);
6246			}
6247			sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
6248		} else if (tmp->rdev
6249		    && tmp->rdev->recovery_offset == MaxSector
6250		    && !test_bit(Faulty, &tmp->rdev->flags)
6251		    && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
6252			count++;
6253			sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
6254		}
6255	}
6256	spin_lock_irqsave(&conf->device_lock, flags);
6257	mddev->degraded = calc_degraded(conf);
6258	spin_unlock_irqrestore(&conf->device_lock, flags);
6259	print_raid5_conf(conf);
6260	return count;
6261}
6262
6263static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
6264{
6265	struct r5conf *conf = mddev->private;
6266	int err = 0;
6267	int number = rdev->raid_disk;
6268	struct md_rdev **rdevp;
6269	struct disk_info *p = conf->disks + number;
6270
6271	print_raid5_conf(conf);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6272	if (rdev == p->rdev)
6273		rdevp = &p->rdev;
6274	else if (rdev == p->replacement)
6275		rdevp = &p->replacement;
6276	else
6277		return 0;
6278
6279	if (number >= conf->raid_disks &&
6280	    conf->reshape_progress == MaxSector)
6281		clear_bit(In_sync, &rdev->flags);
6282
6283	if (test_bit(In_sync, &rdev->flags) ||
6284	    atomic_read(&rdev->nr_pending)) {
6285		err = -EBUSY;
6286		goto abort;
6287	}
6288	/* Only remove non-faulty devices if recovery
6289	 * isn't possible.
6290	 */
6291	if (!test_bit(Faulty, &rdev->flags) &&
6292	    mddev->recovery_disabled != conf->recovery_disabled &&
6293	    !has_failed(conf) &&
6294	    (!p->replacement || p->replacement == rdev) &&
6295	    number < conf->raid_disks) {
6296		err = -EBUSY;
6297		goto abort;
6298	}
6299	*rdevp = NULL;
6300	synchronize_rcu();
6301	if (atomic_read(&rdev->nr_pending)) {
6302		/* lost the race, try later */
6303		err = -EBUSY;
6304		*rdevp = rdev;
6305	} else if (p->replacement) {
 
 
 
 
 
 
 
 
6306		/* We must have just cleared 'rdev' */
6307		p->rdev = p->replacement;
6308		clear_bit(Replacement, &p->replacement->flags);
6309		smp_mb(); /* Make sure other CPUs may see both as identical
6310			   * but will never see neither - if they are careful
6311			   */
6312		p->replacement = NULL;
6313		clear_bit(WantReplacement, &rdev->flags);
6314	} else
6315		/* We might have just removed the Replacement as faulty-
6316		 * clear the bit just in case
6317		 */
6318		clear_bit(WantReplacement, &rdev->flags);
6319abort:
6320
6321	print_raid5_conf(conf);
6322	return err;
6323}
6324
6325static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)
6326{
6327	struct r5conf *conf = mddev->private;
6328	int err = -EEXIST;
6329	int disk;
6330	struct disk_info *p;
6331	int first = 0;
6332	int last = conf->raid_disks - 1;
6333
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6334	if (mddev->recovery_disabled == conf->recovery_disabled)
6335		return -EBUSY;
6336
6337	if (rdev->saved_raid_disk < 0 && has_failed(conf))
6338		/* no point adding a device */
6339		return -EINVAL;
6340
6341	if (rdev->raid_disk >= 0)
6342		first = last = rdev->raid_disk;
6343
6344	/*
6345	 * find the disk ... but prefer rdev->saved_raid_disk
6346	 * if possible.
6347	 */
6348	if (rdev->saved_raid_disk >= 0 &&
6349	    rdev->saved_raid_disk >= first &&
6350	    conf->disks[rdev->saved_raid_disk].rdev == NULL)
6351		first = rdev->saved_raid_disk;
6352
6353	for (disk = first; disk <= last; disk++) {
6354		p = conf->disks + disk;
6355		if (p->rdev == NULL) {
6356			clear_bit(In_sync, &rdev->flags);
6357			rdev->raid_disk = disk;
6358			err = 0;
6359			if (rdev->saved_raid_disk != disk)
6360				conf->fullsync = 1;
6361			rcu_assign_pointer(p->rdev, rdev);
 
 
 
6362			goto out;
6363		}
6364	}
6365	for (disk = first; disk <= last; disk++) {
6366		p = conf->disks + disk;
6367		if (test_bit(WantReplacement, &p->rdev->flags) &&
6368		    p->replacement == NULL) {
6369			clear_bit(In_sync, &rdev->flags);
6370			set_bit(Replacement, &rdev->flags);
6371			rdev->raid_disk = disk;
6372			err = 0;
6373			conf->fullsync = 1;
6374			rcu_assign_pointer(p->replacement, rdev);
6375			break;
6376		}
6377	}
6378out:
6379	print_raid5_conf(conf);
6380	return err;
6381}
6382
6383static int raid5_resize(struct mddev *mddev, sector_t sectors)
6384{
6385	/* no resync is happening, and there is enough space
6386	 * on all devices, so we can resize.
6387	 * We need to make sure resync covers any new space.
6388	 * If the array is shrinking we should possibly wait until
6389	 * any io in the removed space completes, but it hardly seems
6390	 * worth it.
6391	 */
6392	sector_t newsize;
6393	sectors &= ~((sector_t)mddev->chunk_sectors - 1);
 
 
 
 
6394	newsize = raid5_size(mddev, sectors, mddev->raid_disks);
6395	if (mddev->external_size &&
6396	    mddev->array_sectors > newsize)
6397		return -EINVAL;
6398	if (mddev->bitmap) {
6399		int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0);
6400		if (ret)
6401			return ret;
6402	}
6403	md_set_array_sectors(mddev, newsize);
6404	set_capacity(mddev->gendisk, mddev->array_sectors);
6405	revalidate_disk(mddev->gendisk);
6406	if (sectors > mddev->dev_sectors &&
6407	    mddev->recovery_cp > mddev->dev_sectors) {
6408		mddev->recovery_cp = mddev->dev_sectors;
6409		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
6410	}
6411	mddev->dev_sectors = sectors;
6412	mddev->resync_max_sectors = sectors;
6413	return 0;
6414}
6415
6416static int check_stripe_cache(struct mddev *mddev)
6417{
6418	/* Can only proceed if there are plenty of stripe_heads.
6419	 * We need a minimum of one full stripe,, and for sensible progress
6420	 * it is best to have about 4 times that.
6421	 * If we require 4 times, then the default 256 4K stripe_heads will
6422	 * allow for chunk sizes up to 256K, which is probably OK.
6423	 * If the chunk size is greater, user-space should request more
6424	 * stripe_heads first.
6425	 */
6426	struct r5conf *conf = mddev->private;
6427	if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
6428	    > conf->max_nr_stripes ||
6429	    ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
6430	    > conf->max_nr_stripes) {
6431		printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes.  Needed %lu\n",
6432		       mdname(mddev),
6433		       ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
6434			/ STRIPE_SIZE)*4);
6435		return 0;
6436	}
6437	return 1;
6438}
6439
6440static int check_reshape(struct mddev *mddev)
6441{
6442	struct r5conf *conf = mddev->private;
6443
 
 
6444	if (mddev->delta_disks == 0 &&
6445	    mddev->new_layout == mddev->layout &&
6446	    mddev->new_chunk_sectors == mddev->chunk_sectors)
6447		return 0; /* nothing to do */
6448	if (has_failed(conf))
6449		return -EINVAL;
6450	if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) {
6451		/* We might be able to shrink, but the devices must
6452		 * be made bigger first.
6453		 * For raid6, 4 is the minimum size.
6454		 * Otherwise 2 is the minimum
6455		 */
6456		int min = 2;
6457		if (mddev->level == 6)
6458			min = 4;
6459		if (mddev->raid_disks + mddev->delta_disks < min)
6460			return -EINVAL;
6461	}
6462
6463	if (!check_stripe_cache(mddev))
6464		return -ENOSPC;
6465
 
 
 
 
 
 
 
 
 
 
 
 
6466	return resize_stripes(conf, (conf->previous_raid_disks
6467				     + mddev->delta_disks));
6468}
6469
6470static int raid5_start_reshape(struct mddev *mddev)
6471{
6472	struct r5conf *conf = mddev->private;
6473	struct md_rdev *rdev;
6474	int spares = 0;
6475	unsigned long flags;
6476
6477	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
6478		return -EBUSY;
6479
6480	if (!check_stripe_cache(mddev))
6481		return -ENOSPC;
6482
6483	if (has_failed(conf))
6484		return -EINVAL;
6485
6486	rdev_for_each(rdev, mddev) {
6487		if (!test_bit(In_sync, &rdev->flags)
6488		    && !test_bit(Faulty, &rdev->flags))
6489			spares++;
6490	}
6491
6492	if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
6493		/* Not enough devices even to make a degraded array
6494		 * of that size
6495		 */
6496		return -EINVAL;
6497
6498	/* Refuse to reduce size of the array.  Any reductions in
6499	 * array size must be through explicit setting of array_size
6500	 * attribute.
6501	 */
6502	if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
6503	    < mddev->array_sectors) {
6504		printk(KERN_ERR "md/raid:%s: array size must be reduced "
6505		       "before number of disks\n", mdname(mddev));
6506		return -EINVAL;
6507	}
6508
6509	atomic_set(&conf->reshape_stripes, 0);
6510	spin_lock_irq(&conf->device_lock);
6511	write_seqcount_begin(&conf->gen_lock);
6512	conf->previous_raid_disks = conf->raid_disks;
6513	conf->raid_disks += mddev->delta_disks;
6514	conf->prev_chunk_sectors = conf->chunk_sectors;
6515	conf->chunk_sectors = mddev->new_chunk_sectors;
6516	conf->prev_algo = conf->algorithm;
6517	conf->algorithm = mddev->new_layout;
6518	conf->generation++;
6519	/* Code that selects data_offset needs to see the generation update
6520	 * if reshape_progress has been set - so a memory barrier needed.
6521	 */
6522	smp_mb();
6523	if (mddev->reshape_backwards)
6524		conf->reshape_progress = raid5_size(mddev, 0, 0);
6525	else
6526		conf->reshape_progress = 0;
6527	conf->reshape_safe = conf->reshape_progress;
6528	write_seqcount_end(&conf->gen_lock);
6529	spin_unlock_irq(&conf->device_lock);
6530
6531	/* Now make sure any requests that proceeded on the assumption
6532	 * the reshape wasn't running - like Discard or Read - have
6533	 * completed.
6534	 */
6535	mddev_suspend(mddev);
6536	mddev_resume(mddev);
6537
6538	/* Add some new drives, as many as will fit.
6539	 * We know there are enough to make the newly sized array work.
6540	 * Don't add devices if we are reducing the number of
6541	 * devices in the array.  This is because it is not possible
6542	 * to correctly record the "partially reconstructed" state of
6543	 * such devices during the reshape and confusion could result.
6544	 */
6545	if (mddev->delta_disks >= 0) {
6546		rdev_for_each(rdev, mddev)
6547			if (rdev->raid_disk < 0 &&
6548			    !test_bit(Faulty, &rdev->flags)) {
6549				if (raid5_add_disk(mddev, rdev) == 0) {
6550					if (rdev->raid_disk
6551					    >= conf->previous_raid_disks)
6552						set_bit(In_sync, &rdev->flags);
6553					else
6554						rdev->recovery_offset = 0;
6555
6556					if (sysfs_link_rdev(mddev, rdev))
6557						/* Failure here is OK */;
6558				}
6559			} else if (rdev->raid_disk >= conf->previous_raid_disks
6560				   && !test_bit(Faulty, &rdev->flags)) {
6561				/* This is a spare that was manually added */
6562				set_bit(In_sync, &rdev->flags);
6563			}
6564
6565		/* When a reshape changes the number of devices,
6566		 * ->degraded is measured against the larger of the
6567		 * pre and post number of devices.
6568		 */
6569		spin_lock_irqsave(&conf->device_lock, flags);
6570		mddev->degraded = calc_degraded(conf);
6571		spin_unlock_irqrestore(&conf->device_lock, flags);
6572	}
6573	mddev->raid_disks = conf->raid_disks;
6574	mddev->reshape_position = conf->reshape_progress;
6575	set_bit(MD_CHANGE_DEVS, &mddev->flags);
6576
6577	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
6578	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
 
6579	set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
6580	set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
6581	mddev->sync_thread = md_register_thread(md_do_sync, mddev,
6582						"reshape");
6583	if (!mddev->sync_thread) {
6584		mddev->recovery = 0;
6585		spin_lock_irq(&conf->device_lock);
6586		write_seqcount_begin(&conf->gen_lock);
6587		mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
6588		mddev->new_chunk_sectors =
6589			conf->chunk_sectors = conf->prev_chunk_sectors;
6590		mddev->new_layout = conf->algorithm = conf->prev_algo;
6591		rdev_for_each(rdev, mddev)
6592			rdev->new_data_offset = rdev->data_offset;
6593		smp_wmb();
6594		conf->generation --;
6595		conf->reshape_progress = MaxSector;
6596		mddev->reshape_position = MaxSector;
6597		write_seqcount_end(&conf->gen_lock);
6598		spin_unlock_irq(&conf->device_lock);
6599		return -EAGAIN;
6600	}
6601	conf->reshape_checkpoint = jiffies;
6602	md_wakeup_thread(mddev->sync_thread);
6603	md_new_event(mddev);
6604	return 0;
6605}
6606
6607/* This is called from the reshape thread and should make any
6608 * changes needed in 'conf'
6609 */
6610static void end_reshape(struct r5conf *conf)
6611{
6612
6613	if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
6614		struct md_rdev *rdev;
6615
6616		spin_lock_irq(&conf->device_lock);
6617		conf->previous_raid_disks = conf->raid_disks;
6618		rdev_for_each(rdev, conf->mddev)
6619			rdev->data_offset = rdev->new_data_offset;
6620		smp_wmb();
6621		conf->reshape_progress = MaxSector;
 
 
 
 
 
 
6622		spin_unlock_irq(&conf->device_lock);
6623		wake_up(&conf->wait_for_overlap);
6624
6625		/* read-ahead size must cover two whole stripes, which is
6626		 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
6627		 */
6628		if (conf->mddev->queue) {
6629			int data_disks = conf->raid_disks - conf->max_degraded;
6630			int stripe = data_disks * ((conf->chunk_sectors << 9)
6631						   / PAGE_SIZE);
6632			if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
6633				conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
6634		}
6635	}
6636}
6637
6638/* This is called from the raid5d thread with mddev_lock held.
6639 * It makes config changes to the device.
6640 */
6641static void raid5_finish_reshape(struct mddev *mddev)
6642{
6643	struct r5conf *conf = mddev->private;
6644
6645	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
6646
6647		if (mddev->delta_disks > 0) {
6648			md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
6649			set_capacity(mddev->gendisk, mddev->array_sectors);
6650			revalidate_disk(mddev->gendisk);
6651		} else {
6652			int d;
6653			spin_lock_irq(&conf->device_lock);
6654			mddev->degraded = calc_degraded(conf);
6655			spin_unlock_irq(&conf->device_lock);
6656			for (d = conf->raid_disks ;
6657			     d < conf->raid_disks - mddev->delta_disks;
6658			     d++) {
6659				struct md_rdev *rdev = conf->disks[d].rdev;
6660				if (rdev)
6661					clear_bit(In_sync, &rdev->flags);
6662				rdev = conf->disks[d].replacement;
6663				if (rdev)
6664					clear_bit(In_sync, &rdev->flags);
6665			}
6666		}
6667		mddev->layout = conf->algorithm;
6668		mddev->chunk_sectors = conf->chunk_sectors;
6669		mddev->reshape_position = MaxSector;
6670		mddev->delta_disks = 0;
6671		mddev->reshape_backwards = 0;
6672	}
6673}
6674
6675static void raid5_quiesce(struct mddev *mddev, int state)
6676{
6677	struct r5conf *conf = mddev->private;
6678
6679	switch(state) {
6680	case 2: /* resume for a suspend */
6681		wake_up(&conf->wait_for_overlap);
6682		break;
6683
6684	case 1: /* stop all writes */
6685		lock_all_device_hash_locks_irq(conf);
6686		/* '2' tells resync/reshape to pause so that all
6687		 * active stripes can drain
6688		 */
 
6689		conf->quiesce = 2;
6690		wait_event_cmd(conf->wait_for_stripe,
6691				    atomic_read(&conf->active_stripes) == 0 &&
6692				    atomic_read(&conf->active_aligned_reads) == 0,
6693				    unlock_all_device_hash_locks_irq(conf),
6694				    lock_all_device_hash_locks_irq(conf));
6695		conf->quiesce = 1;
6696		unlock_all_device_hash_locks_irq(conf);
6697		/* allow reshape to continue */
6698		wake_up(&conf->wait_for_overlap);
6699		break;
6700
6701	case 0: /* re-enable writes */
6702		lock_all_device_hash_locks_irq(conf);
6703		conf->quiesce = 0;
6704		wake_up(&conf->wait_for_stripe);
6705		wake_up(&conf->wait_for_overlap);
6706		unlock_all_device_hash_locks_irq(conf);
6707		break;
6708	}
 
6709}
6710
6711
6712static void *raid45_takeover_raid0(struct mddev *mddev, int level)
6713{
6714	struct r0conf *raid0_conf = mddev->private;
6715	sector_t sectors;
6716
6717	/* for raid0 takeover only one zone is supported */
6718	if (raid0_conf->nr_strip_zones > 1) {
6719		printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n",
6720		       mdname(mddev));
6721		return ERR_PTR(-EINVAL);
6722	}
6723
6724	sectors = raid0_conf->strip_zone[0].zone_end;
6725	sector_div(sectors, raid0_conf->strip_zone[0].nb_dev);
6726	mddev->dev_sectors = sectors;
6727	mddev->new_level = level;
6728	mddev->new_layout = ALGORITHM_PARITY_N;
6729	mddev->new_chunk_sectors = mddev->chunk_sectors;
6730	mddev->raid_disks += 1;
6731	mddev->delta_disks = 1;
6732	/* make sure it will be not marked as dirty */
6733	mddev->recovery_cp = MaxSector;
6734
6735	return setup_conf(mddev);
6736}
6737
6738
6739static void *raid5_takeover_raid1(struct mddev *mddev)
6740{
6741	int chunksect;
 
6742
6743	if (mddev->raid_disks != 2 ||
6744	    mddev->degraded > 1)
6745		return ERR_PTR(-EINVAL);
6746
6747	/* Should check if there are write-behind devices? */
6748
6749	chunksect = 64*2; /* 64K by default */
6750
6751	/* The array must be an exact multiple of chunksize */
6752	while (chunksect && (mddev->array_sectors & (chunksect-1)))
6753		chunksect >>= 1;
6754
6755	if ((chunksect<<9) < STRIPE_SIZE)
6756		/* array size does not allow a suitable chunk size */
6757		return ERR_PTR(-EINVAL);
6758
6759	mddev->new_level = 5;
6760	mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
6761	mddev->new_chunk_sectors = chunksect;
6762
6763	return setup_conf(mddev);
 
 
 
 
6764}
6765
6766static void *raid5_takeover_raid6(struct mddev *mddev)
6767{
6768	int new_layout;
6769
6770	switch (mddev->layout) {
6771	case ALGORITHM_LEFT_ASYMMETRIC_6:
6772		new_layout = ALGORITHM_LEFT_ASYMMETRIC;
6773		break;
6774	case ALGORITHM_RIGHT_ASYMMETRIC_6:
6775		new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
6776		break;
6777	case ALGORITHM_LEFT_SYMMETRIC_6:
6778		new_layout = ALGORITHM_LEFT_SYMMETRIC;
6779		break;
6780	case ALGORITHM_RIGHT_SYMMETRIC_6:
6781		new_layout = ALGORITHM_RIGHT_SYMMETRIC;
6782		break;
6783	case ALGORITHM_PARITY_0_6:
6784		new_layout = ALGORITHM_PARITY_0;
6785		break;
6786	case ALGORITHM_PARITY_N:
6787		new_layout = ALGORITHM_PARITY_N;
6788		break;
6789	default:
6790		return ERR_PTR(-EINVAL);
6791	}
6792	mddev->new_level = 5;
6793	mddev->new_layout = new_layout;
6794	mddev->delta_disks = -1;
6795	mddev->raid_disks -= 1;
6796	return setup_conf(mddev);
6797}
6798
6799
6800static int raid5_check_reshape(struct mddev *mddev)
6801{
6802	/* For a 2-drive array, the layout and chunk size can be changed
6803	 * immediately as not restriping is needed.
6804	 * For larger arrays we record the new value - after validation
6805	 * to be used by a reshape pass.
6806	 */
6807	struct r5conf *conf = mddev->private;
6808	int new_chunk = mddev->new_chunk_sectors;
6809
6810	if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))
6811		return -EINVAL;
6812	if (new_chunk > 0) {
6813		if (!is_power_of_2(new_chunk))
6814			return -EINVAL;
6815		if (new_chunk < (PAGE_SIZE>>9))
6816			return -EINVAL;
6817		if (mddev->array_sectors & (new_chunk-1))
6818			/* not factor of array size */
6819			return -EINVAL;
6820	}
6821
6822	/* They look valid */
6823
6824	if (mddev->raid_disks == 2) {
6825		/* can make the change immediately */
6826		if (mddev->new_layout >= 0) {
6827			conf->algorithm = mddev->new_layout;
6828			mddev->layout = mddev->new_layout;
6829		}
6830		if (new_chunk > 0) {
6831			conf->chunk_sectors = new_chunk ;
6832			mddev->chunk_sectors = new_chunk;
6833		}
6834		set_bit(MD_CHANGE_DEVS, &mddev->flags);
6835		md_wakeup_thread(mddev->thread);
6836	}
6837	return check_reshape(mddev);
6838}
6839
6840static int raid6_check_reshape(struct mddev *mddev)
6841{
6842	int new_chunk = mddev->new_chunk_sectors;
6843
6844	if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))
6845		return -EINVAL;
6846	if (new_chunk > 0) {
6847		if (!is_power_of_2(new_chunk))
6848			return -EINVAL;
6849		if (new_chunk < (PAGE_SIZE >> 9))
6850			return -EINVAL;
6851		if (mddev->array_sectors & (new_chunk-1))
6852			/* not factor of array size */
6853			return -EINVAL;
6854	}
6855
6856	/* They look valid */
6857	return check_reshape(mddev);
6858}
6859
6860static void *raid5_takeover(struct mddev *mddev)
6861{
6862	/* raid5 can take over:
6863	 *  raid0 - if there is only one strip zone - make it a raid4 layout
6864	 *  raid1 - if there are two drives.  We need to know the chunk size
6865	 *  raid4 - trivial - just use a raid4 layout.
6866	 *  raid6 - Providing it is a *_6 layout
6867	 */
6868	if (mddev->level == 0)
6869		return raid45_takeover_raid0(mddev, 5);
6870	if (mddev->level == 1)
6871		return raid5_takeover_raid1(mddev);
6872	if (mddev->level == 4) {
6873		mddev->new_layout = ALGORITHM_PARITY_N;
6874		mddev->new_level = 5;
6875		return setup_conf(mddev);
6876	}
6877	if (mddev->level == 6)
6878		return raid5_takeover_raid6(mddev);
6879
6880	return ERR_PTR(-EINVAL);
6881}
6882
6883static void *raid4_takeover(struct mddev *mddev)
6884{
6885	/* raid4 can take over:
6886	 *  raid0 - if there is only one strip zone
6887	 *  raid5 - if layout is right
6888	 */
6889	if (mddev->level == 0)
6890		return raid45_takeover_raid0(mddev, 4);
6891	if (mddev->level == 5 &&
6892	    mddev->layout == ALGORITHM_PARITY_N) {
6893		mddev->new_layout = 0;
6894		mddev->new_level = 4;
6895		return setup_conf(mddev);
6896	}
6897	return ERR_PTR(-EINVAL);
6898}
6899
6900static struct md_personality raid5_personality;
6901
6902static void *raid6_takeover(struct mddev *mddev)
6903{
6904	/* Currently can only take over a raid5.  We map the
6905	 * personality to an equivalent raid6 personality
6906	 * with the Q block at the end.
6907	 */
6908	int new_layout;
6909
6910	if (mddev->pers != &raid5_personality)
6911		return ERR_PTR(-EINVAL);
6912	if (mddev->degraded > 1)
6913		return ERR_PTR(-EINVAL);
6914	if (mddev->raid_disks > 253)
6915		return ERR_PTR(-EINVAL);
6916	if (mddev->raid_disks < 3)
6917		return ERR_PTR(-EINVAL);
6918
6919	switch (mddev->layout) {
6920	case ALGORITHM_LEFT_ASYMMETRIC:
6921		new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;
6922		break;
6923	case ALGORITHM_RIGHT_ASYMMETRIC:
6924		new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
6925		break;
6926	case ALGORITHM_LEFT_SYMMETRIC:
6927		new_layout = ALGORITHM_LEFT_SYMMETRIC_6;
6928		break;
6929	case ALGORITHM_RIGHT_SYMMETRIC:
6930		new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;
6931		break;
6932	case ALGORITHM_PARITY_0:
6933		new_layout = ALGORITHM_PARITY_0_6;
6934		break;
6935	case ALGORITHM_PARITY_N:
6936		new_layout = ALGORITHM_PARITY_N;
6937		break;
6938	default:
6939		return ERR_PTR(-EINVAL);
6940	}
6941	mddev->new_level = 6;
6942	mddev->new_layout = new_layout;
6943	mddev->delta_disks = 1;
6944	mddev->raid_disks += 1;
6945	return setup_conf(mddev);
6946}
6947
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6948
6949static struct md_personality raid6_personality =
6950{
6951	.name		= "raid6",
6952	.level		= 6,
6953	.owner		= THIS_MODULE,
6954	.make_request	= make_request,
6955	.run		= run,
6956	.stop		= stop,
6957	.status		= status,
6958	.error_handler	= error,
 
6959	.hot_add_disk	= raid5_add_disk,
6960	.hot_remove_disk= raid5_remove_disk,
6961	.spare_active	= raid5_spare_active,
6962	.sync_request	= sync_request,
6963	.resize		= raid5_resize,
6964	.size		= raid5_size,
6965	.check_reshape	= raid6_check_reshape,
6966	.start_reshape  = raid5_start_reshape,
6967	.finish_reshape = raid5_finish_reshape,
6968	.quiesce	= raid5_quiesce,
6969	.takeover	= raid6_takeover,
 
 
6970};
6971static struct md_personality raid5_personality =
6972{
6973	.name		= "raid5",
6974	.level		= 5,
6975	.owner		= THIS_MODULE,
6976	.make_request	= make_request,
6977	.run		= run,
6978	.stop		= stop,
6979	.status		= status,
6980	.error_handler	= error,
 
6981	.hot_add_disk	= raid5_add_disk,
6982	.hot_remove_disk= raid5_remove_disk,
6983	.spare_active	= raid5_spare_active,
6984	.sync_request	= sync_request,
6985	.resize		= raid5_resize,
6986	.size		= raid5_size,
6987	.check_reshape	= raid5_check_reshape,
6988	.start_reshape  = raid5_start_reshape,
6989	.finish_reshape = raid5_finish_reshape,
6990	.quiesce	= raid5_quiesce,
6991	.takeover	= raid5_takeover,
 
 
6992};
6993
6994static struct md_personality raid4_personality =
6995{
6996	.name		= "raid4",
6997	.level		= 4,
6998	.owner		= THIS_MODULE,
6999	.make_request	= make_request,
7000	.run		= run,
7001	.stop		= stop,
7002	.status		= status,
7003	.error_handler	= error,
 
7004	.hot_add_disk	= raid5_add_disk,
7005	.hot_remove_disk= raid5_remove_disk,
7006	.spare_active	= raid5_spare_active,
7007	.sync_request	= sync_request,
7008	.resize		= raid5_resize,
7009	.size		= raid5_size,
7010	.check_reshape	= raid5_check_reshape,
7011	.start_reshape  = raid5_start_reshape,
7012	.finish_reshape = raid5_finish_reshape,
7013	.quiesce	= raid5_quiesce,
7014	.takeover	= raid4_takeover,
 
 
7015};
7016
7017static int __init raid5_init(void)
7018{
 
 
7019	raid5_wq = alloc_workqueue("raid5wq",
7020		WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0);
7021	if (!raid5_wq)
7022		return -ENOMEM;
 
 
 
 
 
 
 
 
 
7023	register_md_personality(&raid6_personality);
7024	register_md_personality(&raid5_personality);
7025	register_md_personality(&raid4_personality);
7026	return 0;
7027}
7028
7029static void raid5_exit(void)
7030{
7031	unregister_md_personality(&raid6_personality);
7032	unregister_md_personality(&raid5_personality);
7033	unregister_md_personality(&raid4_personality);
 
7034	destroy_workqueue(raid5_wq);
7035}
7036
7037module_init(raid5_init);
7038module_exit(raid5_exit);
7039MODULE_LICENSE("GPL");
7040MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD");
7041MODULE_ALIAS("md-personality-4"); /* RAID5 */
7042MODULE_ALIAS("md-raid5");
7043MODULE_ALIAS("md-raid4");
7044MODULE_ALIAS("md-level-5");
7045MODULE_ALIAS("md-level-4");
7046MODULE_ALIAS("md-personality-8"); /* RAID6 */
7047MODULE_ALIAS("md-raid6");
7048MODULE_ALIAS("md-level-6");
7049
7050/* This used to be two separate modules, they were: */
7051MODULE_ALIAS("raid5");
7052MODULE_ALIAS("raid6");