1/*
   2 * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
   3 *
   4 * bitmap_create  - sets up the bitmap structure
   5 * bitmap_destroy - destroys the bitmap structure
   6 *
   7 * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
   8 * - added disk storage for bitmap
   9 * - changes to allow various bitmap chunk sizes
  10 */
  11
  12/*
  13 * Still to do:
  14 *
  15 * flush after percent set rather than just time based. (maybe both).
  16 */
  17
  18#include <linux/blkdev.h>
  19#include <linux/module.h>
  20#include <linux/errno.h>
  21#include <linux/slab.h>
  22#include <linux/init.h>
  23#include <linux/timer.h>
  24#include <linux/sched.h>
  25#include <linux/list.h>
  26#include <linux/file.h>
  27#include <linux/mount.h>
  28#include <linux/buffer_head.h>
  29#include <linux/seq_file.h>
  30#include "md.h"
  31#include "bitmap.h"
  32
  33static inline char *bmname(struct bitmap *bitmap)
  34{
  35	return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
  36}
  37
  38/*
  39 * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
  40 *
  41 * 1) check to see if this page is allocated, if it's not then try to alloc
  42 * 2) if the alloc fails, set the page's hijacked flag so we'll use the
  43 *    page pointer directly as a counter
  44 *
  45 * if we find our page, we increment the page's refcount so that it stays
  46 * allocated while we're using it
  47 */
  48static int bitmap_checkpage(struct bitmap_counts *bitmap,
  49			    unsigned long page, int create)
  50__releases(bitmap->lock)
  51__acquires(bitmap->lock)
  52{
  53	unsigned char *mappage;
  54
  55	if (page >= bitmap->pages) {
  56		/* This can happen if bitmap_start_sync goes beyond
  57		 * End-of-device while looking for a whole page.
  58		 * It is harmless.
  59		 */
  60		return -EINVAL;
  61	}
  62
  63	if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
  64		return 0;
  65
  66	if (bitmap->bp[page].map) /* page is already allocated, just return */
  67		return 0;
  68
  69	if (!create)
  70		return -ENOENT;
  71
  72	/* this page has not been allocated yet */
  73
  74	spin_unlock_irq(&bitmap->lock);
  75	/* It is possible that this is being called inside a
  76	 * prepare_to_wait/finish_wait loop from raid5c:make_request().
  77	 * In general it is not permitted to sleep in that context as it
  78	 * can cause the loop to spin freely.
  79	 * That doesn't apply here as we can only reach this point
  80	 * once with any loop.
  81	 * When this function completes, either bp[page].map or
  82	 * bp[page].hijacked.  In either case, this function will
  83	 * abort before getting to this point again.  So there is
  84	 * no risk of a free-spin, and so it is safe to assert
  85	 * that sleeping here is allowed.
  86	 */
  87	sched_annotate_sleep();
  88	mappage = kzalloc(PAGE_SIZE, GFP_NOIO);
  89	spin_lock_irq(&bitmap->lock);
  90
  91	if (mappage == NULL) {
  92		pr_debug("md/bitmap: map page allocation failed, hijacking\n");
  93		/* failed - set the hijacked flag so that we can use the
  94		 * pointer as a counter */
  95		if (!bitmap->bp[page].map)
  96			bitmap->bp[page].hijacked = 1;
  97	} else if (bitmap->bp[page].map ||
  98		   bitmap->bp[page].hijacked) {
  99		/* somebody beat us to getting the page */
 100		kfree(mappage);
 
 101	} else {
 102
 103		/* no page was in place and we have one, so install it */
 104
 105		bitmap->bp[page].map = mappage;
 106		bitmap->missing_pages--;
 107	}
 108	return 0;
 109}
 110
 111/* if page is completely empty, put it back on the free list, or dealloc it */
 112/* if page was hijacked, unmark the flag so it might get alloced next time */
 113/* Note: lock should be held when calling this */
 114static void bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
 115{
 116	char *ptr;
 117
 118	if (bitmap->bp[page].count) /* page is still busy */
 119		return;
 120
 121	/* page is no longer in use, it can be released */
 122
 123	if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
 124		bitmap->bp[page].hijacked = 0;
 125		bitmap->bp[page].map = NULL;
 126	} else {
 127		/* normal case, free the page */
 128		ptr = bitmap->bp[page].map;
 129		bitmap->bp[page].map = NULL;
 130		bitmap->missing_pages++;
 131		kfree(ptr);
 132	}
 133}
 134
 135/*
 136 * bitmap file handling - read and write the bitmap file and its superblock
 137 */
 138
 139/*
 140 * basic page I/O operations
 141 */
 142
 143/* IO operations when bitmap is stored near all superblocks */
 144static int read_sb_page(struct mddev *mddev, loff_t offset,
 145			struct page *page,
 146			unsigned long index, int size)
 147{
 148	/* choose a good rdev and read the page from there */
 149
 150	struct md_rdev *rdev;
 151	sector_t target;
 152
 153	rdev_for_each(rdev, mddev) {
 154		if (! test_bit(In_sync, &rdev->flags)
 155		    || test_bit(Faulty, &rdev->flags))
 156			continue;
 157
 158		target = offset + index * (PAGE_SIZE/512);
 159
 160		if (sync_page_io(rdev, target,
 161				 roundup(size, bdev_logical_block_size(rdev->bdev)),
 162				 page, READ, true)) {
 163			page->index = index;
 164			return 0;
 165		}
 166	}
 167	return -EIO;
 168}
 169
 170static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev)
 171{
 172	/* Iterate the disks of an mddev, using rcu to protect access to the
 173	 * linked list, and raising the refcount of devices we return to ensure
 174	 * they don't disappear while in use.
 175	 * As devices are only added or removed when raid_disk is < 0 and
 176	 * nr_pending is 0 and In_sync is clear, the entries we return will
 177	 * still be in the same position on the list when we re-enter
 178	 * list_for_each_entry_continue_rcu.
 179	 *
 180	 * Note that if entered with 'rdev == NULL' to start at the
 181	 * beginning, we temporarily assign 'rdev' to an address which
 182	 * isn't really an rdev, but which can be used by
 183	 * list_for_each_entry_continue_rcu() to find the first entry.
 184	 */
 
 185	rcu_read_lock();
 186	if (rdev == NULL)
 187		/* start at the beginning */
 188		rdev = list_entry(&mddev->disks, struct md_rdev, same_set);
 189	else {
 190		/* release the previous rdev and start from there. */
 191		rdev_dec_pending(rdev, mddev);
 
 192	}
 193	list_for_each_entry_continue_rcu(rdev, &mddev->disks, same_set) {
 
 194		if (rdev->raid_disk >= 0 &&
 195		    !test_bit(Faulty, &rdev->flags)) {
 196			/* this is a usable devices */
 197			atomic_inc(&rdev->nr_pending);
 198			rcu_read_unlock();
 199			return rdev;
 200		}
 201	}
 202	rcu_read_unlock();
 203	return NULL;
 204}
 205
 206static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
 207{
 208	struct md_rdev *rdev = NULL;
 209	struct block_device *bdev;
 210	struct mddev *mddev = bitmap->mddev;
 211	struct bitmap_storage *store = &bitmap->storage;
 212
 213	while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
 214		int size = PAGE_SIZE;
 215		loff_t offset = mddev->bitmap_info.offset;
 216
 217		bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
 218
 219		if (page->index == store->file_pages-1) {
 220			int last_page_size = store->bytes & (PAGE_SIZE-1);
 221			if (last_page_size == 0)
 222				last_page_size = PAGE_SIZE;
 223			size = roundup(last_page_size,
 224				       bdev_logical_block_size(bdev));
 225		}
 226		/* Just make sure we aren't corrupting data or
 227		 * metadata
 228		 */
 229		if (mddev->external) {
 230			/* Bitmap could be anywhere. */
 231			if (rdev->sb_start + offset + (page->index
 232						       * (PAGE_SIZE/512))
 233			    > rdev->data_offset
 234			    &&
 235			    rdev->sb_start + offset
 236			    < (rdev->data_offset + mddev->dev_sectors
 237			     + (PAGE_SIZE/512)))
 238				goto bad_alignment;
 239		} else if (offset < 0) {
 240			/* DATA  BITMAP METADATA  */
 241			if (offset
 242			    + (long)(page->index * (PAGE_SIZE/512))
 243			    + size/512 > 0)
 244				/* bitmap runs in to metadata */
 245				goto bad_alignment;
 246			if (rdev->data_offset + mddev->dev_sectors
 247			    > rdev->sb_start + offset)
 248				/* data runs in to bitmap */
 249				goto bad_alignment;
 250		} else if (rdev->sb_start < rdev->data_offset) {
 251			/* METADATA BITMAP DATA */
 252			if (rdev->sb_start
 253			    + offset
 254			    + page->index*(PAGE_SIZE/512) + size/512
 255			    > rdev->data_offset)
 256				/* bitmap runs in to data */
 257				goto bad_alignment;
 258		} else {
 259			/* DATA METADATA BITMAP - no problems */
 260		}
 261		md_super_write(mddev, rdev,
 262			       rdev->sb_start + offset
 263			       + page->index * (PAGE_SIZE/512),
 264			       size,
 265			       page);
 266	}
 267
 268	if (wait)
 269		md_super_wait(mddev);
 270	return 0;
 271
 272 bad_alignment:
 273	return -EINVAL;
 274}
 275
 276static void bitmap_file_kick(struct bitmap *bitmap);
 277/*
 278 * write out a page to a file
 279 */
 280static void write_page(struct bitmap *bitmap, struct page *page, int wait)
 281{
 282	struct buffer_head *bh;
 283
 284	if (bitmap->storage.file == NULL) {
 285		switch (write_sb_page(bitmap, page, wait)) {
 286		case -EINVAL:
 287			set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
 288		}
 289	} else {
 290
 291		bh = page_buffers(page);
 292
 293		while (bh && bh->b_blocknr) {
 294			atomic_inc(&bitmap->pending_writes);
 295			set_buffer_locked(bh);
 296			set_buffer_mapped(bh);
 297			submit_bh(WRITE | REQ_SYNC, bh);
 298			bh = bh->b_this_page;
 299		}
 300
 301		if (wait)
 302			wait_event(bitmap->write_wait,
 303				   atomic_read(&bitmap->pending_writes)==0);
 304	}
 305	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
 306		bitmap_file_kick(bitmap);
 307}
 308
 309static void end_bitmap_write(struct buffer_head *bh, int uptodate)
 310{
 311	struct bitmap *bitmap = bh->b_private;
 312
 313	if (!uptodate)
 314		set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
 315	if (atomic_dec_and_test(&bitmap->pending_writes))
 316		wake_up(&bitmap->write_wait);
 317}
 318
 319/* copied from buffer.c */
 320static void
 321__clear_page_buffers(struct page *page)
 322{
 323	ClearPagePrivate(page);
 324	set_page_private(page, 0);
 325	put_page(page);
 326}
 327static void free_buffers(struct page *page)
 328{
 329	struct buffer_head *bh;
 330
 331	if (!PagePrivate(page))
 332		return;
 333
 334	bh = page_buffers(page);
 335	while (bh) {
 336		struct buffer_head *next = bh->b_this_page;
 337		free_buffer_head(bh);
 338		bh = next;
 339	}
 340	__clear_page_buffers(page);
 341	put_page(page);
 342}
 343
 344/* read a page from a file.
 345 * We both read the page, and attach buffers to the page to record the
 346 * address of each block (using bmap).  These addresses will be used
 347 * to write the block later, completely bypassing the filesystem.
 348 * This usage is similar to how swap files are handled, and allows us
 349 * to write to a file with no concerns of memory allocation failing.
 350 */
 351static int read_page(struct file *file, unsigned long index,
 352		     struct bitmap *bitmap,
 353		     unsigned long count,
 354		     struct page *page)
 355{
 356	int ret = 0;
 357	struct inode *inode = file_inode(file);
 358	struct buffer_head *bh;
 359	sector_t block;
 360
 361	pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
 362		 (unsigned long long)index << PAGE_SHIFT);
 363
 364	bh = alloc_page_buffers(page, 1<<inode->i_blkbits, 0);
 365	if (!bh) {
 366		ret = -ENOMEM;
 367		goto out;
 368	}
 369	attach_page_buffers(page, bh);
 370	block = index << (PAGE_SHIFT - inode->i_blkbits);
 371	while (bh) {
 372		if (count == 0)
 373			bh->b_blocknr = 0;
 374		else {
 375			bh->b_blocknr = bmap(inode, block);
 376			if (bh->b_blocknr == 0) {
 377				/* Cannot use this file! */
 378				ret = -EINVAL;
 379				goto out;
 380			}
 381			bh->b_bdev = inode->i_sb->s_bdev;
 382			if (count < (1<<inode->i_blkbits))
 383				count = 0;
 384			else
 385				count -= (1<<inode->i_blkbits);
 386
 387			bh->b_end_io = end_bitmap_write;
 388			bh->b_private = bitmap;
 389			atomic_inc(&bitmap->pending_writes);
 390			set_buffer_locked(bh);
 391			set_buffer_mapped(bh);
 392			submit_bh(READ, bh);
 393		}
 394		block++;
 395		bh = bh->b_this_page;
 396	}
 397	page->index = index;
 398
 399	wait_event(bitmap->write_wait,
 400		   atomic_read(&bitmap->pending_writes)==0);
 401	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
 402		ret = -EIO;
 403out:
 404	if (ret)
 405		printk(KERN_ALERT "md: bitmap read error: (%dB @ %llu): %d\n",
 406			(int)PAGE_SIZE,
 407			(unsigned long long)index << PAGE_SHIFT,
 408			ret);
 409	return ret;
 410}
 411
 412/*
 413 * bitmap file superblock operations
 414 */
 415
 416/* update the event counter and sync the superblock to disk */
 417void bitmap_update_sb(struct bitmap *bitmap)
 418{
 419	bitmap_super_t *sb;
 420
 421	if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
 422		return;
 423	if (bitmap->mddev->bitmap_info.external)
 424		return;
 425	if (!bitmap->storage.sb_page) /* no superblock */
 426		return;
 427	sb = kmap_atomic(bitmap->storage.sb_page);
 428	sb->events = cpu_to_le64(bitmap->mddev->events);
 429	if (bitmap->mddev->events < bitmap->events_cleared)
 430		/* rocking back to read-only */
 431		bitmap->events_cleared = bitmap->mddev->events;
 432	sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
 433	sb->state = cpu_to_le32(bitmap->flags);
 434	/* Just in case these have been changed via sysfs: */
 435	sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
 436	sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
 437	/* This might have been changed by a reshape */
 438	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
 439	sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
 440	sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes);
 441	sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
 442					   bitmap_info.space);
 443	kunmap_atomic(sb);
 444	write_page(bitmap, bitmap->storage.sb_page, 1);
 445}
 446
 447/* print out the bitmap file superblock */
 448void bitmap_print_sb(struct bitmap *bitmap)
 449{
 450	bitmap_super_t *sb;
 451
 452	if (!bitmap || !bitmap->storage.sb_page)
 453		return;
 454	sb = kmap_atomic(bitmap->storage.sb_page);
 455	printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap));
 456	printk(KERN_DEBUG "         magic: %08x\n", le32_to_cpu(sb->magic));
 457	printk(KERN_DEBUG "       version: %d\n", le32_to_cpu(sb->version));
 458	printk(KERN_DEBUG "          uuid: %08x.%08x.%08x.%08x\n",
 459					*(__u32 *)(sb->uuid+0),
 460					*(__u32 *)(sb->uuid+4),
 461					*(__u32 *)(sb->uuid+8),
 462					*(__u32 *)(sb->uuid+12));
 463	printk(KERN_DEBUG "        events: %llu\n",
 464			(unsigned long long) le64_to_cpu(sb->events));
 465	printk(KERN_DEBUG "events cleared: %llu\n",
 466			(unsigned long long) le64_to_cpu(sb->events_cleared));
 467	printk(KERN_DEBUG "         state: %08x\n", le32_to_cpu(sb->state));
 468	printk(KERN_DEBUG "     chunksize: %d B\n", le32_to_cpu(sb->chunksize));
 469	printk(KERN_DEBUG "  daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
 470	printk(KERN_DEBUG "     sync size: %llu KB\n",
 471			(unsigned long long)le64_to_cpu(sb->sync_size)/2);
 472	printk(KERN_DEBUG "max write behind: %d\n", le32_to_cpu(sb->write_behind));
 473	kunmap_atomic(sb);
 474}
 475
 476/*
 477 * bitmap_new_disk_sb
 478 * @bitmap
 479 *
 480 * This function is somewhat the reverse of bitmap_read_sb.  bitmap_read_sb
 481 * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
 482 * This function verifies 'bitmap_info' and populates the on-disk bitmap
 483 * structure, which is to be written to disk.
 484 *
 485 * Returns: 0 on success, -Exxx on error
 486 */
 487static int bitmap_new_disk_sb(struct bitmap *bitmap)
 488{
 489	bitmap_super_t *sb;
 490	unsigned long chunksize, daemon_sleep, write_behind;
 
 491
 492	bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 493	if (bitmap->storage.sb_page == NULL)
 494		return -ENOMEM;
 
 
 
 495	bitmap->storage.sb_page->index = 0;
 496
 497	sb = kmap_atomic(bitmap->storage.sb_page);
 498
 499	sb->magic = cpu_to_le32(BITMAP_MAGIC);
 500	sb->version = cpu_to_le32(BITMAP_MAJOR_HI);
 501
 502	chunksize = bitmap->mddev->bitmap_info.chunksize;
 503	BUG_ON(!chunksize);
 504	if (!is_power_of_2(chunksize)) {
 505		kunmap_atomic(sb);
 506		printk(KERN_ERR "bitmap chunksize not a power of 2\n");
 507		return -EINVAL;
 508	}
 509	sb->chunksize = cpu_to_le32(chunksize);
 510
 511	daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
 512	if (!daemon_sleep || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
 
 513		printk(KERN_INFO "Choosing daemon_sleep default (5 sec)\n");
 514		daemon_sleep = 5 * HZ;
 515	}
 516	sb->daemon_sleep = cpu_to_le32(daemon_sleep);
 517	bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
 518
 519	/*
 520	 * FIXME: write_behind for RAID1.  If not specified, what
 521	 * is a good choice?  We choose COUNTER_MAX / 2 arbitrarily.
 522	 */
 523	write_behind = bitmap->mddev->bitmap_info.max_write_behind;
 524	if (write_behind > COUNTER_MAX)
 525		write_behind = COUNTER_MAX / 2;
 526	sb->write_behind = cpu_to_le32(write_behind);
 527	bitmap->mddev->bitmap_info.max_write_behind = write_behind;
 528
 529	/* keep the array size field of the bitmap superblock up to date */
 530	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
 531
 532	memcpy(sb->uuid, bitmap->mddev->uuid, 16);
 533
 534	set_bit(BITMAP_STALE, &bitmap->flags);
 535	sb->state = cpu_to_le32(bitmap->flags);
 536	bitmap->events_cleared = bitmap->mddev->events;
 537	sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
 538	bitmap->mddev->bitmap_info.nodes = 0;
 539
 540	kunmap_atomic(sb);
 541
 542	return 0;
 543}
 544
 545/* read the superblock from the bitmap file and initialize some bitmap fields */
 546static int bitmap_read_sb(struct bitmap *bitmap)
 547{
 548	char *reason = NULL;
 549	bitmap_super_t *sb;
 550	unsigned long chunksize, daemon_sleep, write_behind;
 551	unsigned long long events;
 552	int nodes = 0;
 553	unsigned long sectors_reserved = 0;
 554	int err = -EINVAL;
 555	struct page *sb_page;
 556	loff_t offset = bitmap->mddev->bitmap_info.offset;
 557
 558	if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
 559		chunksize = 128 * 1024 * 1024;
 560		daemon_sleep = 5 * HZ;
 561		write_behind = 0;
 562		set_bit(BITMAP_STALE, &bitmap->flags);
 563		err = 0;
 564		goto out_no_sb;
 565	}
 566	/* page 0 is the superblock, read it... */
 567	sb_page = alloc_page(GFP_KERNEL);
 568	if (!sb_page)
 569		return -ENOMEM;
 570	bitmap->storage.sb_page = sb_page;
 571
 572re_read:
 573	/* If cluster_slot is set, the cluster is setup */
 574	if (bitmap->cluster_slot >= 0) {
 575		sector_t bm_blocks = bitmap->mddev->resync_max_sectors;
 576
 577		sector_div(bm_blocks,
 578			   bitmap->mddev->bitmap_info.chunksize >> 9);
 579		/* bits to bytes */
 580		bm_blocks = ((bm_blocks+7) >> 3) + sizeof(bitmap_super_t);
 581		/* to 4k blocks */
 582		bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks, 4096);
 583		offset = bitmap->mddev->bitmap_info.offset + (bitmap->cluster_slot * (bm_blocks << 3));
 584		pr_info("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
 585			bitmap->cluster_slot, offset);
 586	}
 587
 588	if (bitmap->storage.file) {
 589		loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
 590		int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
 591
 592		err = read_page(bitmap->storage.file, 0,
 593				bitmap, bytes, sb_page);
 594	} else {
 595		err = read_sb_page(bitmap->mddev,
 596				   offset,
 597				   sb_page,
 598				   0, sizeof(bitmap_super_t));
 599	}
 600	if (err)
 601		return err;
 602
 603	err = -EINVAL;
 604	sb = kmap_atomic(sb_page);
 605
 606	chunksize = le32_to_cpu(sb->chunksize);
 607	daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
 608	write_behind = le32_to_cpu(sb->write_behind);
 609	sectors_reserved = le32_to_cpu(sb->sectors_reserved);
 610	/* Setup nodes/clustername only if bitmap version is
 611	 * cluster-compatible
 612	 */
 613	if (sb->version == cpu_to_le32(BITMAP_MAJOR_CLUSTERED)) {
 614		nodes = le32_to_cpu(sb->nodes);
 615		strlcpy(bitmap->mddev->bitmap_info.cluster_name,
 616				sb->cluster_name, 64);
 617	}
 618
 619	/* verify that the bitmap-specific fields are valid */
 620	if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
 621		reason = "bad magic";
 622	else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
 623		 le32_to_cpu(sb->version) > BITMAP_MAJOR_CLUSTERED)
 624		reason = "unrecognized superblock version";
 625	else if (chunksize < 512)
 626		reason = "bitmap chunksize too small";
 627	else if (!is_power_of_2(chunksize))
 628		reason = "bitmap chunksize not a power of 2";
 629	else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT)
 630		reason = "daemon sleep period out of range";
 631	else if (write_behind > COUNTER_MAX)
 632		reason = "write-behind limit out of range (0 - 16383)";
 633	if (reason) {
 634		printk(KERN_INFO "%s: invalid bitmap file superblock: %s\n",
 635			bmname(bitmap), reason);
 636		goto out;
 637	}
 638
 639	/* keep the array size field of the bitmap superblock up to date */
 640	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
 641
 642	if (bitmap->mddev->persistent) {
 643		/*
 644		 * We have a persistent array superblock, so compare the
 645		 * bitmap's UUID and event counter to the mddev's
 646		 */
 647		if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
 648			printk(KERN_INFO
 649			       "%s: bitmap superblock UUID mismatch\n",
 650			       bmname(bitmap));
 651			goto out;
 652		}
 653		events = le64_to_cpu(sb->events);
 654		if (!nodes && (events < bitmap->mddev->events)) {
 655			printk(KERN_INFO
 656			       "%s: bitmap file is out of date (%llu < %llu) "
 657			       "-- forcing full recovery\n",
 658			       bmname(bitmap), events,
 659			       (unsigned long long) bitmap->mddev->events);
 660			set_bit(BITMAP_STALE, &bitmap->flags);
 661		}
 662	}
 663
 664	/* assign fields using values from superblock */
 665	bitmap->flags |= le32_to_cpu(sb->state);
 666	if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
 667		set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
 668	bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
 669	strlcpy(bitmap->mddev->bitmap_info.cluster_name, sb->cluster_name, 64);
 670	err = 0;
 671
 672out:
 673	kunmap_atomic(sb);
 674	/* Assiging chunksize is required for "re_read" */
 675	bitmap->mddev->bitmap_info.chunksize = chunksize;
 676	if (err == 0 && nodes && (bitmap->cluster_slot < 0)) {
 677		err = md_setup_cluster(bitmap->mddev, nodes);
 678		if (err) {
 679			pr_err("%s: Could not setup cluster service (%d)\n",
 680					bmname(bitmap), err);
 681			goto out_no_sb;
 682		}
 683		bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev);
 684		goto re_read;
 685	}
 686
 687
 688out_no_sb:
 689	if (test_bit(BITMAP_STALE, &bitmap->flags))
 690		bitmap->events_cleared = bitmap->mddev->events;
 691	bitmap->mddev->bitmap_info.chunksize = chunksize;
 692	bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
 693	bitmap->mddev->bitmap_info.max_write_behind = write_behind;
 694	bitmap->mddev->bitmap_info.nodes = nodes;
 695	if (bitmap->mddev->bitmap_info.space == 0 ||
 696	    bitmap->mddev->bitmap_info.space > sectors_reserved)
 697		bitmap->mddev->bitmap_info.space = sectors_reserved;
 698	if (err) {
 699		bitmap_print_sb(bitmap);
 700		if (bitmap->cluster_slot < 0)
 701			md_cluster_stop(bitmap->mddev);
 702	}
 703	return err;
 704}
 705
 706/*
 707 * general bitmap file operations
 708 */
 709
 710/*
 711 * on-disk bitmap:
 712 *
 713 * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
 714 * file a page at a time. There's a superblock at the start of the file.
 715 */
 716/* calculate the index of the page that contains this bit */
 717static inline unsigned long file_page_index(struct bitmap_storage *store,
 718					    unsigned long chunk)
 719{
 720	if (store->sb_page)
 721		chunk += sizeof(bitmap_super_t) << 3;
 722	return chunk >> PAGE_BIT_SHIFT;
 723}
 724
 725/* calculate the (bit) offset of this bit within a page */
 726static inline unsigned long file_page_offset(struct bitmap_storage *store,
 727					     unsigned long chunk)
 728{
 729	if (store->sb_page)
 730		chunk += sizeof(bitmap_super_t) << 3;
 731	return chunk & (PAGE_BITS - 1);
 732}
 733
 734/*
 735 * return a pointer to the page in the filemap that contains the given bit
 736 *
 
 
 
 737 */
 738static inline struct page *filemap_get_page(struct bitmap_storage *store,
 739					    unsigned long chunk)
 740{
 741	if (file_page_index(store, chunk) >= store->file_pages)
 742		return NULL;
 743	return store->filemap[file_page_index(store, chunk)];
 
 744}
 745
 746static int bitmap_storage_alloc(struct bitmap_storage *store,
 747				unsigned long chunks, int with_super,
 748				int slot_number)
 749{
 750	int pnum, offset = 0;
 751	unsigned long num_pages;
 752	unsigned long bytes;
 753
 754	bytes = DIV_ROUND_UP(chunks, 8);
 755	if (with_super)
 756		bytes += sizeof(bitmap_super_t);
 757
 758	num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
 759	offset = slot_number * (num_pages - 1);
 760
 761	store->filemap = kmalloc(sizeof(struct page *)
 762				 * num_pages, GFP_KERNEL);
 763	if (!store->filemap)
 764		return -ENOMEM;
 765
 766	if (with_super && !store->sb_page) {
 767		store->sb_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
 768		if (store->sb_page == NULL)
 769			return -ENOMEM;
 
 770	}
 771
 772	pnum = 0;
 773	if (store->sb_page) {
 774		store->filemap[0] = store->sb_page;
 775		pnum = 1;
 776		store->sb_page->index = offset;
 777	}
 778
 779	for ( ; pnum < num_pages; pnum++) {
 780		store->filemap[pnum] = alloc_page(GFP_KERNEL|__GFP_ZERO);
 781		if (!store->filemap[pnum]) {
 782			store->file_pages = pnum;
 783			return -ENOMEM;
 784		}
 785		store->filemap[pnum]->index = pnum + offset;
 786	}
 787	store->file_pages = pnum;
 788
 789	/* We need 4 bits per page, rounded up to a multiple
 790	 * of sizeof(unsigned long) */
 791	store->filemap_attr = kzalloc(
 792		roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
 793		GFP_KERNEL);
 794	if (!store->filemap_attr)
 795		return -ENOMEM;
 796
 797	store->bytes = bytes;
 798
 799	return 0;
 800}
 801
 802static void bitmap_file_unmap(struct bitmap_storage *store)
 803{
 804	struct page **map, *sb_page;
 805	int pages;
 806	struct file *file;
 807
 808	file = store->file;
 809	map = store->filemap;
 810	pages = store->file_pages;
 811	sb_page = store->sb_page;
 812
 813	while (pages--)
 814		if (map[pages] != sb_page) /* 0 is sb_page, release it below */
 815			free_buffers(map[pages]);
 816	kfree(map);
 817	kfree(store->filemap_attr);
 818
 819	if (sb_page)
 820		free_buffers(sb_page);
 821
 822	if (file) {
 823		struct inode *inode = file_inode(file);
 824		invalidate_mapping_pages(inode->i_mapping, 0, -1);
 825		fput(file);
 826	}
 827}
 828
 829/*
 830 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
 831 * then it is no longer reliable, so we stop using it and we mark the file
 832 * as failed in the superblock
 833 */
 834static void bitmap_file_kick(struct bitmap *bitmap)
 835{
 836	char *path, *ptr = NULL;
 837
 838	if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) {
 839		bitmap_update_sb(bitmap);
 840
 841		if (bitmap->storage.file) {
 842			path = kmalloc(PAGE_SIZE, GFP_KERNEL);
 843			if (path)
 844				ptr = file_path(bitmap->storage.file,
 845					     path, PAGE_SIZE);
 846
 847			printk(KERN_ALERT
 848			      "%s: kicking failed bitmap file %s from array!\n",
 849			      bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
 850
 851			kfree(path);
 852		} else
 853			printk(KERN_ALERT
 854			       "%s: disabling internal bitmap due to errors\n",
 855			       bmname(bitmap));
 856	}
 857}
 858
 859enum bitmap_page_attr {
 860	BITMAP_PAGE_DIRTY = 0,     /* there are set bits that need to be synced */
 861	BITMAP_PAGE_PENDING = 1,   /* there are bits that are being cleaned.
 862				    * i.e. counter is 1 or 2. */
 863	BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
 864};
 865
 866static inline void set_page_attr(struct bitmap *bitmap, int pnum,
 867				 enum bitmap_page_attr attr)
 868{
 869	set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
 870}
 871
 872static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
 873				   enum bitmap_page_attr attr)
 874{
 875	clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
 876}
 877
 878static inline int test_page_attr(struct bitmap *bitmap, int pnum,
 879				 enum bitmap_page_attr attr)
 880{
 881	return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
 882}
 883
 884static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
 885					   enum bitmap_page_attr attr)
 886{
 887	return test_and_clear_bit((pnum<<2) + attr,
 888				  bitmap->storage.filemap_attr);
 889}
 890/*
 891 * bitmap_file_set_bit -- called before performing a write to the md device
 892 * to set (and eventually sync) a particular bit in the bitmap file
 893 *
 894 * we set the bit immediately, then we record the page number so that
 895 * when an unplug occurs, we can flush the dirty pages out to disk
 896 */
 897static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
 898{
 899	unsigned long bit;
 900	struct page *page;
 901	void *kaddr;
 902	unsigned long chunk = block >> bitmap->counts.chunkshift;
 903
 904	page = filemap_get_page(&bitmap->storage, chunk);
 905	if (!page)
 906		return;
 907	bit = file_page_offset(&bitmap->storage, chunk);
 908
 909	/* set the bit */
 910	kaddr = kmap_atomic(page);
 911	if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
 912		set_bit(bit, kaddr);
 913	else
 914		set_bit_le(bit, kaddr);
 915	kunmap_atomic(kaddr);
 916	pr_debug("set file bit %lu page %lu\n", bit, page->index);
 917	/* record page number so it gets flushed to disk when unplug occurs */
 918	set_page_attr(bitmap, page->index, BITMAP_PAGE_DIRTY);
 919}
 920
 921static void bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
 922{
 923	unsigned long bit;
 924	struct page *page;
 925	void *paddr;
 926	unsigned long chunk = block >> bitmap->counts.chunkshift;
 927
 928	page = filemap_get_page(&bitmap->storage, chunk);
 929	if (!page)
 930		return;
 931	bit = file_page_offset(&bitmap->storage, chunk);
 932	paddr = kmap_atomic(page);
 933	if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
 934		clear_bit(bit, paddr);
 935	else
 936		clear_bit_le(bit, paddr);
 937	kunmap_atomic(paddr);
 938	if (!test_page_attr(bitmap, page->index, BITMAP_PAGE_NEEDWRITE)) {
 939		set_page_attr(bitmap, page->index, BITMAP_PAGE_PENDING);
 940		bitmap->allclean = 0;
 941	}
 942}
 943
 944static int bitmap_file_test_bit(struct bitmap *bitmap, sector_t block)
 945{
 946	unsigned long bit;
 947	struct page *page;
 948	void *paddr;
 949	unsigned long chunk = block >> bitmap->counts.chunkshift;
 950	int set = 0;
 951
 952	page = filemap_get_page(&bitmap->storage, chunk);
 953	if (!page)
 954		return -EINVAL;
 955	bit = file_page_offset(&bitmap->storage, chunk);
 956	paddr = kmap_atomic(page);
 957	if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
 958		set = test_bit(bit, paddr);
 959	else
 960		set = test_bit_le(bit, paddr);
 961	kunmap_atomic(paddr);
 962	return set;
 963}
 964
 965
 966/* this gets called when the md device is ready to unplug its underlying
 967 * (slave) device queues -- before we let any writes go down, we need to
 968 * sync the dirty pages of the bitmap file to disk */
 969void bitmap_unplug(struct bitmap *bitmap)
 970{
 971	unsigned long i;
 972	int dirty, need_write;
 
 973
 974	if (!bitmap || !bitmap->storage.filemap ||
 975	    test_bit(BITMAP_STALE, &bitmap->flags))
 976		return;
 977
 978	/* look at each page to see if there are any set bits that need to be
 979	 * flushed out to disk */
 980	for (i = 0; i < bitmap->storage.file_pages; i++) {
 981		if (!bitmap->storage.filemap)
 982			return;
 983		dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
 984		need_write = test_and_clear_page_attr(bitmap, i,
 985						      BITMAP_PAGE_NEEDWRITE);
 986		if (dirty || need_write) {
 987			clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
 988			write_page(bitmap, bitmap->storage.filemap[i], 0);
 989		}
 
 
 
 
 
 
 
 
 
 990	}
 991	if (bitmap->storage.file)
 992		wait_event(bitmap->write_wait,
 993			   atomic_read(&bitmap->pending_writes)==0);
 994	else
 995		md_super_wait(bitmap->mddev);
 996
 997	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
 998		bitmap_file_kick(bitmap);
 999}
1000EXPORT_SYMBOL(bitmap_unplug);
1001
1002static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1003/* * bitmap_init_from_disk -- called at bitmap_create time to initialize
1004 * the in-memory bitmap from the on-disk bitmap -- also, sets up the
1005 * memory mapping of the bitmap file
1006 * Special cases:
1007 *   if there's no bitmap file, or if the bitmap file had been
1008 *   previously kicked from the array, we mark all the bits as
1009 *   1's in order to cause a full resync.
1010 *
1011 * We ignore all bits for sectors that end earlier than 'start'.
1012 * This is used when reading an out-of-date bitmap...
1013 */
1014static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
1015{
1016	unsigned long i, chunks, index, oldindex, bit, node_offset = 0;
1017	struct page *page = NULL;
1018	unsigned long bit_cnt = 0;
1019	struct file *file;
1020	unsigned long offset;
1021	int outofdate;
1022	int ret = -ENOSPC;
1023	void *paddr;
1024	struct bitmap_storage *store = &bitmap->storage;
1025
1026	chunks = bitmap->counts.chunks;
1027	file = store->file;
1028
1029	if (!file && !bitmap->mddev->bitmap_info.offset) {
1030		/* No permanent bitmap - fill with '1s'. */
1031		store->filemap = NULL;
1032		store->file_pages = 0;
1033		for (i = 0; i < chunks ; i++) {
1034			/* if the disk bit is set, set the memory bit */
1035			int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
1036				      >= start);
1037			bitmap_set_memory_bits(bitmap,
1038					       (sector_t)i << bitmap->counts.chunkshift,
1039					       needed);
1040		}
1041		return 0;
1042	}
1043
1044	outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
1045	if (outofdate)
1046		printk(KERN_INFO "%s: bitmap file is out of date, doing full "
1047			"recovery\n", bmname(bitmap));
1048
1049	if (file && i_size_read(file->f_mapping->host) < store->bytes) {
1050		printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n",
1051		       bmname(bitmap),
1052		       (unsigned long) i_size_read(file->f_mapping->host),
1053		       store->bytes);
1054		goto err;
1055	}
1056
1057	oldindex = ~0L;
1058	offset = 0;
1059	if (!bitmap->mddev->bitmap_info.external)
1060		offset = sizeof(bitmap_super_t);
1061
1062	if (mddev_is_clustered(bitmap->mddev))
1063		node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE));
1064
1065	for (i = 0; i < chunks; i++) {
1066		int b;
1067		index = file_page_index(&bitmap->storage, i);
1068		bit = file_page_offset(&bitmap->storage, i);
1069		if (index != oldindex) { /* this is a new page, read it in */
1070			int count;
1071			/* unmap the old page, we're done with it */
1072			if (index == store->file_pages-1)
1073				count = store->bytes - index * PAGE_SIZE;
1074			else
1075				count = PAGE_SIZE;
1076			page = store->filemap[index];
1077			if (file)
1078				ret = read_page(file, index, bitmap,
1079						count, page);
1080			else
1081				ret = read_sb_page(
1082					bitmap->mddev,
1083					bitmap->mddev->bitmap_info.offset,
1084					page,
1085					index + node_offset, count);
1086
1087			if (ret)
1088				goto err;
1089
1090			oldindex = index;
1091
1092			if (outofdate) {
1093				/*
1094				 * if bitmap is out of date, dirty the
1095				 * whole page and write it out
1096				 */
1097				paddr = kmap_atomic(page);
1098				memset(paddr + offset, 0xff,
1099				       PAGE_SIZE - offset);
1100				kunmap_atomic(paddr);
1101				write_page(bitmap, page, 1);
1102
1103				ret = -EIO;
1104				if (test_bit(BITMAP_WRITE_ERROR,
1105					     &bitmap->flags))
1106					goto err;
1107			}
1108		}
1109		paddr = kmap_atomic(page);
1110		if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1111			b = test_bit(bit, paddr);
1112		else
1113			b = test_bit_le(bit, paddr);
1114		kunmap_atomic(paddr);
1115		if (b) {
1116			/* if the disk bit is set, set the memory bit */
1117			int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
1118				      >= start);
1119			bitmap_set_memory_bits(bitmap,
1120					       (sector_t)i << bitmap->counts.chunkshift,
1121					       needed);
1122			bit_cnt++;
1123		}
1124		offset = 0;
1125	}
1126
1127	printk(KERN_INFO "%s: bitmap initialized from disk: "
1128	       "read %lu pages, set %lu of %lu bits\n",
1129	       bmname(bitmap), store->file_pages,
1130	       bit_cnt, chunks);
1131
1132	return 0;
1133
1134 err:
1135	printk(KERN_INFO "%s: bitmap initialisation failed: %d\n",
1136	       bmname(bitmap), ret);
1137	return ret;
1138}
1139
1140void bitmap_write_all(struct bitmap *bitmap)
1141{
1142	/* We don't actually write all bitmap blocks here,
1143	 * just flag them as needing to be written
1144	 */
1145	int i;
1146
1147	if (!bitmap || !bitmap->storage.filemap)
1148		return;
1149	if (bitmap->storage.file)
1150		/* Only one copy, so nothing needed */
1151		return;
1152
1153	for (i = 0; i < bitmap->storage.file_pages; i++)
1154		set_page_attr(bitmap, i,
1155			      BITMAP_PAGE_NEEDWRITE);
1156	bitmap->allclean = 0;
1157}
1158
1159static void bitmap_count_page(struct bitmap_counts *bitmap,
1160			      sector_t offset, int inc)
1161{
1162	sector_t chunk = offset >> bitmap->chunkshift;
1163	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1164	bitmap->bp[page].count += inc;
1165	bitmap_checkfree(bitmap, page);
1166}
1167
1168static void bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
1169{
1170	sector_t chunk = offset >> bitmap->chunkshift;
1171	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1172	struct bitmap_page *bp = &bitmap->bp[page];
1173
1174	if (!bp->pending)
1175		bp->pending = 1;
1176}
1177
1178static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
1179					    sector_t offset, sector_t *blocks,
1180					    int create);
1181
1182/*
1183 * bitmap daemon -- periodically wakes up to clean bits and flush pages
1184 *			out to disk
1185 */
1186
1187void bitmap_daemon_work(struct mddev *mddev)
1188{
1189	struct bitmap *bitmap;
1190	unsigned long j;
1191	unsigned long nextpage;
1192	sector_t blocks;
1193	struct bitmap_counts *counts;
1194
1195	/* Use a mutex to guard daemon_work against
1196	 * bitmap_destroy.
1197	 */
1198	mutex_lock(&mddev->bitmap_info.mutex);
1199	bitmap = mddev->bitmap;
1200	if (bitmap == NULL) {
1201		mutex_unlock(&mddev->bitmap_info.mutex);
1202		return;
1203	}
1204	if (time_before(jiffies, bitmap->daemon_lastrun
1205			+ mddev->bitmap_info.daemon_sleep))
1206		goto done;
1207
1208	bitmap->daemon_lastrun = jiffies;
1209	if (bitmap->allclean) {
1210		mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1211		goto done;
1212	}
1213	bitmap->allclean = 1;
1214
1215	/* Any file-page which is PENDING now needs to be written.
1216	 * So set NEEDWRITE now, then after we make any last-minute changes
1217	 * we will write it.
1218	 */
1219	for (j = 0; j < bitmap->storage.file_pages; j++)
1220		if (test_and_clear_page_attr(bitmap, j,
1221					     BITMAP_PAGE_PENDING))
1222			set_page_attr(bitmap, j,
1223				      BITMAP_PAGE_NEEDWRITE);
1224
1225	if (bitmap->need_sync &&
1226	    mddev->bitmap_info.external == 0) {
1227		/* Arrange for superblock update as well as
1228		 * other changes */
1229		bitmap_super_t *sb;
1230		bitmap->need_sync = 0;
1231		if (bitmap->storage.filemap) {
1232			sb = kmap_atomic(bitmap->storage.sb_page);
1233			sb->events_cleared =
1234				cpu_to_le64(bitmap->events_cleared);
1235			kunmap_atomic(sb);
1236			set_page_attr(bitmap, 0,
1237				      BITMAP_PAGE_NEEDWRITE);
1238		}
1239	}
1240	/* Now look at the bitmap counters and if any are '2' or '1',
1241	 * decrement and handle accordingly.
1242	 */
1243	counts = &bitmap->counts;
1244	spin_lock_irq(&counts->lock);
1245	nextpage = 0;
1246	for (j = 0; j < counts->chunks; j++) {
1247		bitmap_counter_t *bmc;
1248		sector_t  block = (sector_t)j << counts->chunkshift;
1249
1250		if (j == nextpage) {
1251			nextpage += PAGE_COUNTER_RATIO;
1252			if (!counts->bp[j >> PAGE_COUNTER_SHIFT].pending) {
1253				j |= PAGE_COUNTER_MASK;
1254				continue;
1255			}
1256			counts->bp[j >> PAGE_COUNTER_SHIFT].pending = 0;
1257		}
1258		bmc = bitmap_get_counter(counts,
1259					 block,
1260					 &blocks, 0);
1261
1262		if (!bmc) {
1263			j |= PAGE_COUNTER_MASK;
1264			continue;
1265		}
1266		if (*bmc == 1 && !bitmap->need_sync) {
1267			/* We can clear the bit */
1268			*bmc = 0;
1269			bitmap_count_page(counts, block, -1);
1270			bitmap_file_clear_bit(bitmap, block);
1271		} else if (*bmc && *bmc <= 2) {
1272			*bmc = 1;
1273			bitmap_set_pending(counts, block);
1274			bitmap->allclean = 0;
1275		}
1276	}
1277	spin_unlock_irq(&counts->lock);
1278
1279	/* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
1280	 * DIRTY pages need to be written by bitmap_unplug so it can wait
1281	 * for them.
1282	 * If we find any DIRTY page we stop there and let bitmap_unplug
1283	 * handle all the rest.  This is important in the case where
1284	 * the first blocking holds the superblock and it has been updated.
1285	 * We mustn't write any other blocks before the superblock.
1286	 */
1287	for (j = 0;
1288	     j < bitmap->storage.file_pages
1289		     && !test_bit(BITMAP_STALE, &bitmap->flags);
1290	     j++) {
 
1291		if (test_page_attr(bitmap, j,
1292				   BITMAP_PAGE_DIRTY))
1293			/* bitmap_unplug will handle the rest */
1294			break;
1295		if (test_and_clear_page_attr(bitmap, j,
1296					     BITMAP_PAGE_NEEDWRITE)) {
1297			write_page(bitmap, bitmap->storage.filemap[j], 0);
1298		}
1299	}
1300
1301 done:
1302	if (bitmap->allclean == 0)
1303		mddev->thread->timeout =
1304			mddev->bitmap_info.daemon_sleep;
1305	mutex_unlock(&mddev->bitmap_info.mutex);
1306}
1307
1308static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
1309					    sector_t offset, sector_t *blocks,
1310					    int create)
1311__releases(bitmap->lock)
1312__acquires(bitmap->lock)
1313{
1314	/* If 'create', we might release the lock and reclaim it.
1315	 * The lock must have been taken with interrupts enabled.
1316	 * If !create, we don't release the lock.
1317	 */
1318	sector_t chunk = offset >> bitmap->chunkshift;
1319	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1320	unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
1321	sector_t csize;
1322	int err;
1323
1324	err = bitmap_checkpage(bitmap, page, create);
1325
1326	if (bitmap->bp[page].hijacked ||
1327	    bitmap->bp[page].map == NULL)
1328		csize = ((sector_t)1) << (bitmap->chunkshift +
1329					  PAGE_COUNTER_SHIFT - 1);
1330	else
1331		csize = ((sector_t)1) << bitmap->chunkshift;
1332	*blocks = csize - (offset & (csize - 1));
1333
1334	if (err < 0)
1335		return NULL;
1336
1337	/* now locked ... */
1338
1339	if (bitmap->bp[page].hijacked) { /* hijacked pointer */
1340		/* should we use the first or second counter field
1341		 * of the hijacked pointer? */
1342		int hi = (pageoff > PAGE_COUNTER_MASK);
1343		return  &((bitmap_counter_t *)
1344			  &bitmap->bp[page].map)[hi];
1345	} else /* page is allocated */
1346		return (bitmap_counter_t *)
1347			&(bitmap->bp[page].map[pageoff]);
1348}
1349
1350int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
1351{
1352	if (!bitmap)
1353		return 0;
1354
1355	if (behind) {
1356		int bw;
1357		atomic_inc(&bitmap->behind_writes);
1358		bw = atomic_read(&bitmap->behind_writes);
1359		if (bw > bitmap->behind_writes_used)
1360			bitmap->behind_writes_used = bw;
1361
1362		pr_debug("inc write-behind count %d/%lu\n",
1363			 bw, bitmap->mddev->bitmap_info.max_write_behind);
1364	}
1365
1366	while (sectors) {
1367		sector_t blocks;
1368		bitmap_counter_t *bmc;
1369
1370		spin_lock_irq(&bitmap->counts.lock);
1371		bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 1);
1372		if (!bmc) {
1373			spin_unlock_irq(&bitmap->counts.lock);
1374			return 0;
1375		}
1376
1377		if (unlikely(COUNTER(*bmc) == COUNTER_MAX)) {
1378			DEFINE_WAIT(__wait);
1379			/* note that it is safe to do the prepare_to_wait
1380			 * after the test as long as we do it before dropping
1381			 * the spinlock.
1382			 */
1383			prepare_to_wait(&bitmap->overflow_wait, &__wait,
1384					TASK_UNINTERRUPTIBLE);
1385			spin_unlock_irq(&bitmap->counts.lock);
1386			schedule();
1387			finish_wait(&bitmap->overflow_wait, &__wait);
1388			continue;
1389		}
1390
1391		switch (*bmc) {
1392		case 0:
1393			bitmap_file_set_bit(bitmap, offset);
1394			bitmap_count_page(&bitmap->counts, offset, 1);
1395			/* fall through */
1396		case 1:
1397			*bmc = 2;
1398		}
1399
1400		(*bmc)++;
1401
1402		spin_unlock_irq(&bitmap->counts.lock);
1403
1404		offset += blocks;
1405		if (sectors > blocks)
1406			sectors -= blocks;
1407		else
1408			sectors = 0;
1409	}
1410	return 0;
1411}
1412EXPORT_SYMBOL(bitmap_startwrite);
1413
1414void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors,
1415		     int success, int behind)
1416{
1417	if (!bitmap)
1418		return;
1419	if (behind) {
1420		if (atomic_dec_and_test(&bitmap->behind_writes))
1421			wake_up(&bitmap->behind_wait);
1422		pr_debug("dec write-behind count %d/%lu\n",
1423			 atomic_read(&bitmap->behind_writes),
1424			 bitmap->mddev->bitmap_info.max_write_behind);
1425	}
1426
1427	while (sectors) {
1428		sector_t blocks;
1429		unsigned long flags;
1430		bitmap_counter_t *bmc;
1431
1432		spin_lock_irqsave(&bitmap->counts.lock, flags);
1433		bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 0);
1434		if (!bmc) {
1435			spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1436			return;
1437		}
1438
1439		if (success && !bitmap->mddev->degraded &&
1440		    bitmap->events_cleared < bitmap->mddev->events) {
1441			bitmap->events_cleared = bitmap->mddev->events;
1442			bitmap->need_sync = 1;
1443			sysfs_notify_dirent_safe(bitmap->sysfs_can_clear);
1444		}
1445
1446		if (!success && !NEEDED(*bmc))
1447			*bmc |= NEEDED_MASK;
1448
1449		if (COUNTER(*bmc) == COUNTER_MAX)
1450			wake_up(&bitmap->overflow_wait);
1451
1452		(*bmc)--;
1453		if (*bmc <= 2) {
1454			bitmap_set_pending(&bitmap->counts, offset);
1455			bitmap->allclean = 0;
1456		}
1457		spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1458		offset += blocks;
1459		if (sectors > blocks)
1460			sectors -= blocks;
1461		else
1462			sectors = 0;
1463	}
1464}
1465EXPORT_SYMBOL(bitmap_endwrite);
1466
1467static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1468			       int degraded)
1469{
1470	bitmap_counter_t *bmc;
1471	int rv;
1472	if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
1473		*blocks = 1024;
1474		return 1; /* always resync if no bitmap */
1475	}
1476	spin_lock_irq(&bitmap->counts.lock);
1477	bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1478	rv = 0;
1479	if (bmc) {
1480		/* locked */
1481		if (RESYNC(*bmc))
1482			rv = 1;
1483		else if (NEEDED(*bmc)) {
1484			rv = 1;
1485			if (!degraded) { /* don't set/clear bits if degraded */
1486				*bmc |= RESYNC_MASK;
1487				*bmc &= ~NEEDED_MASK;
1488			}
1489		}
1490	}
1491	spin_unlock_irq(&bitmap->counts.lock);
1492	return rv;
1493}
1494
1495int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1496		      int degraded)
1497{
1498	/* bitmap_start_sync must always report on multiples of whole
1499	 * pages, otherwise resync (which is very PAGE_SIZE based) will
1500	 * get confused.
1501	 * So call __bitmap_start_sync repeatedly (if needed) until
1502	 * At least PAGE_SIZE>>9 blocks are covered.
1503	 * Return the 'or' of the result.
1504	 */
1505	int rv = 0;
1506	sector_t blocks1;
1507
1508	*blocks = 0;
1509	while (*blocks < (PAGE_SIZE>>9)) {
1510		rv |= __bitmap_start_sync(bitmap, offset,
1511					  &blocks1, degraded);
1512		offset += blocks1;
1513		*blocks += blocks1;
1514	}
1515	return rv;
1516}
1517EXPORT_SYMBOL(bitmap_start_sync);
1518
1519void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
1520{
1521	bitmap_counter_t *bmc;
1522	unsigned long flags;
1523
1524	if (bitmap == NULL) {
1525		*blocks = 1024;
1526		return;
1527	}
1528	spin_lock_irqsave(&bitmap->counts.lock, flags);
1529	bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1530	if (bmc == NULL)
1531		goto unlock;
1532	/* locked */
1533	if (RESYNC(*bmc)) {
1534		*bmc &= ~RESYNC_MASK;
1535
1536		if (!NEEDED(*bmc) && aborted)
1537			*bmc |= NEEDED_MASK;
1538		else {
1539			if (*bmc <= 2) {
1540				bitmap_set_pending(&bitmap->counts, offset);
1541				bitmap->allclean = 0;
1542			}
1543		}
1544	}
1545 unlock:
1546	spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1547}
1548EXPORT_SYMBOL(bitmap_end_sync);
1549
1550void bitmap_close_sync(struct bitmap *bitmap)
1551{
1552	/* Sync has finished, and any bitmap chunks that weren't synced
1553	 * properly have been aborted.  It remains to us to clear the
1554	 * RESYNC bit wherever it is still on
1555	 */
1556	sector_t sector = 0;
1557	sector_t blocks;
1558	if (!bitmap)
1559		return;
1560	while (sector < bitmap->mddev->resync_max_sectors) {
1561		bitmap_end_sync(bitmap, sector, &blocks, 0);
1562		sector += blocks;
1563	}
1564}
1565EXPORT_SYMBOL(bitmap_close_sync);
1566
1567void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector, bool force)
1568{
1569	sector_t s = 0;
1570	sector_t blocks;
1571
1572	if (!bitmap)
1573		return;
1574	if (sector == 0) {
1575		bitmap->last_end_sync = jiffies;
1576		return;
1577	}
1578	if (!force && time_before(jiffies, (bitmap->last_end_sync
1579				  + bitmap->mddev->bitmap_info.daemon_sleep)))
1580		return;
1581	wait_event(bitmap->mddev->recovery_wait,
1582		   atomic_read(&bitmap->mddev->recovery_active) == 0);
1583
1584	bitmap->mddev->curr_resync_completed = sector;
1585	set_bit(MD_CHANGE_CLEAN, &bitmap->mddev->flags);
1586	sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
1587	s = 0;
1588	while (s < sector && s < bitmap->mddev->resync_max_sectors) {
1589		bitmap_end_sync(bitmap, s, &blocks, 0);
1590		s += blocks;
1591	}
1592	bitmap->last_end_sync = jiffies;
1593	sysfs_notify(&bitmap->mddev->kobj, NULL, "sync_completed");
1594}
1595EXPORT_SYMBOL(bitmap_cond_end_sync);
1596
1597static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1598{
1599	/* For each chunk covered by any of these sectors, set the
1600	 * counter to 2 and possibly set resync_needed.  They should all
1601	 * be 0 at this point
1602	 */
1603
1604	sector_t secs;
1605	bitmap_counter_t *bmc;
1606	spin_lock_irq(&bitmap->counts.lock);
1607	bmc = bitmap_get_counter(&bitmap->counts, offset, &secs, 1);
1608	if (!bmc) {
1609		spin_unlock_irq(&bitmap->counts.lock);
1610		return;
1611	}
1612	if (!*bmc) {
1613		*bmc = 2;
1614		bitmap_count_page(&bitmap->counts, offset, 1);
1615		bitmap_set_pending(&bitmap->counts, offset);
1616		bitmap->allclean = 0;
1617	}
1618	if (needed)
1619		*bmc |= NEEDED_MASK;
1620	spin_unlock_irq(&bitmap->counts.lock);
1621}
1622
1623/* dirty the memory and file bits for bitmap chunks "s" to "e" */
1624void bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
1625{
1626	unsigned long chunk;
1627
1628	for (chunk = s; chunk <= e; chunk++) {
1629		sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;
1630		bitmap_set_memory_bits(bitmap, sec, 1);
1631		bitmap_file_set_bit(bitmap, sec);
1632		if (sec < bitmap->mddev->recovery_cp)
1633			/* We are asserting that the array is dirty,
1634			 * so move the recovery_cp address back so
1635			 * that it is obvious that it is dirty
1636			 */
1637			bitmap->mddev->recovery_cp = sec;
1638	}
1639}
1640
1641/*
1642 * flush out any pending updates
1643 */
1644void bitmap_flush(struct mddev *mddev)
1645{
1646	struct bitmap *bitmap = mddev->bitmap;
1647	long sleep;
1648
1649	if (!bitmap) /* there was no bitmap */
1650		return;
1651
1652	/* run the daemon_work three time to ensure everything is flushed
1653	 * that can be
1654	 */
1655	sleep = mddev->bitmap_info.daemon_sleep * 2;
1656	bitmap->daemon_lastrun -= sleep;
1657	bitmap_daemon_work(mddev);
1658	bitmap->daemon_lastrun -= sleep;
1659	bitmap_daemon_work(mddev);
1660	bitmap->daemon_lastrun -= sleep;
1661	bitmap_daemon_work(mddev);
1662	bitmap_update_sb(bitmap);
1663}
1664
1665/*
1666 * free memory that was allocated
1667 */
1668static void bitmap_free(struct bitmap *bitmap)
1669{
1670	unsigned long k, pages;
1671	struct bitmap_page *bp;
1672
1673	if (!bitmap) /* there was no bitmap */
1674		return;
1675
1676	if (bitmap->sysfs_can_clear)
1677		sysfs_put(bitmap->sysfs_can_clear);
1678
1679	if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info &&
1680		bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev))
1681		md_cluster_stop(bitmap->mddev);
1682
1683	/* Shouldn't be needed - but just in case.... */
1684	wait_event(bitmap->write_wait,
1685		   atomic_read(&bitmap->pending_writes) == 0);
1686
1687	/* release the bitmap file  */
1688	bitmap_file_unmap(&bitmap->storage);
1689
1690	bp = bitmap->counts.bp;
1691	pages = bitmap->counts.pages;
1692
1693	/* free all allocated memory */
1694
1695	if (bp) /* deallocate the page memory */
1696		for (k = 0; k < pages; k++)
1697			if (bp[k].map && !bp[k].hijacked)
1698				kfree(bp[k].map);
1699	kfree(bp);
1700	kfree(bitmap);
1701}
1702
1703void bitmap_destroy(struct mddev *mddev)
1704{
1705	struct bitmap *bitmap = mddev->bitmap;
1706
1707	if (!bitmap) /* there was no bitmap */
1708		return;
1709
1710	mutex_lock(&mddev->bitmap_info.mutex);
1711	spin_lock(&mddev->lock);
1712	mddev->bitmap = NULL; /* disconnect from the md device */
1713	spin_unlock(&mddev->lock);
1714	mutex_unlock(&mddev->bitmap_info.mutex);
1715	if (mddev->thread)
1716		mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1717
 
 
 
1718	bitmap_free(bitmap);
1719}
1720
1721/*
1722 * initialize the bitmap structure
1723 * if this returns an error, bitmap_destroy must be called to do clean up
1724 * once mddev->bitmap is set
1725 */
1726struct bitmap *bitmap_create(struct mddev *mddev, int slot)
1727{
1728	struct bitmap *bitmap;
1729	sector_t blocks = mddev->resync_max_sectors;
1730	struct file *file = mddev->bitmap_info.file;
1731	int err;
1732	struct kernfs_node *bm = NULL;
1733
1734	BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
1735
1736	BUG_ON(file && mddev->bitmap_info.offset);
1737
1738	bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
1739	if (!bitmap)
1740		return ERR_PTR(-ENOMEM);
1741
1742	spin_lock_init(&bitmap->counts.lock);
1743	atomic_set(&bitmap->pending_writes, 0);
1744	init_waitqueue_head(&bitmap->write_wait);
1745	init_waitqueue_head(&bitmap->overflow_wait);
1746	init_waitqueue_head(&bitmap->behind_wait);
1747
1748	bitmap->mddev = mddev;
1749	bitmap->cluster_slot = slot;
1750
1751	if (mddev->kobj.sd)
1752		bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap");
1753	if (bm) {
1754		bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear");
1755		sysfs_put(bm);
1756	} else
1757		bitmap->sysfs_can_clear = NULL;
1758
1759	bitmap->storage.file = file;
1760	if (file) {
1761		get_file(file);
1762		/* As future accesses to this file will use bmap,
1763		 * and bypass the page cache, we must sync the file
1764		 * first.
1765		 */
1766		vfs_fsync(file, 1);
1767	}
1768	/* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
1769	if (!mddev->bitmap_info.external) {
1770		/*
1771		 * If 'MD_ARRAY_FIRST_USE' is set, then device-mapper is
1772		 * instructing us to create a new on-disk bitmap instance.
1773		 */
1774		if (test_and_clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags))
1775			err = bitmap_new_disk_sb(bitmap);
1776		else
1777			err = bitmap_read_sb(bitmap);
1778	} else {
1779		err = 0;
1780		if (mddev->bitmap_info.chunksize == 0 ||
1781		    mddev->bitmap_info.daemon_sleep == 0)
1782			/* chunksize and time_base need to be
1783			 * set first. */
1784			err = -EINVAL;
1785	}
1786	if (err)
1787		goto error;
1788
1789	bitmap->daemon_lastrun = jiffies;
1790	err = bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1);
1791	if (err)
1792		goto error;
1793
1794	printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
1795	       bitmap->counts.pages, bmname(bitmap));
1796
1797	err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
1798	if (err)
1799		goto error;
1800
1801	return bitmap;
1802 error:
1803	bitmap_free(bitmap);
1804	return ERR_PTR(err);
1805}
1806
1807int bitmap_load(struct mddev *mddev)
1808{
1809	int err = 0;
1810	sector_t start = 0;
1811	sector_t sector = 0;
1812	struct bitmap *bitmap = mddev->bitmap;
1813
1814	if (!bitmap)
1815		goto out;
1816
1817	/* Clear out old bitmap info first:  Either there is none, or we
1818	 * are resuming after someone else has possibly changed things,
1819	 * so we should forget old cached info.
1820	 * All chunks should be clean, but some might need_sync.
1821	 */
1822	while (sector < mddev->resync_max_sectors) {
1823		sector_t blocks;
1824		bitmap_start_sync(bitmap, sector, &blocks, 0);
1825		sector += blocks;
1826	}
1827	bitmap_close_sync(bitmap);
1828
1829	if (mddev->degraded == 0
1830	    || bitmap->events_cleared == mddev->events)
1831		/* no need to keep dirty bits to optimise a
1832		 * re-add of a missing device */
1833		start = mddev->recovery_cp;
1834
1835	mutex_lock(&mddev->bitmap_info.mutex);
1836	err = bitmap_init_from_disk(bitmap, start);
1837	mutex_unlock(&mddev->bitmap_info.mutex);
1838
1839	if (err)
1840		goto out;
1841	clear_bit(BITMAP_STALE, &bitmap->flags);
1842
1843	/* Kick recovery in case any bits were set */
1844	set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
1845
1846	mddev->thread->timeout = mddev->bitmap_info.daemon_sleep;
1847	md_wakeup_thread(mddev->thread);
1848
1849	bitmap_update_sb(bitmap);
1850
1851	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1852		err = -EIO;
1853out:
1854	return err;
1855}
1856EXPORT_SYMBOL_GPL(bitmap_load);
1857
1858/* Loads the bitmap associated with slot and copies the resync information
1859 * to our bitmap
1860 */
1861int bitmap_copy_from_slot(struct mddev *mddev, int slot,
1862		sector_t *low, sector_t *high, bool clear_bits)
1863{
1864	int rv = 0, i, j;
1865	sector_t block, lo = 0, hi = 0;
1866	struct bitmap_counts *counts;
1867	struct bitmap *bitmap = bitmap_create(mddev, slot);
1868
1869	if (IS_ERR(bitmap)) {
1870		bitmap_free(bitmap);
1871		return PTR_ERR(bitmap);
1872	}
1873
1874	rv = bitmap_init_from_disk(bitmap, 0);
1875	if (rv)
1876		goto err;
1877
1878	counts = &bitmap->counts;
1879	for (j = 0; j < counts->chunks; j++) {
1880		block = (sector_t)j << counts->chunkshift;
1881		if (bitmap_file_test_bit(bitmap, block)) {
1882			if (!lo)
1883				lo = block;
1884			hi = block;
1885			bitmap_file_clear_bit(bitmap, block);
1886			bitmap_set_memory_bits(mddev->bitmap, block, 1);
1887			bitmap_file_set_bit(mddev->bitmap, block);
1888		}
1889	}
1890
1891	if (clear_bits) {
1892		bitmap_update_sb(bitmap);
1893		/* Setting this for the ev_page should be enough.
1894		 * And we do not require both write_all and PAGE_DIRT either
1895		 */
1896		for (i = 0; i < bitmap->storage.file_pages; i++)
1897			set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
1898		bitmap_write_all(bitmap);
1899		bitmap_unplug(bitmap);
1900	}
1901	*low = lo;
1902	*high = hi;
1903err:
1904	bitmap_free(bitmap);
1905	return rv;
1906}
1907EXPORT_SYMBOL_GPL(bitmap_copy_from_slot);
1908
1909
1910void bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
1911{
1912	unsigned long chunk_kb;
1913	struct bitmap_counts *counts;
1914
1915	if (!bitmap)
1916		return;
1917
1918	counts = &bitmap->counts;
1919
1920	chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10;
1921	seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], "
1922		   "%lu%s chunk",
1923		   counts->pages - counts->missing_pages,
1924		   counts->pages,
1925		   (counts->pages - counts->missing_pages)
1926		   << (PAGE_SHIFT - 10),
1927		   chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize,
1928		   chunk_kb ? "KB" : "B");
1929	if (bitmap->storage.file) {
1930		seq_printf(seq, ", file: ");
1931		seq_file_path(seq, bitmap->storage.file, " \t\n");
1932	}
1933
1934	seq_printf(seq, "\n");
1935}
1936
1937int bitmap_resize(struct bitmap *bitmap, sector_t blocks,
1938		  int chunksize, int init)
1939{
1940	/* If chunk_size is 0, choose an appropriate chunk size.
1941	 * Then possibly allocate new storage space.
1942	 * Then quiesce, copy bits, replace bitmap, and re-start
1943	 *
1944	 * This function is called both to set up the initial bitmap
1945	 * and to resize the bitmap while the array is active.
1946	 * If this happens as a result of the array being resized,
1947	 * chunksize will be zero, and we need to choose a suitable
1948	 * chunksize, otherwise we use what we are given.
1949	 */
1950	struct bitmap_storage store;
1951	struct bitmap_counts old_counts;
1952	unsigned long chunks;
1953	sector_t block;
1954	sector_t old_blocks, new_blocks;
1955	int chunkshift;
1956	int ret = 0;
1957	long pages;
1958	struct bitmap_page *new_bp;
1959
1960	if (chunksize == 0) {
1961		/* If there is enough space, leave the chunk size unchanged,
1962		 * else increase by factor of two until there is enough space.
1963		 */
1964		long bytes;
1965		long space = bitmap->mddev->bitmap_info.space;
1966
1967		if (space == 0) {
1968			/* We don't know how much space there is, so limit
1969			 * to current size - in sectors.
1970			 */
1971			bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
1972			if (!bitmap->mddev->bitmap_info.external)
1973				bytes += sizeof(bitmap_super_t);
1974			space = DIV_ROUND_UP(bytes, 512);
1975			bitmap->mddev->bitmap_info.space = space;
1976		}
1977		chunkshift = bitmap->counts.chunkshift;
1978		chunkshift--;
1979		do {
1980			/* 'chunkshift' is shift from block size to chunk size */
1981			chunkshift++;
1982			chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
1983			bytes = DIV_ROUND_UP(chunks, 8);
1984			if (!bitmap->mddev->bitmap_info.external)
1985				bytes += sizeof(bitmap_super_t);
1986		} while (bytes > (space << 9));
1987	} else
1988		chunkshift = ffz(~chunksize) - BITMAP_BLOCK_SHIFT;
1989
1990	chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
1991	memset(&store, 0, sizeof(store));
1992	if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
1993		ret = bitmap_storage_alloc(&store, chunks,
1994					   !bitmap->mddev->bitmap_info.external,
1995					   mddev_is_clustered(bitmap->mddev)
1996					   ? bitmap->cluster_slot : 0);
1997	if (ret)
1998		goto err;
1999
2000	pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);
2001
2002	new_bp = kzalloc(pages * sizeof(*new_bp), GFP_KERNEL);
2003	ret = -ENOMEM;
2004	if (!new_bp) {
2005		bitmap_file_unmap(&store);
2006		goto err;
2007	}
2008
2009	if (!init)
2010		bitmap->mddev->pers->quiesce(bitmap->mddev, 1);
2011
2012	store.file = bitmap->storage.file;
2013	bitmap->storage.file = NULL;
2014
2015	if (store.sb_page && bitmap->storage.sb_page)
2016		memcpy(page_address(store.sb_page),
2017		       page_address(bitmap->storage.sb_page),
2018		       sizeof(bitmap_super_t));
2019	bitmap_file_unmap(&bitmap->storage);
2020	bitmap->storage = store;
2021
2022	old_counts = bitmap->counts;
2023	bitmap->counts.bp = new_bp;
2024	bitmap->counts.pages = pages;
2025	bitmap->counts.missing_pages = pages;
2026	bitmap->counts.chunkshift = chunkshift;
2027	bitmap->counts.chunks = chunks;
2028	bitmap->mddev->bitmap_info.chunksize = 1 << (chunkshift +
2029						     BITMAP_BLOCK_SHIFT);
2030
2031	blocks = min(old_counts.chunks << old_counts.chunkshift,
2032		     chunks << chunkshift);
2033
2034	spin_lock_irq(&bitmap->counts.lock);
2035	for (block = 0; block < blocks; ) {
2036		bitmap_counter_t *bmc_old, *bmc_new;
2037		int set;
2038
2039		bmc_old = bitmap_get_counter(&old_counts, block,
2040					     &old_blocks, 0);
2041		set = bmc_old && NEEDED(*bmc_old);
2042
2043		if (set) {
2044			bmc_new = bitmap_get_counter(&bitmap->counts, block,
2045						     &new_blocks, 1);
2046			if (*bmc_new == 0) {
2047				/* need to set on-disk bits too. */
2048				sector_t end = block + new_blocks;
2049				sector_t start = block >> chunkshift;
2050				start <<= chunkshift;
2051				while (start < end) {
2052					bitmap_file_set_bit(bitmap, block);
2053					start += 1 << chunkshift;
2054				}
2055				*bmc_new = 2;
2056				bitmap_count_page(&bitmap->counts,
2057						  block, 1);
2058				bitmap_set_pending(&bitmap->counts,
2059						   block);
2060			}
2061			*bmc_new |= NEEDED_MASK;
2062			if (new_blocks < old_blocks)
2063				old_blocks = new_blocks;
2064		}
2065		block += old_blocks;
2066	}
2067
2068	if (!init) {
2069		int i;
2070		while (block < (chunks << chunkshift)) {
2071			bitmap_counter_t *bmc;
2072			bmc = bitmap_get_counter(&bitmap->counts, block,
2073						 &new_blocks, 1);
2074			if (bmc) {
2075				/* new space.  It needs to be resynced, so
2076				 * we set NEEDED_MASK.
2077				 */
2078				if (*bmc == 0) {
2079					*bmc = NEEDED_MASK | 2;
2080					bitmap_count_page(&bitmap->counts,
2081							  block, 1);
2082					bitmap_set_pending(&bitmap->counts,
2083							   block);
2084				}
2085			}
2086			block += new_blocks;
2087		}
2088		for (i = 0; i < bitmap->storage.file_pages; i++)
2089			set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
2090	}
2091	spin_unlock_irq(&bitmap->counts.lock);
2092
2093	if (!init) {
2094		bitmap_unplug(bitmap);
2095		bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
2096	}
2097	ret = 0;
2098err:
2099	return ret;
2100}
2101EXPORT_SYMBOL_GPL(bitmap_resize);
2102
2103static ssize_t
2104location_show(struct mddev *mddev, char *page)
2105{
2106	ssize_t len;
2107	if (mddev->bitmap_info.file)
2108		len = sprintf(page, "file");
2109	else if (mddev->bitmap_info.offset)
2110		len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset);
2111	else
2112		len = sprintf(page, "none");
2113	len += sprintf(page+len, "\n");
2114	return len;
2115}
2116
2117static ssize_t
2118location_store(struct mddev *mddev, const char *buf, size_t len)
2119{
2120
2121	if (mddev->pers) {
2122		if (!mddev->pers->quiesce)
2123			return -EBUSY;
2124		if (mddev->recovery || mddev->sync_thread)
2125			return -EBUSY;
2126	}
2127
2128	if (mddev->bitmap || mddev->bitmap_info.file ||
2129	    mddev->bitmap_info.offset) {
2130		/* bitmap already configured.  Only option is to clear it */
2131		if (strncmp(buf, "none", 4) != 0)
2132			return -EBUSY;
2133		if (mddev->pers) {
2134			mddev->pers->quiesce(mddev, 1);
2135			bitmap_destroy(mddev);
2136			mddev->pers->quiesce(mddev, 0);
2137		}
2138		mddev->bitmap_info.offset = 0;
2139		if (mddev->bitmap_info.file) {
2140			struct file *f = mddev->bitmap_info.file;
2141			mddev->bitmap_info.file = NULL;
 
2142			fput(f);
2143		}
2144	} else {
2145		/* No bitmap, OK to set a location */
2146		long long offset;
2147		if (strncmp(buf, "none", 4) == 0)
2148			/* nothing to be done */;
2149		else if (strncmp(buf, "file:", 5) == 0) {
2150			/* Not supported yet */
2151			return -EINVAL;
2152		} else {
2153			int rv;
2154			if (buf[0] == '+')
2155				rv = kstrtoll(buf+1, 10, &offset);
2156			else
2157				rv = kstrtoll(buf, 10, &offset);
2158			if (rv)
2159				return rv;
2160			if (offset == 0)
2161				return -EINVAL;
2162			if (mddev->bitmap_info.external == 0 &&
2163			    mddev->major_version == 0 &&
2164			    offset != mddev->bitmap_info.default_offset)
2165				return -EINVAL;
2166			mddev->bitmap_info.offset = offset;
2167			if (mddev->pers) {
2168				struct bitmap *bitmap;
2169				mddev->pers->quiesce(mddev, 1);
2170				bitmap = bitmap_create(mddev, -1);
2171				if (IS_ERR(bitmap))
2172					rv = PTR_ERR(bitmap);
2173				else {
2174					mddev->bitmap = bitmap;
2175					rv = bitmap_load(mddev);
2176					if (rv)
2177						mddev->bitmap_info.offset = 0;
2178				}
2179				mddev->pers->quiesce(mddev, 0);
2180				if (rv) {
2181					bitmap_destroy(mddev);
2182					return rv;
2183				}
 
 
 
2184			}
2185		}
2186	}
2187	if (!mddev->external) {
2188		/* Ensure new bitmap info is stored in
2189		 * metadata promptly.
2190		 */
2191		set_bit(MD_CHANGE_DEVS, &mddev->flags);
2192		md_wakeup_thread(mddev->thread);
2193	}
2194	return len;
2195}
2196
2197static struct md_sysfs_entry bitmap_location =
2198__ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);
2199
2200/* 'bitmap/space' is the space available at 'location' for the
2201 * bitmap.  This allows the kernel to know when it is safe to
2202 * resize the bitmap to match a resized array.
2203 */
2204static ssize_t
2205space_show(struct mddev *mddev, char *page)
2206{
2207	return sprintf(page, "%lu\n", mddev->bitmap_info.space);
2208}
2209
2210static ssize_t
2211space_store(struct mddev *mddev, const char *buf, size_t len)
2212{
2213	unsigned long sectors;
2214	int rv;
2215
2216	rv = kstrtoul(buf, 10, &sectors);
2217	if (rv)
2218		return rv;
2219
2220	if (sectors == 0)
2221		return -EINVAL;
2222
2223	if (mddev->bitmap &&
2224	    sectors < (mddev->bitmap->storage.bytes + 511) >> 9)
2225		return -EFBIG; /* Bitmap is too big for this small space */
2226
2227	/* could make sure it isn't too big, but that isn't really
2228	 * needed - user-space should be careful.
2229	 */
2230	mddev->bitmap_info.space = sectors;
2231	return len;
2232}
2233
2234static struct md_sysfs_entry bitmap_space =
2235__ATTR(space, S_IRUGO|S_IWUSR, space_show, space_store);
2236
2237static ssize_t
2238timeout_show(struct mddev *mddev, char *page)
2239{
2240	ssize_t len;
2241	unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
2242	unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;
2243
2244	len = sprintf(page, "%lu", secs);
2245	if (jifs)
2246		len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs));
2247	len += sprintf(page+len, "\n");
2248	return len;
2249}
2250
2251static ssize_t
2252timeout_store(struct mddev *mddev, const char *buf, size_t len)
2253{
2254	/* timeout can be set at any time */
2255	unsigned long timeout;
2256	int rv = strict_strtoul_scaled(buf, &timeout, 4);
2257	if (rv)
2258		return rv;
2259
2260	/* just to make sure we don't overflow... */
2261	if (timeout >= LONG_MAX / HZ)
2262		return -EINVAL;
2263
2264	timeout = timeout * HZ / 10000;
2265
2266	if (timeout >= MAX_SCHEDULE_TIMEOUT)
2267		timeout = MAX_SCHEDULE_TIMEOUT-1;
2268	if (timeout < 1)
2269		timeout = 1;
2270	mddev->bitmap_info.daemon_sleep = timeout;
2271	if (mddev->thread) {
2272		/* if thread->timeout is MAX_SCHEDULE_TIMEOUT, then
2273		 * the bitmap is all clean and we don't need to
2274		 * adjust the timeout right now
2275		 */
2276		if (mddev->thread->timeout < MAX_SCHEDULE_TIMEOUT) {
2277			mddev->thread->timeout = timeout;
2278			md_wakeup_thread(mddev->thread);
2279		}
2280	}
2281	return len;
2282}
2283
2284static struct md_sysfs_entry bitmap_timeout =
2285__ATTR(time_base, S_IRUGO|S_IWUSR, timeout_show, timeout_store);
2286
2287static ssize_t
2288backlog_show(struct mddev *mddev, char *page)
2289{
2290	return sprintf(page, "%lu\n", mddev->bitmap_info.max_write_behind);
2291}
2292
2293static ssize_t
2294backlog_store(struct mddev *mddev, const char *buf, size_t len)
2295{
2296	unsigned long backlog;
2297	int rv = kstrtoul(buf, 10, &backlog);
2298	if (rv)
2299		return rv;
2300	if (backlog > COUNTER_MAX)
2301		return -EINVAL;
2302	mddev->bitmap_info.max_write_behind = backlog;
2303	return len;
2304}
2305
2306static struct md_sysfs_entry bitmap_backlog =
2307__ATTR(backlog, S_IRUGO|S_IWUSR, backlog_show, backlog_store);
2308
2309static ssize_t
2310chunksize_show(struct mddev *mddev, char *page)
2311{
2312	return sprintf(page, "%lu\n", mddev->bitmap_info.chunksize);
2313}
2314
2315static ssize_t
2316chunksize_store(struct mddev *mddev, const char *buf, size_t len)
2317{
2318	/* Can only be changed when no bitmap is active */
2319	int rv;
2320	unsigned long csize;
2321	if (mddev->bitmap)
2322		return -EBUSY;
2323	rv = kstrtoul(buf, 10, &csize);
2324	if (rv)
2325		return rv;
2326	if (csize < 512 ||
2327	    !is_power_of_2(csize))
2328		return -EINVAL;
2329	mddev->bitmap_info.chunksize = csize;
2330	return len;
2331}
2332
2333static struct md_sysfs_entry bitmap_chunksize =
2334__ATTR(chunksize, S_IRUGO|S_IWUSR, chunksize_show, chunksize_store);
2335
2336static ssize_t metadata_show(struct mddev *mddev, char *page)
2337{
2338	if (mddev_is_clustered(mddev))
2339		return sprintf(page, "clustered\n");
2340	return sprintf(page, "%s\n", (mddev->bitmap_info.external
2341				      ? "external" : "internal"));
2342}
2343
2344static ssize_t metadata_store(struct mddev *mddev, const char *buf, size_t len)
2345{
2346	if (mddev->bitmap ||
2347	    mddev->bitmap_info.file ||
2348	    mddev->bitmap_info.offset)
2349		return -EBUSY;
2350	if (strncmp(buf, "external", 8) == 0)
2351		mddev->bitmap_info.external = 1;
2352	else if ((strncmp(buf, "internal", 8) == 0) ||
2353			(strncmp(buf, "clustered", 9) == 0))
2354		mddev->bitmap_info.external = 0;
2355	else
2356		return -EINVAL;
2357	return len;
2358}
2359
2360static struct md_sysfs_entry bitmap_metadata =
2361__ATTR(metadata, S_IRUGO|S_IWUSR, metadata_show, metadata_store);
2362
2363static ssize_t can_clear_show(struct mddev *mddev, char *page)
2364{
2365	int len;
2366	spin_lock(&mddev->lock);
2367	if (mddev->bitmap)
2368		len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ?
2369					     "false" : "true"));
2370	else
2371		len = sprintf(page, "\n");
2372	spin_unlock(&mddev->lock);
2373	return len;
2374}
2375
2376static ssize_t can_clear_store(struct mddev *mddev, const char *buf, size_t len)
2377{
2378	if (mddev->bitmap == NULL)
2379		return -ENOENT;
2380	if (strncmp(buf, "false", 5) == 0)
2381		mddev->bitmap->need_sync = 1;
2382	else if (strncmp(buf, "true", 4) == 0) {
2383		if (mddev->degraded)
2384			return -EBUSY;
2385		mddev->bitmap->need_sync = 0;
2386	} else
2387		return -EINVAL;
2388	return len;
2389}
2390
2391static struct md_sysfs_entry bitmap_can_clear =
2392__ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);
2393
2394static ssize_t
2395behind_writes_used_show(struct mddev *mddev, char *page)
2396{
2397	ssize_t ret;
2398	spin_lock(&mddev->lock);
2399	if (mddev->bitmap == NULL)
2400		ret = sprintf(page, "0\n");
2401	else
2402		ret = sprintf(page, "%lu\n",
2403			      mddev->bitmap->behind_writes_used);
2404	spin_unlock(&mddev->lock);
2405	return ret;
2406}
2407
2408static ssize_t
2409behind_writes_used_reset(struct mddev *mddev, const char *buf, size_t len)
2410{
2411	if (mddev->bitmap)
2412		mddev->bitmap->behind_writes_used = 0;
2413	return len;
2414}
2415
2416static struct md_sysfs_entry max_backlog_used =
2417__ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
2418       behind_writes_used_show, behind_writes_used_reset);
2419
2420static struct attribute *md_bitmap_attrs[] = {
2421	&bitmap_location.attr,
2422	&bitmap_space.attr,
2423	&bitmap_timeout.attr,
2424	&bitmap_backlog.attr,
2425	&bitmap_chunksize.attr,
2426	&bitmap_metadata.attr,
2427	&bitmap_can_clear.attr,
2428	&max_backlog_used.attr,
2429	NULL
2430};
2431struct attribute_group md_bitmap_group = {
2432	.name = "bitmap",
2433	.attrs = md_bitmap_attrs,
2434};
2435