Loading...
Note: File does not exist in v6.2.
1/*
2 * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
3 *
4 * bitmap_create - sets up the bitmap structure
5 * bitmap_destroy - destroys the bitmap structure
6 *
7 * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
8 * - added disk storage for bitmap
9 * - changes to allow various bitmap chunk sizes
10 */
11
12/*
13 * Still to do:
14 *
15 * flush after percent set rather than just time based. (maybe both).
16 */
17
18#include <linux/blkdev.h>
19#include <linux/module.h>
20#include <linux/errno.h>
21#include <linux/slab.h>
22#include <linux/init.h>
23#include <linux/timer.h>
24#include <linux/sched.h>
25#include <linux/list.h>
26#include <linux/file.h>
27#include <linux/mount.h>
28#include <linux/buffer_head.h>
29#include <linux/seq_file.h>
30#include <trace/events/block.h>
31#include "md.h"
32#include "bitmap.h"
33
34static inline char *bmname(struct bitmap *bitmap)
35{
36 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
37}
38
39/*
40 * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
41 *
42 * 1) check to see if this page is allocated, if it's not then try to alloc
43 * 2) if the alloc fails, set the page's hijacked flag so we'll use the
44 * page pointer directly as a counter
45 *
46 * if we find our page, we increment the page's refcount so that it stays
47 * allocated while we're using it
48 */
49static int bitmap_checkpage(struct bitmap_counts *bitmap,
50 unsigned long page, int create, int no_hijack)
51__releases(bitmap->lock)
52__acquires(bitmap->lock)
53{
54 unsigned char *mappage;
55
56 if (page >= bitmap->pages) {
57 /* This can happen if bitmap_start_sync goes beyond
58 * End-of-device while looking for a whole page.
59 * It is harmless.
60 */
61 return -EINVAL;
62 }
63
64 if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
65 return 0;
66
67 if (bitmap->bp[page].map) /* page is already allocated, just return */
68 return 0;
69
70 if (!create)
71 return -ENOENT;
72
73 /* this page has not been allocated yet */
74
75 spin_unlock_irq(&bitmap->lock);
76 /* It is possible that this is being called inside a
77 * prepare_to_wait/finish_wait loop from raid5c:make_request().
78 * In general it is not permitted to sleep in that context as it
79 * can cause the loop to spin freely.
80 * That doesn't apply here as we can only reach this point
81 * once with any loop.
82 * When this function completes, either bp[page].map or
83 * bp[page].hijacked. In either case, this function will
84 * abort before getting to this point again. So there is
85 * no risk of a free-spin, and so it is safe to assert
86 * that sleeping here is allowed.
87 */
88 sched_annotate_sleep();
89 mappage = kzalloc(PAGE_SIZE, GFP_NOIO);
90 spin_lock_irq(&bitmap->lock);
91
92 if (mappage == NULL) {
93 pr_debug("md/bitmap: map page allocation failed, hijacking\n");
94 /* We don't support hijack for cluster raid */
95 if (no_hijack)
96 return -ENOMEM;
97 /* failed - set the hijacked flag so that we can use the
98 * pointer as a counter */
99 if (!bitmap->bp[page].map)
100 bitmap->bp[page].hijacked = 1;
101 } else if (bitmap->bp[page].map ||
102 bitmap->bp[page].hijacked) {
103 /* somebody beat us to getting the page */
104 kfree(mappage);
105 } else {
106
107 /* no page was in place and we have one, so install it */
108
109 bitmap->bp[page].map = mappage;
110 bitmap->missing_pages--;
111 }
112 return 0;
113}
114
115/* if page is completely empty, put it back on the free list, or dealloc it */
116/* if page was hijacked, unmark the flag so it might get alloced next time */
117/* Note: lock should be held when calling this */
118static void bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
119{
120 char *ptr;
121
122 if (bitmap->bp[page].count) /* page is still busy */
123 return;
124
125 /* page is no longer in use, it can be released */
126
127 if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
128 bitmap->bp[page].hijacked = 0;
129 bitmap->bp[page].map = NULL;
130 } else {
131 /* normal case, free the page */
132 ptr = bitmap->bp[page].map;
133 bitmap->bp[page].map = NULL;
134 bitmap->missing_pages++;
135 kfree(ptr);
136 }
137}
138
139/*
140 * bitmap file handling - read and write the bitmap file and its superblock
141 */
142
143/*
144 * basic page I/O operations
145 */
146
147/* IO operations when bitmap is stored near all superblocks */
148static int read_sb_page(struct mddev *mddev, loff_t offset,
149 struct page *page,
150 unsigned long index, int size)
151{
152 /* choose a good rdev and read the page from there */
153
154 struct md_rdev *rdev;
155 sector_t target;
156
157 rdev_for_each(rdev, mddev) {
158 if (! test_bit(In_sync, &rdev->flags)
159 || test_bit(Faulty, &rdev->flags))
160 continue;
161
162 target = offset + index * (PAGE_SIZE/512);
163
164 if (sync_page_io(rdev, target,
165 roundup(size, bdev_logical_block_size(rdev->bdev)),
166 page, REQ_OP_READ, 0, true)) {
167 page->index = index;
168 return 0;
169 }
170 }
171 return -EIO;
172}
173
174static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev)
175{
176 /* Iterate the disks of an mddev, using rcu to protect access to the
177 * linked list, and raising the refcount of devices we return to ensure
178 * they don't disappear while in use.
179 * As devices are only added or removed when raid_disk is < 0 and
180 * nr_pending is 0 and In_sync is clear, the entries we return will
181 * still be in the same position on the list when we re-enter
182 * list_for_each_entry_continue_rcu.
183 *
184 * Note that if entered with 'rdev == NULL' to start at the
185 * beginning, we temporarily assign 'rdev' to an address which
186 * isn't really an rdev, but which can be used by
187 * list_for_each_entry_continue_rcu() to find the first entry.
188 */
189 rcu_read_lock();
190 if (rdev == NULL)
191 /* start at the beginning */
192 rdev = list_entry(&mddev->disks, struct md_rdev, same_set);
193 else {
194 /* release the previous rdev and start from there. */
195 rdev_dec_pending(rdev, mddev);
196 }
197 list_for_each_entry_continue_rcu(rdev, &mddev->disks, same_set) {
198 if (rdev->raid_disk >= 0 &&
199 !test_bit(Faulty, &rdev->flags)) {
200 /* this is a usable devices */
201 atomic_inc(&rdev->nr_pending);
202 rcu_read_unlock();
203 return rdev;
204 }
205 }
206 rcu_read_unlock();
207 return NULL;
208}
209
210static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
211{
212 struct md_rdev *rdev;
213 struct block_device *bdev;
214 struct mddev *mddev = bitmap->mddev;
215 struct bitmap_storage *store = &bitmap->storage;
216
217restart:
218 rdev = NULL;
219 while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
220 int size = PAGE_SIZE;
221 loff_t offset = mddev->bitmap_info.offset;
222
223 bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
224
225 if (page->index == store->file_pages-1) {
226 int last_page_size = store->bytes & (PAGE_SIZE-1);
227 if (last_page_size == 0)
228 last_page_size = PAGE_SIZE;
229 size = roundup(last_page_size,
230 bdev_logical_block_size(bdev));
231 }
232 /* Just make sure we aren't corrupting data or
233 * metadata
234 */
235 if (mddev->external) {
236 /* Bitmap could be anywhere. */
237 if (rdev->sb_start + offset + (page->index
238 * (PAGE_SIZE/512))
239 > rdev->data_offset
240 &&
241 rdev->sb_start + offset
242 < (rdev->data_offset + mddev->dev_sectors
243 + (PAGE_SIZE/512)))
244 goto bad_alignment;
245 } else if (offset < 0) {
246 /* DATA BITMAP METADATA */
247 if (offset
248 + (long)(page->index * (PAGE_SIZE/512))
249 + size/512 > 0)
250 /* bitmap runs in to metadata */
251 goto bad_alignment;
252 if (rdev->data_offset + mddev->dev_sectors
253 > rdev->sb_start + offset)
254 /* data runs in to bitmap */
255 goto bad_alignment;
256 } else if (rdev->sb_start < rdev->data_offset) {
257 /* METADATA BITMAP DATA */
258 if (rdev->sb_start
259 + offset
260 + page->index*(PAGE_SIZE/512) + size/512
261 > rdev->data_offset)
262 /* bitmap runs in to data */
263 goto bad_alignment;
264 } else {
265 /* DATA METADATA BITMAP - no problems */
266 }
267 md_super_write(mddev, rdev,
268 rdev->sb_start + offset
269 + page->index * (PAGE_SIZE/512),
270 size,
271 page);
272 }
273
274 if (wait && md_super_wait(mddev) < 0)
275 goto restart;
276 return 0;
277
278 bad_alignment:
279 return -EINVAL;
280}
281
282static void bitmap_file_kick(struct bitmap *bitmap);
283/*
284 * write out a page to a file
285 */
286static void write_page(struct bitmap *bitmap, struct page *page, int wait)
287{
288 struct buffer_head *bh;
289
290 if (bitmap->storage.file == NULL) {
291 switch (write_sb_page(bitmap, page, wait)) {
292 case -EINVAL:
293 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
294 }
295 } else {
296
297 bh = page_buffers(page);
298
299 while (bh && bh->b_blocknr) {
300 atomic_inc(&bitmap->pending_writes);
301 set_buffer_locked(bh);
302 set_buffer_mapped(bh);
303 submit_bh(REQ_OP_WRITE, REQ_SYNC, bh);
304 bh = bh->b_this_page;
305 }
306
307 if (wait)
308 wait_event(bitmap->write_wait,
309 atomic_read(&bitmap->pending_writes)==0);
310 }
311 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
312 bitmap_file_kick(bitmap);
313}
314
315static void end_bitmap_write(struct buffer_head *bh, int uptodate)
316{
317 struct bitmap *bitmap = bh->b_private;
318
319 if (!uptodate)
320 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
321 if (atomic_dec_and_test(&bitmap->pending_writes))
322 wake_up(&bitmap->write_wait);
323}
324
325/* copied from buffer.c */
326static void
327__clear_page_buffers(struct page *page)
328{
329 ClearPagePrivate(page);
330 set_page_private(page, 0);
331 put_page(page);
332}
333static void free_buffers(struct page *page)
334{
335 struct buffer_head *bh;
336
337 if (!PagePrivate(page))
338 return;
339
340 bh = page_buffers(page);
341 while (bh) {
342 struct buffer_head *next = bh->b_this_page;
343 free_buffer_head(bh);
344 bh = next;
345 }
346 __clear_page_buffers(page);
347 put_page(page);
348}
349
350/* read a page from a file.
351 * We both read the page, and attach buffers to the page to record the
352 * address of each block (using bmap). These addresses will be used
353 * to write the block later, completely bypassing the filesystem.
354 * This usage is similar to how swap files are handled, and allows us
355 * to write to a file with no concerns of memory allocation failing.
356 */
357static int read_page(struct file *file, unsigned long index,
358 struct bitmap *bitmap,
359 unsigned long count,
360 struct page *page)
361{
362 int ret = 0;
363 struct inode *inode = file_inode(file);
364 struct buffer_head *bh;
365 sector_t block;
366
367 pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
368 (unsigned long long)index << PAGE_SHIFT);
369
370 bh = alloc_page_buffers(page, 1<<inode->i_blkbits, 0);
371 if (!bh) {
372 ret = -ENOMEM;
373 goto out;
374 }
375 attach_page_buffers(page, bh);
376 block = index << (PAGE_SHIFT - inode->i_blkbits);
377 while (bh) {
378 if (count == 0)
379 bh->b_blocknr = 0;
380 else {
381 bh->b_blocknr = bmap(inode, block);
382 if (bh->b_blocknr == 0) {
383 /* Cannot use this file! */
384 ret = -EINVAL;
385 goto out;
386 }
387 bh->b_bdev = inode->i_sb->s_bdev;
388 if (count < (1<<inode->i_blkbits))
389 count = 0;
390 else
391 count -= (1<<inode->i_blkbits);
392
393 bh->b_end_io = end_bitmap_write;
394 bh->b_private = bitmap;
395 atomic_inc(&bitmap->pending_writes);
396 set_buffer_locked(bh);
397 set_buffer_mapped(bh);
398 submit_bh(REQ_OP_READ, 0, bh);
399 }
400 block++;
401 bh = bh->b_this_page;
402 }
403 page->index = index;
404
405 wait_event(bitmap->write_wait,
406 atomic_read(&bitmap->pending_writes)==0);
407 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
408 ret = -EIO;
409out:
410 if (ret)
411 pr_err("md: bitmap read error: (%dB @ %llu): %d\n",
412 (int)PAGE_SIZE,
413 (unsigned long long)index << PAGE_SHIFT,
414 ret);
415 return ret;
416}
417
418/*
419 * bitmap file superblock operations
420 */
421
422/*
423 * bitmap_wait_writes() should be called before writing any bitmap
424 * blocks, to ensure previous writes, particularly from
425 * bitmap_daemon_work(), have completed.
426 */
427static void bitmap_wait_writes(struct bitmap *bitmap)
428{
429 if (bitmap->storage.file)
430 wait_event(bitmap->write_wait,
431 atomic_read(&bitmap->pending_writes)==0);
432 else
433 /* Note that we ignore the return value. The writes
434 * might have failed, but that would just mean that
435 * some bits which should be cleared haven't been,
436 * which is safe. The relevant bitmap blocks will
437 * probably get written again, but there is no great
438 * loss if they aren't.
439 */
440 md_super_wait(bitmap->mddev);
441}
442
443
444/* update the event counter and sync the superblock to disk */
445void bitmap_update_sb(struct bitmap *bitmap)
446{
447 bitmap_super_t *sb;
448
449 if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
450 return;
451 if (bitmap->mddev->bitmap_info.external)
452 return;
453 if (!bitmap->storage.sb_page) /* no superblock */
454 return;
455 sb = kmap_atomic(bitmap->storage.sb_page);
456 sb->events = cpu_to_le64(bitmap->mddev->events);
457 if (bitmap->mddev->events < bitmap->events_cleared)
458 /* rocking back to read-only */
459 bitmap->events_cleared = bitmap->mddev->events;
460 sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
461 sb->state = cpu_to_le32(bitmap->flags);
462 /* Just in case these have been changed via sysfs: */
463 sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
464 sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
465 /* This might have been changed by a reshape */
466 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
467 sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
468 sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes);
469 sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
470 bitmap_info.space);
471 kunmap_atomic(sb);
472 write_page(bitmap, bitmap->storage.sb_page, 1);
473}
474
475/* print out the bitmap file superblock */
476void bitmap_print_sb(struct bitmap *bitmap)
477{
478 bitmap_super_t *sb;
479
480 if (!bitmap || !bitmap->storage.sb_page)
481 return;
482 sb = kmap_atomic(bitmap->storage.sb_page);
483 pr_debug("%s: bitmap file superblock:\n", bmname(bitmap));
484 pr_debug(" magic: %08x\n", le32_to_cpu(sb->magic));
485 pr_debug(" version: %d\n", le32_to_cpu(sb->version));
486 pr_debug(" uuid: %08x.%08x.%08x.%08x\n",
487 *(__u32 *)(sb->uuid+0),
488 *(__u32 *)(sb->uuid+4),
489 *(__u32 *)(sb->uuid+8),
490 *(__u32 *)(sb->uuid+12));
491 pr_debug(" events: %llu\n",
492 (unsigned long long) le64_to_cpu(sb->events));
493 pr_debug("events cleared: %llu\n",
494 (unsigned long long) le64_to_cpu(sb->events_cleared));
495 pr_debug(" state: %08x\n", le32_to_cpu(sb->state));
496 pr_debug(" chunksize: %d B\n", le32_to_cpu(sb->chunksize));
497 pr_debug(" daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
498 pr_debug(" sync size: %llu KB\n",
499 (unsigned long long)le64_to_cpu(sb->sync_size)/2);
500 pr_debug("max write behind: %d\n", le32_to_cpu(sb->write_behind));
501 kunmap_atomic(sb);
502}
503
504/*
505 * bitmap_new_disk_sb
506 * @bitmap
507 *
508 * This function is somewhat the reverse of bitmap_read_sb. bitmap_read_sb
509 * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
510 * This function verifies 'bitmap_info' and populates the on-disk bitmap
511 * structure, which is to be written to disk.
512 *
513 * Returns: 0 on success, -Exxx on error
514 */
515static int bitmap_new_disk_sb(struct bitmap *bitmap)
516{
517 bitmap_super_t *sb;
518 unsigned long chunksize, daemon_sleep, write_behind;
519
520 bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
521 if (bitmap->storage.sb_page == NULL)
522 return -ENOMEM;
523 bitmap->storage.sb_page->index = 0;
524
525 sb = kmap_atomic(bitmap->storage.sb_page);
526
527 sb->magic = cpu_to_le32(BITMAP_MAGIC);
528 sb->version = cpu_to_le32(BITMAP_MAJOR_HI);
529
530 chunksize = bitmap->mddev->bitmap_info.chunksize;
531 BUG_ON(!chunksize);
532 if (!is_power_of_2(chunksize)) {
533 kunmap_atomic(sb);
534 pr_warn("bitmap chunksize not a power of 2\n");
535 return -EINVAL;
536 }
537 sb->chunksize = cpu_to_le32(chunksize);
538
539 daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
540 if (!daemon_sleep || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
541 pr_debug("Choosing daemon_sleep default (5 sec)\n");
542 daemon_sleep = 5 * HZ;
543 }
544 sb->daemon_sleep = cpu_to_le32(daemon_sleep);
545 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
546
547 /*
548 * FIXME: write_behind for RAID1. If not specified, what
549 * is a good choice? We choose COUNTER_MAX / 2 arbitrarily.
550 */
551 write_behind = bitmap->mddev->bitmap_info.max_write_behind;
552 if (write_behind > COUNTER_MAX)
553 write_behind = COUNTER_MAX / 2;
554 sb->write_behind = cpu_to_le32(write_behind);
555 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
556
557 /* keep the array size field of the bitmap superblock up to date */
558 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
559
560 memcpy(sb->uuid, bitmap->mddev->uuid, 16);
561
562 set_bit(BITMAP_STALE, &bitmap->flags);
563 sb->state = cpu_to_le32(bitmap->flags);
564 bitmap->events_cleared = bitmap->mddev->events;
565 sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
566 bitmap->mddev->bitmap_info.nodes = 0;
567
568 kunmap_atomic(sb);
569
570 return 0;
571}
572
573/* read the superblock from the bitmap file and initialize some bitmap fields */
574static int bitmap_read_sb(struct bitmap *bitmap)
575{
576 char *reason = NULL;
577 bitmap_super_t *sb;
578 unsigned long chunksize, daemon_sleep, write_behind;
579 unsigned long long events;
580 int nodes = 0;
581 unsigned long sectors_reserved = 0;
582 int err = -EINVAL;
583 struct page *sb_page;
584 loff_t offset = bitmap->mddev->bitmap_info.offset;
585
586 if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
587 chunksize = 128 * 1024 * 1024;
588 daemon_sleep = 5 * HZ;
589 write_behind = 0;
590 set_bit(BITMAP_STALE, &bitmap->flags);
591 err = 0;
592 goto out_no_sb;
593 }
594 /* page 0 is the superblock, read it... */
595 sb_page = alloc_page(GFP_KERNEL);
596 if (!sb_page)
597 return -ENOMEM;
598 bitmap->storage.sb_page = sb_page;
599
600re_read:
601 /* If cluster_slot is set, the cluster is setup */
602 if (bitmap->cluster_slot >= 0) {
603 sector_t bm_blocks = bitmap->mddev->resync_max_sectors;
604
605 sector_div(bm_blocks,
606 bitmap->mddev->bitmap_info.chunksize >> 9);
607 /* bits to bytes */
608 bm_blocks = ((bm_blocks+7) >> 3) + sizeof(bitmap_super_t);
609 /* to 4k blocks */
610 bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks, 4096);
611 offset = bitmap->mddev->bitmap_info.offset + (bitmap->cluster_slot * (bm_blocks << 3));
612 pr_debug("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
613 bitmap->cluster_slot, offset);
614 }
615
616 if (bitmap->storage.file) {
617 loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
618 int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
619
620 err = read_page(bitmap->storage.file, 0,
621 bitmap, bytes, sb_page);
622 } else {
623 err = read_sb_page(bitmap->mddev,
624 offset,
625 sb_page,
626 0, sizeof(bitmap_super_t));
627 }
628 if (err)
629 return err;
630
631 err = -EINVAL;
632 sb = kmap_atomic(sb_page);
633
634 chunksize = le32_to_cpu(sb->chunksize);
635 daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
636 write_behind = le32_to_cpu(sb->write_behind);
637 sectors_reserved = le32_to_cpu(sb->sectors_reserved);
638 /* Setup nodes/clustername only if bitmap version is
639 * cluster-compatible
640 */
641 if (sb->version == cpu_to_le32(BITMAP_MAJOR_CLUSTERED)) {
642 nodes = le32_to_cpu(sb->nodes);
643 strlcpy(bitmap->mddev->bitmap_info.cluster_name,
644 sb->cluster_name, 64);
645 }
646
647 /* verify that the bitmap-specific fields are valid */
648 if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
649 reason = "bad magic";
650 else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
651 le32_to_cpu(sb->version) > BITMAP_MAJOR_CLUSTERED)
652 reason = "unrecognized superblock version";
653 else if (chunksize < 512)
654 reason = "bitmap chunksize too small";
655 else if (!is_power_of_2(chunksize))
656 reason = "bitmap chunksize not a power of 2";
657 else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT)
658 reason = "daemon sleep period out of range";
659 else if (write_behind > COUNTER_MAX)
660 reason = "write-behind limit out of range (0 - 16383)";
661 if (reason) {
662 pr_warn("%s: invalid bitmap file superblock: %s\n",
663 bmname(bitmap), reason);
664 goto out;
665 }
666
667 /* keep the array size field of the bitmap superblock up to date */
668 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
669
670 if (bitmap->mddev->persistent) {
671 /*
672 * We have a persistent array superblock, so compare the
673 * bitmap's UUID and event counter to the mddev's
674 */
675 if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
676 pr_warn("%s: bitmap superblock UUID mismatch\n",
677 bmname(bitmap));
678 goto out;
679 }
680 events = le64_to_cpu(sb->events);
681 if (!nodes && (events < bitmap->mddev->events)) {
682 pr_warn("%s: bitmap file is out of date (%llu < %llu) -- forcing full recovery\n",
683 bmname(bitmap), events,
684 (unsigned long long) bitmap->mddev->events);
685 set_bit(BITMAP_STALE, &bitmap->flags);
686 }
687 }
688
689 /* assign fields using values from superblock */
690 bitmap->flags |= le32_to_cpu(sb->state);
691 if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
692 set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
693 bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
694 strlcpy(bitmap->mddev->bitmap_info.cluster_name, sb->cluster_name, 64);
695 err = 0;
696
697out:
698 kunmap_atomic(sb);
699 /* Assiging chunksize is required for "re_read" */
700 bitmap->mddev->bitmap_info.chunksize = chunksize;
701 if (err == 0 && nodes && (bitmap->cluster_slot < 0)) {
702 err = md_setup_cluster(bitmap->mddev, nodes);
703 if (err) {
704 pr_warn("%s: Could not setup cluster service (%d)\n",
705 bmname(bitmap), err);
706 goto out_no_sb;
707 }
708 bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev);
709 goto re_read;
710 }
711
712
713out_no_sb:
714 if (test_bit(BITMAP_STALE, &bitmap->flags))
715 bitmap->events_cleared = bitmap->mddev->events;
716 bitmap->mddev->bitmap_info.chunksize = chunksize;
717 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
718 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
719 bitmap->mddev->bitmap_info.nodes = nodes;
720 if (bitmap->mddev->bitmap_info.space == 0 ||
721 bitmap->mddev->bitmap_info.space > sectors_reserved)
722 bitmap->mddev->bitmap_info.space = sectors_reserved;
723 if (err) {
724 bitmap_print_sb(bitmap);
725 if (bitmap->cluster_slot < 0)
726 md_cluster_stop(bitmap->mddev);
727 }
728 return err;
729}
730
731/*
732 * general bitmap file operations
733 */
734
735/*
736 * on-disk bitmap:
737 *
738 * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
739 * file a page at a time. There's a superblock at the start of the file.
740 */
741/* calculate the index of the page that contains this bit */
742static inline unsigned long file_page_index(struct bitmap_storage *store,
743 unsigned long chunk)
744{
745 if (store->sb_page)
746 chunk += sizeof(bitmap_super_t) << 3;
747 return chunk >> PAGE_BIT_SHIFT;
748}
749
750/* calculate the (bit) offset of this bit within a page */
751static inline unsigned long file_page_offset(struct bitmap_storage *store,
752 unsigned long chunk)
753{
754 if (store->sb_page)
755 chunk += sizeof(bitmap_super_t) << 3;
756 return chunk & (PAGE_BITS - 1);
757}
758
759/*
760 * return a pointer to the page in the filemap that contains the given bit
761 *
762 */
763static inline struct page *filemap_get_page(struct bitmap_storage *store,
764 unsigned long chunk)
765{
766 if (file_page_index(store, chunk) >= store->file_pages)
767 return NULL;
768 return store->filemap[file_page_index(store, chunk)];
769}
770
771static int bitmap_storage_alloc(struct bitmap_storage *store,
772 unsigned long chunks, int with_super,
773 int slot_number)
774{
775 int pnum, offset = 0;
776 unsigned long num_pages;
777 unsigned long bytes;
778
779 bytes = DIV_ROUND_UP(chunks, 8);
780 if (with_super)
781 bytes += sizeof(bitmap_super_t);
782
783 num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
784 offset = slot_number * num_pages;
785
786 store->filemap = kmalloc(sizeof(struct page *)
787 * num_pages, GFP_KERNEL);
788 if (!store->filemap)
789 return -ENOMEM;
790
791 if (with_super && !store->sb_page) {
792 store->sb_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
793 if (store->sb_page == NULL)
794 return -ENOMEM;
795 }
796
797 pnum = 0;
798 if (store->sb_page) {
799 store->filemap[0] = store->sb_page;
800 pnum = 1;
801 store->sb_page->index = offset;
802 }
803
804 for ( ; pnum < num_pages; pnum++) {
805 store->filemap[pnum] = alloc_page(GFP_KERNEL|__GFP_ZERO);
806 if (!store->filemap[pnum]) {
807 store->file_pages = pnum;
808 return -ENOMEM;
809 }
810 store->filemap[pnum]->index = pnum + offset;
811 }
812 store->file_pages = pnum;
813
814 /* We need 4 bits per page, rounded up to a multiple
815 * of sizeof(unsigned long) */
816 store->filemap_attr = kzalloc(
817 roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
818 GFP_KERNEL);
819 if (!store->filemap_attr)
820 return -ENOMEM;
821
822 store->bytes = bytes;
823
824 return 0;
825}
826
827static void bitmap_file_unmap(struct bitmap_storage *store)
828{
829 struct page **map, *sb_page;
830 int pages;
831 struct file *file;
832
833 file = store->file;
834 map = store->filemap;
835 pages = store->file_pages;
836 sb_page = store->sb_page;
837
838 while (pages--)
839 if (map[pages] != sb_page) /* 0 is sb_page, release it below */
840 free_buffers(map[pages]);
841 kfree(map);
842 kfree(store->filemap_attr);
843
844 if (sb_page)
845 free_buffers(sb_page);
846
847 if (file) {
848 struct inode *inode = file_inode(file);
849 invalidate_mapping_pages(inode->i_mapping, 0, -1);
850 fput(file);
851 }
852}
853
854/*
855 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
856 * then it is no longer reliable, so we stop using it and we mark the file
857 * as failed in the superblock
858 */
859static void bitmap_file_kick(struct bitmap *bitmap)
860{
861 char *path, *ptr = NULL;
862
863 if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) {
864 bitmap_update_sb(bitmap);
865
866 if (bitmap->storage.file) {
867 path = kmalloc(PAGE_SIZE, GFP_KERNEL);
868 if (path)
869 ptr = file_path(bitmap->storage.file,
870 path, PAGE_SIZE);
871
872 pr_warn("%s: kicking failed bitmap file %s from array!\n",
873 bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
874
875 kfree(path);
876 } else
877 pr_warn("%s: disabling internal bitmap due to errors\n",
878 bmname(bitmap));
879 }
880}
881
882enum bitmap_page_attr {
883 BITMAP_PAGE_DIRTY = 0, /* there are set bits that need to be synced */
884 BITMAP_PAGE_PENDING = 1, /* there are bits that are being cleaned.
885 * i.e. counter is 1 or 2. */
886 BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
887};
888
889static inline void set_page_attr(struct bitmap *bitmap, int pnum,
890 enum bitmap_page_attr attr)
891{
892 set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
893}
894
895static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
896 enum bitmap_page_attr attr)
897{
898 clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
899}
900
901static inline int test_page_attr(struct bitmap *bitmap, int pnum,
902 enum bitmap_page_attr attr)
903{
904 return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
905}
906
907static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
908 enum bitmap_page_attr attr)
909{
910 return test_and_clear_bit((pnum<<2) + attr,
911 bitmap->storage.filemap_attr);
912}
913/*
914 * bitmap_file_set_bit -- called before performing a write to the md device
915 * to set (and eventually sync) a particular bit in the bitmap file
916 *
917 * we set the bit immediately, then we record the page number so that
918 * when an unplug occurs, we can flush the dirty pages out to disk
919 */
920static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
921{
922 unsigned long bit;
923 struct page *page;
924 void *kaddr;
925 unsigned long chunk = block >> bitmap->counts.chunkshift;
926 struct bitmap_storage *store = &bitmap->storage;
927 unsigned long node_offset = 0;
928
929 if (mddev_is_clustered(bitmap->mddev))
930 node_offset = bitmap->cluster_slot * store->file_pages;
931
932 page = filemap_get_page(&bitmap->storage, chunk);
933 if (!page)
934 return;
935 bit = file_page_offset(&bitmap->storage, chunk);
936
937 /* set the bit */
938 kaddr = kmap_atomic(page);
939 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
940 set_bit(bit, kaddr);
941 else
942 set_bit_le(bit, kaddr);
943 kunmap_atomic(kaddr);
944 pr_debug("set file bit %lu page %lu\n", bit, page->index);
945 /* record page number so it gets flushed to disk when unplug occurs */
946 set_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_DIRTY);
947}
948
949static void bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
950{
951 unsigned long bit;
952 struct page *page;
953 void *paddr;
954 unsigned long chunk = block >> bitmap->counts.chunkshift;
955 struct bitmap_storage *store = &bitmap->storage;
956 unsigned long node_offset = 0;
957
958 if (mddev_is_clustered(bitmap->mddev))
959 node_offset = bitmap->cluster_slot * store->file_pages;
960
961 page = filemap_get_page(&bitmap->storage, chunk);
962 if (!page)
963 return;
964 bit = file_page_offset(&bitmap->storage, chunk);
965 paddr = kmap_atomic(page);
966 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
967 clear_bit(bit, paddr);
968 else
969 clear_bit_le(bit, paddr);
970 kunmap_atomic(paddr);
971 if (!test_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_NEEDWRITE)) {
972 set_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_PENDING);
973 bitmap->allclean = 0;
974 }
975}
976
977static int bitmap_file_test_bit(struct bitmap *bitmap, sector_t block)
978{
979 unsigned long bit;
980 struct page *page;
981 void *paddr;
982 unsigned long chunk = block >> bitmap->counts.chunkshift;
983 int set = 0;
984
985 page = filemap_get_page(&bitmap->storage, chunk);
986 if (!page)
987 return -EINVAL;
988 bit = file_page_offset(&bitmap->storage, chunk);
989 paddr = kmap_atomic(page);
990 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
991 set = test_bit(bit, paddr);
992 else
993 set = test_bit_le(bit, paddr);
994 kunmap_atomic(paddr);
995 return set;
996}
997
998
999/* this gets called when the md device is ready to unplug its underlying
1000 * (slave) device queues -- before we let any writes go down, we need to
1001 * sync the dirty pages of the bitmap file to disk */
1002void bitmap_unplug(struct bitmap *bitmap)
1003{
1004 unsigned long i;
1005 int dirty, need_write;
1006 int writing = 0;
1007
1008 if (!bitmap || !bitmap->storage.filemap ||
1009 test_bit(BITMAP_STALE, &bitmap->flags))
1010 return;
1011
1012 /* look at each page to see if there are any set bits that need to be
1013 * flushed out to disk */
1014 for (i = 0; i < bitmap->storage.file_pages; i++) {
1015 if (!bitmap->storage.filemap)
1016 return;
1017 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
1018 need_write = test_and_clear_page_attr(bitmap, i,
1019 BITMAP_PAGE_NEEDWRITE);
1020 if (dirty || need_write) {
1021 if (!writing) {
1022 bitmap_wait_writes(bitmap);
1023 if (bitmap->mddev->queue)
1024 blk_add_trace_msg(bitmap->mddev->queue,
1025 "md bitmap_unplug");
1026 }
1027 clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
1028 write_page(bitmap, bitmap->storage.filemap[i], 0);
1029 writing = 1;
1030 }
1031 }
1032 if (writing)
1033 bitmap_wait_writes(bitmap);
1034
1035 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1036 bitmap_file_kick(bitmap);
1037}
1038EXPORT_SYMBOL(bitmap_unplug);
1039
1040static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1041/* * bitmap_init_from_disk -- called at bitmap_create time to initialize
1042 * the in-memory bitmap from the on-disk bitmap -- also, sets up the
1043 * memory mapping of the bitmap file
1044 * Special cases:
1045 * if there's no bitmap file, or if the bitmap file had been
1046 * previously kicked from the array, we mark all the bits as
1047 * 1's in order to cause a full resync.
1048 *
1049 * We ignore all bits for sectors that end earlier than 'start'.
1050 * This is used when reading an out-of-date bitmap...
1051 */
1052static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
1053{
1054 unsigned long i, chunks, index, oldindex, bit, node_offset = 0;
1055 struct page *page = NULL;
1056 unsigned long bit_cnt = 0;
1057 struct file *file;
1058 unsigned long offset;
1059 int outofdate;
1060 int ret = -ENOSPC;
1061 void *paddr;
1062 struct bitmap_storage *store = &bitmap->storage;
1063
1064 chunks = bitmap->counts.chunks;
1065 file = store->file;
1066
1067 if (!file && !bitmap->mddev->bitmap_info.offset) {
1068 /* No permanent bitmap - fill with '1s'. */
1069 store->filemap = NULL;
1070 store->file_pages = 0;
1071 for (i = 0; i < chunks ; i++) {
1072 /* if the disk bit is set, set the memory bit */
1073 int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
1074 >= start);
1075 bitmap_set_memory_bits(bitmap,
1076 (sector_t)i << bitmap->counts.chunkshift,
1077 needed);
1078 }
1079 return 0;
1080 }
1081
1082 outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
1083 if (outofdate)
1084 pr_warn("%s: bitmap file is out of date, doing full recovery\n", bmname(bitmap));
1085
1086 if (file && i_size_read(file->f_mapping->host) < store->bytes) {
1087 pr_warn("%s: bitmap file too short %lu < %lu\n",
1088 bmname(bitmap),
1089 (unsigned long) i_size_read(file->f_mapping->host),
1090 store->bytes);
1091 goto err;
1092 }
1093
1094 oldindex = ~0L;
1095 offset = 0;
1096 if (!bitmap->mddev->bitmap_info.external)
1097 offset = sizeof(bitmap_super_t);
1098
1099 if (mddev_is_clustered(bitmap->mddev))
1100 node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE));
1101
1102 for (i = 0; i < chunks; i++) {
1103 int b;
1104 index = file_page_index(&bitmap->storage, i);
1105 bit = file_page_offset(&bitmap->storage, i);
1106 if (index != oldindex) { /* this is a new page, read it in */
1107 int count;
1108 /* unmap the old page, we're done with it */
1109 if (index == store->file_pages-1)
1110 count = store->bytes - index * PAGE_SIZE;
1111 else
1112 count = PAGE_SIZE;
1113 page = store->filemap[index];
1114 if (file)
1115 ret = read_page(file, index, bitmap,
1116 count, page);
1117 else
1118 ret = read_sb_page(
1119 bitmap->mddev,
1120 bitmap->mddev->bitmap_info.offset,
1121 page,
1122 index + node_offset, count);
1123
1124 if (ret)
1125 goto err;
1126
1127 oldindex = index;
1128
1129 if (outofdate) {
1130 /*
1131 * if bitmap is out of date, dirty the
1132 * whole page and write it out
1133 */
1134 paddr = kmap_atomic(page);
1135 memset(paddr + offset, 0xff,
1136 PAGE_SIZE - offset);
1137 kunmap_atomic(paddr);
1138 write_page(bitmap, page, 1);
1139
1140 ret = -EIO;
1141 if (test_bit(BITMAP_WRITE_ERROR,
1142 &bitmap->flags))
1143 goto err;
1144 }
1145 }
1146 paddr = kmap_atomic(page);
1147 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1148 b = test_bit(bit, paddr);
1149 else
1150 b = test_bit_le(bit, paddr);
1151 kunmap_atomic(paddr);
1152 if (b) {
1153 /* if the disk bit is set, set the memory bit */
1154 int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
1155 >= start);
1156 bitmap_set_memory_bits(bitmap,
1157 (sector_t)i << bitmap->counts.chunkshift,
1158 needed);
1159 bit_cnt++;
1160 }
1161 offset = 0;
1162 }
1163
1164 pr_debug("%s: bitmap initialized from disk: read %lu pages, set %lu of %lu bits\n",
1165 bmname(bitmap), store->file_pages,
1166 bit_cnt, chunks);
1167
1168 return 0;
1169
1170 err:
1171 pr_warn("%s: bitmap initialisation failed: %d\n",
1172 bmname(bitmap), ret);
1173 return ret;
1174}
1175
1176void bitmap_write_all(struct bitmap *bitmap)
1177{
1178 /* We don't actually write all bitmap blocks here,
1179 * just flag them as needing to be written
1180 */
1181 int i;
1182
1183 if (!bitmap || !bitmap->storage.filemap)
1184 return;
1185 if (bitmap->storage.file)
1186 /* Only one copy, so nothing needed */
1187 return;
1188
1189 for (i = 0; i < bitmap->storage.file_pages; i++)
1190 set_page_attr(bitmap, i,
1191 BITMAP_PAGE_NEEDWRITE);
1192 bitmap->allclean = 0;
1193}
1194
1195static void bitmap_count_page(struct bitmap_counts *bitmap,
1196 sector_t offset, int inc)
1197{
1198 sector_t chunk = offset >> bitmap->chunkshift;
1199 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1200 bitmap->bp[page].count += inc;
1201 bitmap_checkfree(bitmap, page);
1202}
1203
1204static void bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
1205{
1206 sector_t chunk = offset >> bitmap->chunkshift;
1207 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1208 struct bitmap_page *bp = &bitmap->bp[page];
1209
1210 if (!bp->pending)
1211 bp->pending = 1;
1212}
1213
1214static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
1215 sector_t offset, sector_t *blocks,
1216 int create);
1217
1218/*
1219 * bitmap daemon -- periodically wakes up to clean bits and flush pages
1220 * out to disk
1221 */
1222
1223void bitmap_daemon_work(struct mddev *mddev)
1224{
1225 struct bitmap *bitmap;
1226 unsigned long j;
1227 unsigned long nextpage;
1228 sector_t blocks;
1229 struct bitmap_counts *counts;
1230
1231 /* Use a mutex to guard daemon_work against
1232 * bitmap_destroy.
1233 */
1234 mutex_lock(&mddev->bitmap_info.mutex);
1235 bitmap = mddev->bitmap;
1236 if (bitmap == NULL) {
1237 mutex_unlock(&mddev->bitmap_info.mutex);
1238 return;
1239 }
1240 if (time_before(jiffies, bitmap->daemon_lastrun
1241 + mddev->bitmap_info.daemon_sleep))
1242 goto done;
1243
1244 bitmap->daemon_lastrun = jiffies;
1245 if (bitmap->allclean) {
1246 mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1247 goto done;
1248 }
1249 bitmap->allclean = 1;
1250
1251 if (bitmap->mddev->queue)
1252 blk_add_trace_msg(bitmap->mddev->queue,
1253 "md bitmap_daemon_work");
1254
1255 /* Any file-page which is PENDING now needs to be written.
1256 * So set NEEDWRITE now, then after we make any last-minute changes
1257 * we will write it.
1258 */
1259 for (j = 0; j < bitmap->storage.file_pages; j++)
1260 if (test_and_clear_page_attr(bitmap, j,
1261 BITMAP_PAGE_PENDING))
1262 set_page_attr(bitmap, j,
1263 BITMAP_PAGE_NEEDWRITE);
1264
1265 if (bitmap->need_sync &&
1266 mddev->bitmap_info.external == 0) {
1267 /* Arrange for superblock update as well as
1268 * other changes */
1269 bitmap_super_t *sb;
1270 bitmap->need_sync = 0;
1271 if (bitmap->storage.filemap) {
1272 sb = kmap_atomic(bitmap->storage.sb_page);
1273 sb->events_cleared =
1274 cpu_to_le64(bitmap->events_cleared);
1275 kunmap_atomic(sb);
1276 set_page_attr(bitmap, 0,
1277 BITMAP_PAGE_NEEDWRITE);
1278 }
1279 }
1280 /* Now look at the bitmap counters and if any are '2' or '1',
1281 * decrement and handle accordingly.
1282 */
1283 counts = &bitmap->counts;
1284 spin_lock_irq(&counts->lock);
1285 nextpage = 0;
1286 for (j = 0; j < counts->chunks; j++) {
1287 bitmap_counter_t *bmc;
1288 sector_t block = (sector_t)j << counts->chunkshift;
1289
1290 if (j == nextpage) {
1291 nextpage += PAGE_COUNTER_RATIO;
1292 if (!counts->bp[j >> PAGE_COUNTER_SHIFT].pending) {
1293 j |= PAGE_COUNTER_MASK;
1294 continue;
1295 }
1296 counts->bp[j >> PAGE_COUNTER_SHIFT].pending = 0;
1297 }
1298 bmc = bitmap_get_counter(counts,
1299 block,
1300 &blocks, 0);
1301
1302 if (!bmc) {
1303 j |= PAGE_COUNTER_MASK;
1304 continue;
1305 }
1306 if (*bmc == 1 && !bitmap->need_sync) {
1307 /* We can clear the bit */
1308 *bmc = 0;
1309 bitmap_count_page(counts, block, -1);
1310 bitmap_file_clear_bit(bitmap, block);
1311 } else if (*bmc && *bmc <= 2) {
1312 *bmc = 1;
1313 bitmap_set_pending(counts, block);
1314 bitmap->allclean = 0;
1315 }
1316 }
1317 spin_unlock_irq(&counts->lock);
1318
1319 bitmap_wait_writes(bitmap);
1320 /* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
1321 * DIRTY pages need to be written by bitmap_unplug so it can wait
1322 * for them.
1323 * If we find any DIRTY page we stop there and let bitmap_unplug
1324 * handle all the rest. This is important in the case where
1325 * the first blocking holds the superblock and it has been updated.
1326 * We mustn't write any other blocks before the superblock.
1327 */
1328 for (j = 0;
1329 j < bitmap->storage.file_pages
1330 && !test_bit(BITMAP_STALE, &bitmap->flags);
1331 j++) {
1332 if (test_page_attr(bitmap, j,
1333 BITMAP_PAGE_DIRTY))
1334 /* bitmap_unplug will handle the rest */
1335 break;
1336 if (test_and_clear_page_attr(bitmap, j,
1337 BITMAP_PAGE_NEEDWRITE)) {
1338 write_page(bitmap, bitmap->storage.filemap[j], 0);
1339 }
1340 }
1341
1342 done:
1343 if (bitmap->allclean == 0)
1344 mddev->thread->timeout =
1345 mddev->bitmap_info.daemon_sleep;
1346 mutex_unlock(&mddev->bitmap_info.mutex);
1347}
1348
1349static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
1350 sector_t offset, sector_t *blocks,
1351 int create)
1352__releases(bitmap->lock)
1353__acquires(bitmap->lock)
1354{
1355 /* If 'create', we might release the lock and reclaim it.
1356 * The lock must have been taken with interrupts enabled.
1357 * If !create, we don't release the lock.
1358 */
1359 sector_t chunk = offset >> bitmap->chunkshift;
1360 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1361 unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
1362 sector_t csize;
1363 int err;
1364
1365 err = bitmap_checkpage(bitmap, page, create, 0);
1366
1367 if (bitmap->bp[page].hijacked ||
1368 bitmap->bp[page].map == NULL)
1369 csize = ((sector_t)1) << (bitmap->chunkshift +
1370 PAGE_COUNTER_SHIFT - 1);
1371 else
1372 csize = ((sector_t)1) << bitmap->chunkshift;
1373 *blocks = csize - (offset & (csize - 1));
1374
1375 if (err < 0)
1376 return NULL;
1377
1378 /* now locked ... */
1379
1380 if (bitmap->bp[page].hijacked) { /* hijacked pointer */
1381 /* should we use the first or second counter field
1382 * of the hijacked pointer? */
1383 int hi = (pageoff > PAGE_COUNTER_MASK);
1384 return &((bitmap_counter_t *)
1385 &bitmap->bp[page].map)[hi];
1386 } else /* page is allocated */
1387 return (bitmap_counter_t *)
1388 &(bitmap->bp[page].map[pageoff]);
1389}
1390
1391int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
1392{
1393 if (!bitmap)
1394 return 0;
1395
1396 if (behind) {
1397 int bw;
1398 atomic_inc(&bitmap->behind_writes);
1399 bw = atomic_read(&bitmap->behind_writes);
1400 if (bw > bitmap->behind_writes_used)
1401 bitmap->behind_writes_used = bw;
1402
1403 pr_debug("inc write-behind count %d/%lu\n",
1404 bw, bitmap->mddev->bitmap_info.max_write_behind);
1405 }
1406
1407 while (sectors) {
1408 sector_t blocks;
1409 bitmap_counter_t *bmc;
1410
1411 spin_lock_irq(&bitmap->counts.lock);
1412 bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 1);
1413 if (!bmc) {
1414 spin_unlock_irq(&bitmap->counts.lock);
1415 return 0;
1416 }
1417
1418 if (unlikely(COUNTER(*bmc) == COUNTER_MAX)) {
1419 DEFINE_WAIT(__wait);
1420 /* note that it is safe to do the prepare_to_wait
1421 * after the test as long as we do it before dropping
1422 * the spinlock.
1423 */
1424 prepare_to_wait(&bitmap->overflow_wait, &__wait,
1425 TASK_UNINTERRUPTIBLE);
1426 spin_unlock_irq(&bitmap->counts.lock);
1427 schedule();
1428 finish_wait(&bitmap->overflow_wait, &__wait);
1429 continue;
1430 }
1431
1432 switch (*bmc) {
1433 case 0:
1434 bitmap_file_set_bit(bitmap, offset);
1435 bitmap_count_page(&bitmap->counts, offset, 1);
1436 /* fall through */
1437 case 1:
1438 *bmc = 2;
1439 }
1440
1441 (*bmc)++;
1442
1443 spin_unlock_irq(&bitmap->counts.lock);
1444
1445 offset += blocks;
1446 if (sectors > blocks)
1447 sectors -= blocks;
1448 else
1449 sectors = 0;
1450 }
1451 return 0;
1452}
1453EXPORT_SYMBOL(bitmap_startwrite);
1454
1455void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors,
1456 int success, int behind)
1457{
1458 if (!bitmap)
1459 return;
1460 if (behind) {
1461 if (atomic_dec_and_test(&bitmap->behind_writes))
1462 wake_up(&bitmap->behind_wait);
1463 pr_debug("dec write-behind count %d/%lu\n",
1464 atomic_read(&bitmap->behind_writes),
1465 bitmap->mddev->bitmap_info.max_write_behind);
1466 }
1467
1468 while (sectors) {
1469 sector_t blocks;
1470 unsigned long flags;
1471 bitmap_counter_t *bmc;
1472
1473 spin_lock_irqsave(&bitmap->counts.lock, flags);
1474 bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 0);
1475 if (!bmc) {
1476 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1477 return;
1478 }
1479
1480 if (success && !bitmap->mddev->degraded &&
1481 bitmap->events_cleared < bitmap->mddev->events) {
1482 bitmap->events_cleared = bitmap->mddev->events;
1483 bitmap->need_sync = 1;
1484 sysfs_notify_dirent_safe(bitmap->sysfs_can_clear);
1485 }
1486
1487 if (!success && !NEEDED(*bmc))
1488 *bmc |= NEEDED_MASK;
1489
1490 if (COUNTER(*bmc) == COUNTER_MAX)
1491 wake_up(&bitmap->overflow_wait);
1492
1493 (*bmc)--;
1494 if (*bmc <= 2) {
1495 bitmap_set_pending(&bitmap->counts, offset);
1496 bitmap->allclean = 0;
1497 }
1498 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1499 offset += blocks;
1500 if (sectors > blocks)
1501 sectors -= blocks;
1502 else
1503 sectors = 0;
1504 }
1505}
1506EXPORT_SYMBOL(bitmap_endwrite);
1507
1508static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1509 int degraded)
1510{
1511 bitmap_counter_t *bmc;
1512 int rv;
1513 if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
1514 *blocks = 1024;
1515 return 1; /* always resync if no bitmap */
1516 }
1517 spin_lock_irq(&bitmap->counts.lock);
1518 bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1519 rv = 0;
1520 if (bmc) {
1521 /* locked */
1522 if (RESYNC(*bmc))
1523 rv = 1;
1524 else if (NEEDED(*bmc)) {
1525 rv = 1;
1526 if (!degraded) { /* don't set/clear bits if degraded */
1527 *bmc |= RESYNC_MASK;
1528 *bmc &= ~NEEDED_MASK;
1529 }
1530 }
1531 }
1532 spin_unlock_irq(&bitmap->counts.lock);
1533 return rv;
1534}
1535
1536int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1537 int degraded)
1538{
1539 /* bitmap_start_sync must always report on multiples of whole
1540 * pages, otherwise resync (which is very PAGE_SIZE based) will
1541 * get confused.
1542 * So call __bitmap_start_sync repeatedly (if needed) until
1543 * At least PAGE_SIZE>>9 blocks are covered.
1544 * Return the 'or' of the result.
1545 */
1546 int rv = 0;
1547 sector_t blocks1;
1548
1549 *blocks = 0;
1550 while (*blocks < (PAGE_SIZE>>9)) {
1551 rv |= __bitmap_start_sync(bitmap, offset,
1552 &blocks1, degraded);
1553 offset += blocks1;
1554 *blocks += blocks1;
1555 }
1556 return rv;
1557}
1558EXPORT_SYMBOL(bitmap_start_sync);
1559
1560void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
1561{
1562 bitmap_counter_t *bmc;
1563 unsigned long flags;
1564
1565 if (bitmap == NULL) {
1566 *blocks = 1024;
1567 return;
1568 }
1569 spin_lock_irqsave(&bitmap->counts.lock, flags);
1570 bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1571 if (bmc == NULL)
1572 goto unlock;
1573 /* locked */
1574 if (RESYNC(*bmc)) {
1575 *bmc &= ~RESYNC_MASK;
1576
1577 if (!NEEDED(*bmc) && aborted)
1578 *bmc |= NEEDED_MASK;
1579 else {
1580 if (*bmc <= 2) {
1581 bitmap_set_pending(&bitmap->counts, offset);
1582 bitmap->allclean = 0;
1583 }
1584 }
1585 }
1586 unlock:
1587 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1588}
1589EXPORT_SYMBOL(bitmap_end_sync);
1590
1591void bitmap_close_sync(struct bitmap *bitmap)
1592{
1593 /* Sync has finished, and any bitmap chunks that weren't synced
1594 * properly have been aborted. It remains to us to clear the
1595 * RESYNC bit wherever it is still on
1596 */
1597 sector_t sector = 0;
1598 sector_t blocks;
1599 if (!bitmap)
1600 return;
1601 while (sector < bitmap->mddev->resync_max_sectors) {
1602 bitmap_end_sync(bitmap, sector, &blocks, 0);
1603 sector += blocks;
1604 }
1605}
1606EXPORT_SYMBOL(bitmap_close_sync);
1607
1608void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector, bool force)
1609{
1610 sector_t s = 0;
1611 sector_t blocks;
1612
1613 if (!bitmap)
1614 return;
1615 if (sector == 0) {
1616 bitmap->last_end_sync = jiffies;
1617 return;
1618 }
1619 if (!force && time_before(jiffies, (bitmap->last_end_sync
1620 + bitmap->mddev->bitmap_info.daemon_sleep)))
1621 return;
1622 wait_event(bitmap->mddev->recovery_wait,
1623 atomic_read(&bitmap->mddev->recovery_active) == 0);
1624
1625 bitmap->mddev->curr_resync_completed = sector;
1626 set_bit(MD_SB_CHANGE_CLEAN, &bitmap->mddev->sb_flags);
1627 sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
1628 s = 0;
1629 while (s < sector && s < bitmap->mddev->resync_max_sectors) {
1630 bitmap_end_sync(bitmap, s, &blocks, 0);
1631 s += blocks;
1632 }
1633 bitmap->last_end_sync = jiffies;
1634 sysfs_notify(&bitmap->mddev->kobj, NULL, "sync_completed");
1635}
1636EXPORT_SYMBOL(bitmap_cond_end_sync);
1637
1638void bitmap_sync_with_cluster(struct mddev *mddev,
1639 sector_t old_lo, sector_t old_hi,
1640 sector_t new_lo, sector_t new_hi)
1641{
1642 struct bitmap *bitmap = mddev->bitmap;
1643 sector_t sector, blocks = 0;
1644
1645 for (sector = old_lo; sector < new_lo; ) {
1646 bitmap_end_sync(bitmap, sector, &blocks, 0);
1647 sector += blocks;
1648 }
1649 WARN((blocks > new_lo) && old_lo, "alignment is not correct for lo\n");
1650
1651 for (sector = old_hi; sector < new_hi; ) {
1652 bitmap_start_sync(bitmap, sector, &blocks, 0);
1653 sector += blocks;
1654 }
1655 WARN((blocks > new_hi) && old_hi, "alignment is not correct for hi\n");
1656}
1657EXPORT_SYMBOL(bitmap_sync_with_cluster);
1658
1659static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1660{
1661 /* For each chunk covered by any of these sectors, set the
1662 * counter to 2 and possibly set resync_needed. They should all
1663 * be 0 at this point
1664 */
1665
1666 sector_t secs;
1667 bitmap_counter_t *bmc;
1668 spin_lock_irq(&bitmap->counts.lock);
1669 bmc = bitmap_get_counter(&bitmap->counts, offset, &secs, 1);
1670 if (!bmc) {
1671 spin_unlock_irq(&bitmap->counts.lock);
1672 return;
1673 }
1674 if (!*bmc) {
1675 *bmc = 2;
1676 bitmap_count_page(&bitmap->counts, offset, 1);
1677 bitmap_set_pending(&bitmap->counts, offset);
1678 bitmap->allclean = 0;
1679 }
1680 if (needed)
1681 *bmc |= NEEDED_MASK;
1682 spin_unlock_irq(&bitmap->counts.lock);
1683}
1684
1685/* dirty the memory and file bits for bitmap chunks "s" to "e" */
1686void bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
1687{
1688 unsigned long chunk;
1689
1690 for (chunk = s; chunk <= e; chunk++) {
1691 sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;
1692 bitmap_set_memory_bits(bitmap, sec, 1);
1693 bitmap_file_set_bit(bitmap, sec);
1694 if (sec < bitmap->mddev->recovery_cp)
1695 /* We are asserting that the array is dirty,
1696 * so move the recovery_cp address back so
1697 * that it is obvious that it is dirty
1698 */
1699 bitmap->mddev->recovery_cp = sec;
1700 }
1701}
1702
1703/*
1704 * flush out any pending updates
1705 */
1706void bitmap_flush(struct mddev *mddev)
1707{
1708 struct bitmap *bitmap = mddev->bitmap;
1709 long sleep;
1710
1711 if (!bitmap) /* there was no bitmap */
1712 return;
1713
1714 /* run the daemon_work three time to ensure everything is flushed
1715 * that can be
1716 */
1717 sleep = mddev->bitmap_info.daemon_sleep * 2;
1718 bitmap->daemon_lastrun -= sleep;
1719 bitmap_daemon_work(mddev);
1720 bitmap->daemon_lastrun -= sleep;
1721 bitmap_daemon_work(mddev);
1722 bitmap->daemon_lastrun -= sleep;
1723 bitmap_daemon_work(mddev);
1724 bitmap_update_sb(bitmap);
1725}
1726
1727/*
1728 * free memory that was allocated
1729 */
1730static void bitmap_free(struct bitmap *bitmap)
1731{
1732 unsigned long k, pages;
1733 struct bitmap_page *bp;
1734
1735 if (!bitmap) /* there was no bitmap */
1736 return;
1737
1738 if (bitmap->sysfs_can_clear)
1739 sysfs_put(bitmap->sysfs_can_clear);
1740
1741 if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info &&
1742 bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev))
1743 md_cluster_stop(bitmap->mddev);
1744
1745 /* Shouldn't be needed - but just in case.... */
1746 wait_event(bitmap->write_wait,
1747 atomic_read(&bitmap->pending_writes) == 0);
1748
1749 /* release the bitmap file */
1750 bitmap_file_unmap(&bitmap->storage);
1751
1752 bp = bitmap->counts.bp;
1753 pages = bitmap->counts.pages;
1754
1755 /* free all allocated memory */
1756
1757 if (bp) /* deallocate the page memory */
1758 for (k = 0; k < pages; k++)
1759 if (bp[k].map && !bp[k].hijacked)
1760 kfree(bp[k].map);
1761 kfree(bp);
1762 kfree(bitmap);
1763}
1764
1765void bitmap_destroy(struct mddev *mddev)
1766{
1767 struct bitmap *bitmap = mddev->bitmap;
1768
1769 if (!bitmap) /* there was no bitmap */
1770 return;
1771
1772 mutex_lock(&mddev->bitmap_info.mutex);
1773 spin_lock(&mddev->lock);
1774 mddev->bitmap = NULL; /* disconnect from the md device */
1775 spin_unlock(&mddev->lock);
1776 mutex_unlock(&mddev->bitmap_info.mutex);
1777 if (mddev->thread)
1778 mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1779
1780 bitmap_free(bitmap);
1781}
1782
1783/*
1784 * initialize the bitmap structure
1785 * if this returns an error, bitmap_destroy must be called to do clean up
1786 * once mddev->bitmap is set
1787 */
1788struct bitmap *bitmap_create(struct mddev *mddev, int slot)
1789{
1790 struct bitmap *bitmap;
1791 sector_t blocks = mddev->resync_max_sectors;
1792 struct file *file = mddev->bitmap_info.file;
1793 int err;
1794 struct kernfs_node *bm = NULL;
1795
1796 BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
1797
1798 BUG_ON(file && mddev->bitmap_info.offset);
1799
1800 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
1801 if (!bitmap)
1802 return ERR_PTR(-ENOMEM);
1803
1804 spin_lock_init(&bitmap->counts.lock);
1805 atomic_set(&bitmap->pending_writes, 0);
1806 init_waitqueue_head(&bitmap->write_wait);
1807 init_waitqueue_head(&bitmap->overflow_wait);
1808 init_waitqueue_head(&bitmap->behind_wait);
1809
1810 bitmap->mddev = mddev;
1811 bitmap->cluster_slot = slot;
1812
1813 if (mddev->kobj.sd)
1814 bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap");
1815 if (bm) {
1816 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear");
1817 sysfs_put(bm);
1818 } else
1819 bitmap->sysfs_can_clear = NULL;
1820
1821 bitmap->storage.file = file;
1822 if (file) {
1823 get_file(file);
1824 /* As future accesses to this file will use bmap,
1825 * and bypass the page cache, we must sync the file
1826 * first.
1827 */
1828 vfs_fsync(file, 1);
1829 }
1830 /* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
1831 if (!mddev->bitmap_info.external) {
1832 /*
1833 * If 'MD_ARRAY_FIRST_USE' is set, then device-mapper is
1834 * instructing us to create a new on-disk bitmap instance.
1835 */
1836 if (test_and_clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags))
1837 err = bitmap_new_disk_sb(bitmap);
1838 else
1839 err = bitmap_read_sb(bitmap);
1840 } else {
1841 err = 0;
1842 if (mddev->bitmap_info.chunksize == 0 ||
1843 mddev->bitmap_info.daemon_sleep == 0)
1844 /* chunksize and time_base need to be
1845 * set first. */
1846 err = -EINVAL;
1847 }
1848 if (err)
1849 goto error;
1850
1851 bitmap->daemon_lastrun = jiffies;
1852 err = bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1);
1853 if (err)
1854 goto error;
1855
1856 pr_debug("created bitmap (%lu pages) for device %s\n",
1857 bitmap->counts.pages, bmname(bitmap));
1858
1859 err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
1860 if (err)
1861 goto error;
1862
1863 return bitmap;
1864 error:
1865 bitmap_free(bitmap);
1866 return ERR_PTR(err);
1867}
1868
1869int bitmap_load(struct mddev *mddev)
1870{
1871 int err = 0;
1872 sector_t start = 0;
1873 sector_t sector = 0;
1874 struct bitmap *bitmap = mddev->bitmap;
1875
1876 if (!bitmap)
1877 goto out;
1878
1879 if (mddev_is_clustered(mddev))
1880 md_cluster_ops->load_bitmaps(mddev, mddev->bitmap_info.nodes);
1881
1882 /* Clear out old bitmap info first: Either there is none, or we
1883 * are resuming after someone else has possibly changed things,
1884 * so we should forget old cached info.
1885 * All chunks should be clean, but some might need_sync.
1886 */
1887 while (sector < mddev->resync_max_sectors) {
1888 sector_t blocks;
1889 bitmap_start_sync(bitmap, sector, &blocks, 0);
1890 sector += blocks;
1891 }
1892 bitmap_close_sync(bitmap);
1893
1894 if (mddev->degraded == 0
1895 || bitmap->events_cleared == mddev->events)
1896 /* no need to keep dirty bits to optimise a
1897 * re-add of a missing device */
1898 start = mddev->recovery_cp;
1899
1900 mutex_lock(&mddev->bitmap_info.mutex);
1901 err = bitmap_init_from_disk(bitmap, start);
1902 mutex_unlock(&mddev->bitmap_info.mutex);
1903
1904 if (err)
1905 goto out;
1906 clear_bit(BITMAP_STALE, &bitmap->flags);
1907
1908 /* Kick recovery in case any bits were set */
1909 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
1910
1911 mddev->thread->timeout = mddev->bitmap_info.daemon_sleep;
1912 md_wakeup_thread(mddev->thread);
1913
1914 bitmap_update_sb(bitmap);
1915
1916 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1917 err = -EIO;
1918out:
1919 return err;
1920}
1921EXPORT_SYMBOL_GPL(bitmap_load);
1922
1923/* Loads the bitmap associated with slot and copies the resync information
1924 * to our bitmap
1925 */
1926int bitmap_copy_from_slot(struct mddev *mddev, int slot,
1927 sector_t *low, sector_t *high, bool clear_bits)
1928{
1929 int rv = 0, i, j;
1930 sector_t block, lo = 0, hi = 0;
1931 struct bitmap_counts *counts;
1932 struct bitmap *bitmap = bitmap_create(mddev, slot);
1933
1934 if (IS_ERR(bitmap))
1935 return PTR_ERR(bitmap);
1936
1937 rv = bitmap_init_from_disk(bitmap, 0);
1938 if (rv)
1939 goto err;
1940
1941 counts = &bitmap->counts;
1942 for (j = 0; j < counts->chunks; j++) {
1943 block = (sector_t)j << counts->chunkshift;
1944 if (bitmap_file_test_bit(bitmap, block)) {
1945 if (!lo)
1946 lo = block;
1947 hi = block;
1948 bitmap_file_clear_bit(bitmap, block);
1949 bitmap_set_memory_bits(mddev->bitmap, block, 1);
1950 bitmap_file_set_bit(mddev->bitmap, block);
1951 }
1952 }
1953
1954 if (clear_bits) {
1955 bitmap_update_sb(bitmap);
1956 /* BITMAP_PAGE_PENDING is set, but bitmap_unplug needs
1957 * BITMAP_PAGE_DIRTY or _NEEDWRITE to write ... */
1958 for (i = 0; i < bitmap->storage.file_pages; i++)
1959 if (test_page_attr(bitmap, i, BITMAP_PAGE_PENDING))
1960 set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE);
1961 bitmap_unplug(bitmap);
1962 }
1963 bitmap_unplug(mddev->bitmap);
1964 *low = lo;
1965 *high = hi;
1966err:
1967 bitmap_free(bitmap);
1968 return rv;
1969}
1970EXPORT_SYMBOL_GPL(bitmap_copy_from_slot);
1971
1972
1973void bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
1974{
1975 unsigned long chunk_kb;
1976 struct bitmap_counts *counts;
1977
1978 if (!bitmap)
1979 return;
1980
1981 counts = &bitmap->counts;
1982
1983 chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10;
1984 seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], "
1985 "%lu%s chunk",
1986 counts->pages - counts->missing_pages,
1987 counts->pages,
1988 (counts->pages - counts->missing_pages)
1989 << (PAGE_SHIFT - 10),
1990 chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize,
1991 chunk_kb ? "KB" : "B");
1992 if (bitmap->storage.file) {
1993 seq_printf(seq, ", file: ");
1994 seq_file_path(seq, bitmap->storage.file, " \t\n");
1995 }
1996
1997 seq_printf(seq, "\n");
1998}
1999
2000int bitmap_resize(struct bitmap *bitmap, sector_t blocks,
2001 int chunksize, int init)
2002{
2003 /* If chunk_size is 0, choose an appropriate chunk size.
2004 * Then possibly allocate new storage space.
2005 * Then quiesce, copy bits, replace bitmap, and re-start
2006 *
2007 * This function is called both to set up the initial bitmap
2008 * and to resize the bitmap while the array is active.
2009 * If this happens as a result of the array being resized,
2010 * chunksize will be zero, and we need to choose a suitable
2011 * chunksize, otherwise we use what we are given.
2012 */
2013 struct bitmap_storage store;
2014 struct bitmap_counts old_counts;
2015 unsigned long chunks;
2016 sector_t block;
2017 sector_t old_blocks, new_blocks;
2018 int chunkshift;
2019 int ret = 0;
2020 long pages;
2021 struct bitmap_page *new_bp;
2022
2023 if (chunksize == 0) {
2024 /* If there is enough space, leave the chunk size unchanged,
2025 * else increase by factor of two until there is enough space.
2026 */
2027 long bytes;
2028 long space = bitmap->mddev->bitmap_info.space;
2029
2030 if (space == 0) {
2031 /* We don't know how much space there is, so limit
2032 * to current size - in sectors.
2033 */
2034 bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
2035 if (!bitmap->mddev->bitmap_info.external)
2036 bytes += sizeof(bitmap_super_t);
2037 space = DIV_ROUND_UP(bytes, 512);
2038 bitmap->mddev->bitmap_info.space = space;
2039 }
2040 chunkshift = bitmap->counts.chunkshift;
2041 chunkshift--;
2042 do {
2043 /* 'chunkshift' is shift from block size to chunk size */
2044 chunkshift++;
2045 chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
2046 bytes = DIV_ROUND_UP(chunks, 8);
2047 if (!bitmap->mddev->bitmap_info.external)
2048 bytes += sizeof(bitmap_super_t);
2049 } while (bytes > (space << 9));
2050 } else
2051 chunkshift = ffz(~chunksize) - BITMAP_BLOCK_SHIFT;
2052
2053 chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
2054 memset(&store, 0, sizeof(store));
2055 if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
2056 ret = bitmap_storage_alloc(&store, chunks,
2057 !bitmap->mddev->bitmap_info.external,
2058 mddev_is_clustered(bitmap->mddev)
2059 ? bitmap->cluster_slot : 0);
2060 if (ret) {
2061 bitmap_file_unmap(&store);
2062 goto err;
2063 }
2064
2065 pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);
2066
2067 new_bp = kzalloc(pages * sizeof(*new_bp), GFP_KERNEL);
2068 ret = -ENOMEM;
2069 if (!new_bp) {
2070 bitmap_file_unmap(&store);
2071 goto err;
2072 }
2073
2074 if (!init)
2075 bitmap->mddev->pers->quiesce(bitmap->mddev, 1);
2076
2077 store.file = bitmap->storage.file;
2078 bitmap->storage.file = NULL;
2079
2080 if (store.sb_page && bitmap->storage.sb_page)
2081 memcpy(page_address(store.sb_page),
2082 page_address(bitmap->storage.sb_page),
2083 sizeof(bitmap_super_t));
2084 bitmap_file_unmap(&bitmap->storage);
2085 bitmap->storage = store;
2086
2087 old_counts = bitmap->counts;
2088 bitmap->counts.bp = new_bp;
2089 bitmap->counts.pages = pages;
2090 bitmap->counts.missing_pages = pages;
2091 bitmap->counts.chunkshift = chunkshift;
2092 bitmap->counts.chunks = chunks;
2093 bitmap->mddev->bitmap_info.chunksize = 1 << (chunkshift +
2094 BITMAP_BLOCK_SHIFT);
2095
2096 blocks = min(old_counts.chunks << old_counts.chunkshift,
2097 chunks << chunkshift);
2098
2099 spin_lock_irq(&bitmap->counts.lock);
2100 /* For cluster raid, need to pre-allocate bitmap */
2101 if (mddev_is_clustered(bitmap->mddev)) {
2102 unsigned long page;
2103 for (page = 0; page < pages; page++) {
2104 ret = bitmap_checkpage(&bitmap->counts, page, 1, 1);
2105 if (ret) {
2106 unsigned long k;
2107
2108 /* deallocate the page memory */
2109 for (k = 0; k < page; k++) {
2110 kfree(new_bp[k].map);
2111 }
2112
2113 /* restore some fields from old_counts */
2114 bitmap->counts.bp = old_counts.bp;
2115 bitmap->counts.pages = old_counts.pages;
2116 bitmap->counts.missing_pages = old_counts.pages;
2117 bitmap->counts.chunkshift = old_counts.chunkshift;
2118 bitmap->counts.chunks = old_counts.chunks;
2119 bitmap->mddev->bitmap_info.chunksize = 1 << (old_counts.chunkshift +
2120 BITMAP_BLOCK_SHIFT);
2121 blocks = old_counts.chunks << old_counts.chunkshift;
2122 pr_warn("Could not pre-allocate in-memory bitmap for cluster raid\n");
2123 break;
2124 } else
2125 bitmap->counts.bp[page].count += 1;
2126 }
2127 }
2128
2129 for (block = 0; block < blocks; ) {
2130 bitmap_counter_t *bmc_old, *bmc_new;
2131 int set;
2132
2133 bmc_old = bitmap_get_counter(&old_counts, block,
2134 &old_blocks, 0);
2135 set = bmc_old && NEEDED(*bmc_old);
2136
2137 if (set) {
2138 bmc_new = bitmap_get_counter(&bitmap->counts, block,
2139 &new_blocks, 1);
2140 if (*bmc_new == 0) {
2141 /* need to set on-disk bits too. */
2142 sector_t end = block + new_blocks;
2143 sector_t start = block >> chunkshift;
2144 start <<= chunkshift;
2145 while (start < end) {
2146 bitmap_file_set_bit(bitmap, block);
2147 start += 1 << chunkshift;
2148 }
2149 *bmc_new = 2;
2150 bitmap_count_page(&bitmap->counts,
2151 block, 1);
2152 bitmap_set_pending(&bitmap->counts,
2153 block);
2154 }
2155 *bmc_new |= NEEDED_MASK;
2156 if (new_blocks < old_blocks)
2157 old_blocks = new_blocks;
2158 }
2159 block += old_blocks;
2160 }
2161
2162 if (!init) {
2163 int i;
2164 while (block < (chunks << chunkshift)) {
2165 bitmap_counter_t *bmc;
2166 bmc = bitmap_get_counter(&bitmap->counts, block,
2167 &new_blocks, 1);
2168 if (bmc) {
2169 /* new space. It needs to be resynced, so
2170 * we set NEEDED_MASK.
2171 */
2172 if (*bmc == 0) {
2173 *bmc = NEEDED_MASK | 2;
2174 bitmap_count_page(&bitmap->counts,
2175 block, 1);
2176 bitmap_set_pending(&bitmap->counts,
2177 block);
2178 }
2179 }
2180 block += new_blocks;
2181 }
2182 for (i = 0; i < bitmap->storage.file_pages; i++)
2183 set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
2184 }
2185 spin_unlock_irq(&bitmap->counts.lock);
2186
2187 if (!init) {
2188 bitmap_unplug(bitmap);
2189 bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
2190 }
2191 ret = 0;
2192err:
2193 return ret;
2194}
2195EXPORT_SYMBOL_GPL(bitmap_resize);
2196
2197static ssize_t
2198location_show(struct mddev *mddev, char *page)
2199{
2200 ssize_t len;
2201 if (mddev->bitmap_info.file)
2202 len = sprintf(page, "file");
2203 else if (mddev->bitmap_info.offset)
2204 len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset);
2205 else
2206 len = sprintf(page, "none");
2207 len += sprintf(page+len, "\n");
2208 return len;
2209}
2210
2211static ssize_t
2212location_store(struct mddev *mddev, const char *buf, size_t len)
2213{
2214 int rv;
2215
2216 rv = mddev_lock(mddev);
2217 if (rv)
2218 return rv;
2219 if (mddev->pers) {
2220 if (!mddev->pers->quiesce) {
2221 rv = -EBUSY;
2222 goto out;
2223 }
2224 if (mddev->recovery || mddev->sync_thread) {
2225 rv = -EBUSY;
2226 goto out;
2227 }
2228 }
2229
2230 if (mddev->bitmap || mddev->bitmap_info.file ||
2231 mddev->bitmap_info.offset) {
2232 /* bitmap already configured. Only option is to clear it */
2233 if (strncmp(buf, "none", 4) != 0) {
2234 rv = -EBUSY;
2235 goto out;
2236 }
2237 if (mddev->pers) {
2238 mddev->pers->quiesce(mddev, 1);
2239 bitmap_destroy(mddev);
2240 mddev->pers->quiesce(mddev, 0);
2241 }
2242 mddev->bitmap_info.offset = 0;
2243 if (mddev->bitmap_info.file) {
2244 struct file *f = mddev->bitmap_info.file;
2245 mddev->bitmap_info.file = NULL;
2246 fput(f);
2247 }
2248 } else {
2249 /* No bitmap, OK to set a location */
2250 long long offset;
2251 if (strncmp(buf, "none", 4) == 0)
2252 /* nothing to be done */;
2253 else if (strncmp(buf, "file:", 5) == 0) {
2254 /* Not supported yet */
2255 rv = -EINVAL;
2256 goto out;
2257 } else {
2258 if (buf[0] == '+')
2259 rv = kstrtoll(buf+1, 10, &offset);
2260 else
2261 rv = kstrtoll(buf, 10, &offset);
2262 if (rv)
2263 goto out;
2264 if (offset == 0) {
2265 rv = -EINVAL;
2266 goto out;
2267 }
2268 if (mddev->bitmap_info.external == 0 &&
2269 mddev->major_version == 0 &&
2270 offset != mddev->bitmap_info.default_offset) {
2271 rv = -EINVAL;
2272 goto out;
2273 }
2274 mddev->bitmap_info.offset = offset;
2275 if (mddev->pers) {
2276 struct bitmap *bitmap;
2277 mddev->pers->quiesce(mddev, 1);
2278 bitmap = bitmap_create(mddev, -1);
2279 if (IS_ERR(bitmap))
2280 rv = PTR_ERR(bitmap);
2281 else {
2282 mddev->bitmap = bitmap;
2283 rv = bitmap_load(mddev);
2284 if (rv)
2285 mddev->bitmap_info.offset = 0;
2286 }
2287 mddev->pers->quiesce(mddev, 0);
2288 if (rv) {
2289 bitmap_destroy(mddev);
2290 goto out;
2291 }
2292 }
2293 }
2294 }
2295 if (!mddev->external) {
2296 /* Ensure new bitmap info is stored in
2297 * metadata promptly.
2298 */
2299 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2300 md_wakeup_thread(mddev->thread);
2301 }
2302 rv = 0;
2303out:
2304 mddev_unlock(mddev);
2305 if (rv)
2306 return rv;
2307 return len;
2308}
2309
2310static struct md_sysfs_entry bitmap_location =
2311__ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);
2312
2313/* 'bitmap/space' is the space available at 'location' for the
2314 * bitmap. This allows the kernel to know when it is safe to
2315 * resize the bitmap to match a resized array.
2316 */
2317static ssize_t
2318space_show(struct mddev *mddev, char *page)
2319{
2320 return sprintf(page, "%lu\n", mddev->bitmap_info.space);
2321}
2322
2323static ssize_t
2324space_store(struct mddev *mddev, const char *buf, size_t len)
2325{
2326 unsigned long sectors;
2327 int rv;
2328
2329 rv = kstrtoul(buf, 10, §ors);
2330 if (rv)
2331 return rv;
2332
2333 if (sectors == 0)
2334 return -EINVAL;
2335
2336 if (mddev->bitmap &&
2337 sectors < (mddev->bitmap->storage.bytes + 511) >> 9)
2338 return -EFBIG; /* Bitmap is too big for this small space */
2339
2340 /* could make sure it isn't too big, but that isn't really
2341 * needed - user-space should be careful.
2342 */
2343 mddev->bitmap_info.space = sectors;
2344 return len;
2345}
2346
2347static struct md_sysfs_entry bitmap_space =
2348__ATTR(space, S_IRUGO|S_IWUSR, space_show, space_store);
2349
2350static ssize_t
2351timeout_show(struct mddev *mddev, char *page)
2352{
2353 ssize_t len;
2354 unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
2355 unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;
2356
2357 len = sprintf(page, "%lu", secs);
2358 if (jifs)
2359 len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs));
2360 len += sprintf(page+len, "\n");
2361 return len;
2362}
2363
2364static ssize_t
2365timeout_store(struct mddev *mddev, const char *buf, size_t len)
2366{
2367 /* timeout can be set at any time */
2368 unsigned long timeout;
2369 int rv = strict_strtoul_scaled(buf, &timeout, 4);
2370 if (rv)
2371 return rv;
2372
2373 /* just to make sure we don't overflow... */
2374 if (timeout >= LONG_MAX / HZ)
2375 return -EINVAL;
2376
2377 timeout = timeout * HZ / 10000;
2378
2379 if (timeout >= MAX_SCHEDULE_TIMEOUT)
2380 timeout = MAX_SCHEDULE_TIMEOUT-1;
2381 if (timeout < 1)
2382 timeout = 1;
2383 mddev->bitmap_info.daemon_sleep = timeout;
2384 if (mddev->thread) {
2385 /* if thread->timeout is MAX_SCHEDULE_TIMEOUT, then
2386 * the bitmap is all clean and we don't need to
2387 * adjust the timeout right now
2388 */
2389 if (mddev->thread->timeout < MAX_SCHEDULE_TIMEOUT) {
2390 mddev->thread->timeout = timeout;
2391 md_wakeup_thread(mddev->thread);
2392 }
2393 }
2394 return len;
2395}
2396
2397static struct md_sysfs_entry bitmap_timeout =
2398__ATTR(time_base, S_IRUGO|S_IWUSR, timeout_show, timeout_store);
2399
2400static ssize_t
2401backlog_show(struct mddev *mddev, char *page)
2402{
2403 return sprintf(page, "%lu\n", mddev->bitmap_info.max_write_behind);
2404}
2405
2406static ssize_t
2407backlog_store(struct mddev *mddev, const char *buf, size_t len)
2408{
2409 unsigned long backlog;
2410 int rv = kstrtoul(buf, 10, &backlog);
2411 if (rv)
2412 return rv;
2413 if (backlog > COUNTER_MAX)
2414 return -EINVAL;
2415 mddev->bitmap_info.max_write_behind = backlog;
2416 return len;
2417}
2418
2419static struct md_sysfs_entry bitmap_backlog =
2420__ATTR(backlog, S_IRUGO|S_IWUSR, backlog_show, backlog_store);
2421
2422static ssize_t
2423chunksize_show(struct mddev *mddev, char *page)
2424{
2425 return sprintf(page, "%lu\n", mddev->bitmap_info.chunksize);
2426}
2427
2428static ssize_t
2429chunksize_store(struct mddev *mddev, const char *buf, size_t len)
2430{
2431 /* Can only be changed when no bitmap is active */
2432 int rv;
2433 unsigned long csize;
2434 if (mddev->bitmap)
2435 return -EBUSY;
2436 rv = kstrtoul(buf, 10, &csize);
2437 if (rv)
2438 return rv;
2439 if (csize < 512 ||
2440 !is_power_of_2(csize))
2441 return -EINVAL;
2442 mddev->bitmap_info.chunksize = csize;
2443 return len;
2444}
2445
2446static struct md_sysfs_entry bitmap_chunksize =
2447__ATTR(chunksize, S_IRUGO|S_IWUSR, chunksize_show, chunksize_store);
2448
2449static ssize_t metadata_show(struct mddev *mddev, char *page)
2450{
2451 if (mddev_is_clustered(mddev))
2452 return sprintf(page, "clustered\n");
2453 return sprintf(page, "%s\n", (mddev->bitmap_info.external
2454 ? "external" : "internal"));
2455}
2456
2457static ssize_t metadata_store(struct mddev *mddev, const char *buf, size_t len)
2458{
2459 if (mddev->bitmap ||
2460 mddev->bitmap_info.file ||
2461 mddev->bitmap_info.offset)
2462 return -EBUSY;
2463 if (strncmp(buf, "external", 8) == 0)
2464 mddev->bitmap_info.external = 1;
2465 else if ((strncmp(buf, "internal", 8) == 0) ||
2466 (strncmp(buf, "clustered", 9) == 0))
2467 mddev->bitmap_info.external = 0;
2468 else
2469 return -EINVAL;
2470 return len;
2471}
2472
2473static struct md_sysfs_entry bitmap_metadata =
2474__ATTR(metadata, S_IRUGO|S_IWUSR, metadata_show, metadata_store);
2475
2476static ssize_t can_clear_show(struct mddev *mddev, char *page)
2477{
2478 int len;
2479 spin_lock(&mddev->lock);
2480 if (mddev->bitmap)
2481 len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ?
2482 "false" : "true"));
2483 else
2484 len = sprintf(page, "\n");
2485 spin_unlock(&mddev->lock);
2486 return len;
2487}
2488
2489static ssize_t can_clear_store(struct mddev *mddev, const char *buf, size_t len)
2490{
2491 if (mddev->bitmap == NULL)
2492 return -ENOENT;
2493 if (strncmp(buf, "false", 5) == 0)
2494 mddev->bitmap->need_sync = 1;
2495 else if (strncmp(buf, "true", 4) == 0) {
2496 if (mddev->degraded)
2497 return -EBUSY;
2498 mddev->bitmap->need_sync = 0;
2499 } else
2500 return -EINVAL;
2501 return len;
2502}
2503
2504static struct md_sysfs_entry bitmap_can_clear =
2505__ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);
2506
2507static ssize_t
2508behind_writes_used_show(struct mddev *mddev, char *page)
2509{
2510 ssize_t ret;
2511 spin_lock(&mddev->lock);
2512 if (mddev->bitmap == NULL)
2513 ret = sprintf(page, "0\n");
2514 else
2515 ret = sprintf(page, "%lu\n",
2516 mddev->bitmap->behind_writes_used);
2517 spin_unlock(&mddev->lock);
2518 return ret;
2519}
2520
2521static ssize_t
2522behind_writes_used_reset(struct mddev *mddev, const char *buf, size_t len)
2523{
2524 if (mddev->bitmap)
2525 mddev->bitmap->behind_writes_used = 0;
2526 return len;
2527}
2528
2529static struct md_sysfs_entry max_backlog_used =
2530__ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
2531 behind_writes_used_show, behind_writes_used_reset);
2532
2533static struct attribute *md_bitmap_attrs[] = {
2534 &bitmap_location.attr,
2535 &bitmap_space.attr,
2536 &bitmap_timeout.attr,
2537 &bitmap_backlog.attr,
2538 &bitmap_chunksize.attr,
2539 &bitmap_metadata.attr,
2540 &bitmap_can_clear.attr,
2541 &max_backlog_used.attr,
2542 NULL
2543};
2544struct attribute_group md_bitmap_group = {
2545 .name = "bitmap",
2546 .attrs = md_bitmap_attrs,
2547};
2548