Loading...
1/*
2 * Copyright (C) 2007 Oracle. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18#include <linux/sched.h>
19#include <linux/bio.h>
20#include <linux/slab.h>
21#include <linux/buffer_head.h>
22#include <linux/blkdev.h>
23#include <linux/random.h>
24#include <linux/iocontext.h>
25#include <linux/capability.h>
26#include <asm/div64.h>
27#include "compat.h"
28#include "ctree.h"
29#include "extent_map.h"
30#include "disk-io.h"
31#include "transaction.h"
32#include "print-tree.h"
33#include "volumes.h"
34#include "async-thread.h"
35
36static int init_first_rw_device(struct btrfs_trans_handle *trans,
37 struct btrfs_root *root,
38 struct btrfs_device *device);
39static int btrfs_relocate_sys_chunks(struct btrfs_root *root);
40
41static DEFINE_MUTEX(uuid_mutex);
42static LIST_HEAD(fs_uuids);
43
44static void lock_chunks(struct btrfs_root *root)
45{
46 mutex_lock(&root->fs_info->chunk_mutex);
47}
48
49static void unlock_chunks(struct btrfs_root *root)
50{
51 mutex_unlock(&root->fs_info->chunk_mutex);
52}
53
54static void free_fs_devices(struct btrfs_fs_devices *fs_devices)
55{
56 struct btrfs_device *device;
57 WARN_ON(fs_devices->opened);
58 while (!list_empty(&fs_devices->devices)) {
59 device = list_entry(fs_devices->devices.next,
60 struct btrfs_device, dev_list);
61 list_del(&device->dev_list);
62 kfree(device->name);
63 kfree(device);
64 }
65 kfree(fs_devices);
66}
67
68int btrfs_cleanup_fs_uuids(void)
69{
70 struct btrfs_fs_devices *fs_devices;
71
72 while (!list_empty(&fs_uuids)) {
73 fs_devices = list_entry(fs_uuids.next,
74 struct btrfs_fs_devices, list);
75 list_del(&fs_devices->list);
76 free_fs_devices(fs_devices);
77 }
78 return 0;
79}
80
81static noinline struct btrfs_device *__find_device(struct list_head *head,
82 u64 devid, u8 *uuid)
83{
84 struct btrfs_device *dev;
85
86 list_for_each_entry(dev, head, dev_list) {
87 if (dev->devid == devid &&
88 (!uuid || !memcmp(dev->uuid, uuid, BTRFS_UUID_SIZE))) {
89 return dev;
90 }
91 }
92 return NULL;
93}
94
95static noinline struct btrfs_fs_devices *find_fsid(u8 *fsid)
96{
97 struct btrfs_fs_devices *fs_devices;
98
99 list_for_each_entry(fs_devices, &fs_uuids, list) {
100 if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) == 0)
101 return fs_devices;
102 }
103 return NULL;
104}
105
106static void requeue_list(struct btrfs_pending_bios *pending_bios,
107 struct bio *head, struct bio *tail)
108{
109
110 struct bio *old_head;
111
112 old_head = pending_bios->head;
113 pending_bios->head = head;
114 if (pending_bios->tail)
115 tail->bi_next = old_head;
116 else
117 pending_bios->tail = tail;
118}
119
120/*
121 * we try to collect pending bios for a device so we don't get a large
122 * number of procs sending bios down to the same device. This greatly
123 * improves the schedulers ability to collect and merge the bios.
124 *
125 * But, it also turns into a long list of bios to process and that is sure
126 * to eventually make the worker thread block. The solution here is to
127 * make some progress and then put this work struct back at the end of
128 * the list if the block device is congested. This way, multiple devices
129 * can make progress from a single worker thread.
130 */
131static noinline int run_scheduled_bios(struct btrfs_device *device)
132{
133 struct bio *pending;
134 struct backing_dev_info *bdi;
135 struct btrfs_fs_info *fs_info;
136 struct btrfs_pending_bios *pending_bios;
137 struct bio *tail;
138 struct bio *cur;
139 int again = 0;
140 unsigned long num_run;
141 unsigned long batch_run = 0;
142 unsigned long limit;
143 unsigned long last_waited = 0;
144 int force_reg = 0;
145 int sync_pending = 0;
146 struct blk_plug plug;
147
148 /*
149 * this function runs all the bios we've collected for
150 * a particular device. We don't want to wander off to
151 * another device without first sending all of these down.
152 * So, setup a plug here and finish it off before we return
153 */
154 blk_start_plug(&plug);
155
156 bdi = blk_get_backing_dev_info(device->bdev);
157 fs_info = device->dev_root->fs_info;
158 limit = btrfs_async_submit_limit(fs_info);
159 limit = limit * 2 / 3;
160
161loop:
162 spin_lock(&device->io_lock);
163
164loop_lock:
165 num_run = 0;
166
167 /* take all the bios off the list at once and process them
168 * later on (without the lock held). But, remember the
169 * tail and other pointers so the bios can be properly reinserted
170 * into the list if we hit congestion
171 */
172 if (!force_reg && device->pending_sync_bios.head) {
173 pending_bios = &device->pending_sync_bios;
174 force_reg = 1;
175 } else {
176 pending_bios = &device->pending_bios;
177 force_reg = 0;
178 }
179
180 pending = pending_bios->head;
181 tail = pending_bios->tail;
182 WARN_ON(pending && !tail);
183
184 /*
185 * if pending was null this time around, no bios need processing
186 * at all and we can stop. Otherwise it'll loop back up again
187 * and do an additional check so no bios are missed.
188 *
189 * device->running_pending is used to synchronize with the
190 * schedule_bio code.
191 */
192 if (device->pending_sync_bios.head == NULL &&
193 device->pending_bios.head == NULL) {
194 again = 0;
195 device->running_pending = 0;
196 } else {
197 again = 1;
198 device->running_pending = 1;
199 }
200
201 pending_bios->head = NULL;
202 pending_bios->tail = NULL;
203
204 spin_unlock(&device->io_lock);
205
206 while (pending) {
207
208 rmb();
209 /* we want to work on both lists, but do more bios on the
210 * sync list than the regular list
211 */
212 if ((num_run > 32 &&
213 pending_bios != &device->pending_sync_bios &&
214 device->pending_sync_bios.head) ||
215 (num_run > 64 && pending_bios == &device->pending_sync_bios &&
216 device->pending_bios.head)) {
217 spin_lock(&device->io_lock);
218 requeue_list(pending_bios, pending, tail);
219 goto loop_lock;
220 }
221
222 cur = pending;
223 pending = pending->bi_next;
224 cur->bi_next = NULL;
225 atomic_dec(&fs_info->nr_async_bios);
226
227 if (atomic_read(&fs_info->nr_async_bios) < limit &&
228 waitqueue_active(&fs_info->async_submit_wait))
229 wake_up(&fs_info->async_submit_wait);
230
231 BUG_ON(atomic_read(&cur->bi_cnt) == 0);
232
233 /*
234 * if we're doing the sync list, record that our
235 * plug has some sync requests on it
236 *
237 * If we're doing the regular list and there are
238 * sync requests sitting around, unplug before
239 * we add more
240 */
241 if (pending_bios == &device->pending_sync_bios) {
242 sync_pending = 1;
243 } else if (sync_pending) {
244 blk_finish_plug(&plug);
245 blk_start_plug(&plug);
246 sync_pending = 0;
247 }
248
249 submit_bio(cur->bi_rw, cur);
250 num_run++;
251 batch_run++;
252 if (need_resched())
253 cond_resched();
254
255 /*
256 * we made progress, there is more work to do and the bdi
257 * is now congested. Back off and let other work structs
258 * run instead
259 */
260 if (pending && bdi_write_congested(bdi) && batch_run > 8 &&
261 fs_info->fs_devices->open_devices > 1) {
262 struct io_context *ioc;
263
264 ioc = current->io_context;
265
266 /*
267 * the main goal here is that we don't want to
268 * block if we're going to be able to submit
269 * more requests without blocking.
270 *
271 * This code does two great things, it pokes into
272 * the elevator code from a filesystem _and_
273 * it makes assumptions about how batching works.
274 */
275 if (ioc && ioc->nr_batch_requests > 0 &&
276 time_before(jiffies, ioc->last_waited + HZ/50UL) &&
277 (last_waited == 0 ||
278 ioc->last_waited == last_waited)) {
279 /*
280 * we want to go through our batch of
281 * requests and stop. So, we copy out
282 * the ioc->last_waited time and test
283 * against it before looping
284 */
285 last_waited = ioc->last_waited;
286 if (need_resched())
287 cond_resched();
288 continue;
289 }
290 spin_lock(&device->io_lock);
291 requeue_list(pending_bios, pending, tail);
292 device->running_pending = 1;
293
294 spin_unlock(&device->io_lock);
295 btrfs_requeue_work(&device->work);
296 goto done;
297 }
298 }
299
300 cond_resched();
301 if (again)
302 goto loop;
303
304 spin_lock(&device->io_lock);
305 if (device->pending_bios.head || device->pending_sync_bios.head)
306 goto loop_lock;
307 spin_unlock(&device->io_lock);
308
309done:
310 blk_finish_plug(&plug);
311 return 0;
312}
313
314static void pending_bios_fn(struct btrfs_work *work)
315{
316 struct btrfs_device *device;
317
318 device = container_of(work, struct btrfs_device, work);
319 run_scheduled_bios(device);
320}
321
322static noinline int device_list_add(const char *path,
323 struct btrfs_super_block *disk_super,
324 u64 devid, struct btrfs_fs_devices **fs_devices_ret)
325{
326 struct btrfs_device *device;
327 struct btrfs_fs_devices *fs_devices;
328 u64 found_transid = btrfs_super_generation(disk_super);
329 char *name;
330
331 fs_devices = find_fsid(disk_super->fsid);
332 if (!fs_devices) {
333 fs_devices = kzalloc(sizeof(*fs_devices), GFP_NOFS);
334 if (!fs_devices)
335 return -ENOMEM;
336 INIT_LIST_HEAD(&fs_devices->devices);
337 INIT_LIST_HEAD(&fs_devices->alloc_list);
338 list_add(&fs_devices->list, &fs_uuids);
339 memcpy(fs_devices->fsid, disk_super->fsid, BTRFS_FSID_SIZE);
340 fs_devices->latest_devid = devid;
341 fs_devices->latest_trans = found_transid;
342 mutex_init(&fs_devices->device_list_mutex);
343 device = NULL;
344 } else {
345 device = __find_device(&fs_devices->devices, devid,
346 disk_super->dev_item.uuid);
347 }
348 if (!device) {
349 if (fs_devices->opened)
350 return -EBUSY;
351
352 device = kzalloc(sizeof(*device), GFP_NOFS);
353 if (!device) {
354 /* we can safely leave the fs_devices entry around */
355 return -ENOMEM;
356 }
357 device->devid = devid;
358 device->work.func = pending_bios_fn;
359 memcpy(device->uuid, disk_super->dev_item.uuid,
360 BTRFS_UUID_SIZE);
361 spin_lock_init(&device->io_lock);
362 device->name = kstrdup(path, GFP_NOFS);
363 if (!device->name) {
364 kfree(device);
365 return -ENOMEM;
366 }
367 INIT_LIST_HEAD(&device->dev_alloc_list);
368
369 mutex_lock(&fs_devices->device_list_mutex);
370 list_add_rcu(&device->dev_list, &fs_devices->devices);
371 mutex_unlock(&fs_devices->device_list_mutex);
372
373 device->fs_devices = fs_devices;
374 fs_devices->num_devices++;
375 } else if (!device->name || strcmp(device->name, path)) {
376 name = kstrdup(path, GFP_NOFS);
377 if (!name)
378 return -ENOMEM;
379 kfree(device->name);
380 device->name = name;
381 if (device->missing) {
382 fs_devices->missing_devices--;
383 device->missing = 0;
384 }
385 }
386
387 if (found_transid > fs_devices->latest_trans) {
388 fs_devices->latest_devid = devid;
389 fs_devices->latest_trans = found_transid;
390 }
391 *fs_devices_ret = fs_devices;
392 return 0;
393}
394
395static struct btrfs_fs_devices *clone_fs_devices(struct btrfs_fs_devices *orig)
396{
397 struct btrfs_fs_devices *fs_devices;
398 struct btrfs_device *device;
399 struct btrfs_device *orig_dev;
400
401 fs_devices = kzalloc(sizeof(*fs_devices), GFP_NOFS);
402 if (!fs_devices)
403 return ERR_PTR(-ENOMEM);
404
405 INIT_LIST_HEAD(&fs_devices->devices);
406 INIT_LIST_HEAD(&fs_devices->alloc_list);
407 INIT_LIST_HEAD(&fs_devices->list);
408 mutex_init(&fs_devices->device_list_mutex);
409 fs_devices->latest_devid = orig->latest_devid;
410 fs_devices->latest_trans = orig->latest_trans;
411 memcpy(fs_devices->fsid, orig->fsid, sizeof(fs_devices->fsid));
412
413 /* We have held the volume lock, it is safe to get the devices. */
414 list_for_each_entry(orig_dev, &orig->devices, dev_list) {
415 device = kzalloc(sizeof(*device), GFP_NOFS);
416 if (!device)
417 goto error;
418
419 device->name = kstrdup(orig_dev->name, GFP_NOFS);
420 if (!device->name) {
421 kfree(device);
422 goto error;
423 }
424
425 device->devid = orig_dev->devid;
426 device->work.func = pending_bios_fn;
427 memcpy(device->uuid, orig_dev->uuid, sizeof(device->uuid));
428 spin_lock_init(&device->io_lock);
429 INIT_LIST_HEAD(&device->dev_list);
430 INIT_LIST_HEAD(&device->dev_alloc_list);
431
432 list_add(&device->dev_list, &fs_devices->devices);
433 device->fs_devices = fs_devices;
434 fs_devices->num_devices++;
435 }
436 return fs_devices;
437error:
438 free_fs_devices(fs_devices);
439 return ERR_PTR(-ENOMEM);
440}
441
442int btrfs_close_extra_devices(struct btrfs_fs_devices *fs_devices)
443{
444 struct btrfs_device *device, *next;
445
446 mutex_lock(&uuid_mutex);
447again:
448 /* This is the initialized path, it is safe to release the devices. */
449 list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) {
450 if (device->in_fs_metadata)
451 continue;
452
453 if (device->bdev) {
454 blkdev_put(device->bdev, device->mode);
455 device->bdev = NULL;
456 fs_devices->open_devices--;
457 }
458 if (device->writeable) {
459 list_del_init(&device->dev_alloc_list);
460 device->writeable = 0;
461 fs_devices->rw_devices--;
462 }
463 list_del_init(&device->dev_list);
464 fs_devices->num_devices--;
465 kfree(device->name);
466 kfree(device);
467 }
468
469 if (fs_devices->seed) {
470 fs_devices = fs_devices->seed;
471 goto again;
472 }
473
474 mutex_unlock(&uuid_mutex);
475 return 0;
476}
477
478static void __free_device(struct work_struct *work)
479{
480 struct btrfs_device *device;
481
482 device = container_of(work, struct btrfs_device, rcu_work);
483
484 if (device->bdev)
485 blkdev_put(device->bdev, device->mode);
486
487 kfree(device->name);
488 kfree(device);
489}
490
491static void free_device(struct rcu_head *head)
492{
493 struct btrfs_device *device;
494
495 device = container_of(head, struct btrfs_device, rcu);
496
497 INIT_WORK(&device->rcu_work, __free_device);
498 schedule_work(&device->rcu_work);
499}
500
501static int __btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
502{
503 struct btrfs_device *device;
504
505 if (--fs_devices->opened > 0)
506 return 0;
507
508 mutex_lock(&fs_devices->device_list_mutex);
509 list_for_each_entry(device, &fs_devices->devices, dev_list) {
510 struct btrfs_device *new_device;
511
512 if (device->bdev)
513 fs_devices->open_devices--;
514
515 if (device->writeable) {
516 list_del_init(&device->dev_alloc_list);
517 fs_devices->rw_devices--;
518 }
519
520 if (device->can_discard)
521 fs_devices->num_can_discard--;
522
523 new_device = kmalloc(sizeof(*new_device), GFP_NOFS);
524 BUG_ON(!new_device);
525 memcpy(new_device, device, sizeof(*new_device));
526 new_device->name = kstrdup(device->name, GFP_NOFS);
527 BUG_ON(device->name && !new_device->name);
528 new_device->bdev = NULL;
529 new_device->writeable = 0;
530 new_device->in_fs_metadata = 0;
531 new_device->can_discard = 0;
532 list_replace_rcu(&device->dev_list, &new_device->dev_list);
533
534 call_rcu(&device->rcu, free_device);
535 }
536 mutex_unlock(&fs_devices->device_list_mutex);
537
538 WARN_ON(fs_devices->open_devices);
539 WARN_ON(fs_devices->rw_devices);
540 fs_devices->opened = 0;
541 fs_devices->seeding = 0;
542
543 return 0;
544}
545
546int btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
547{
548 struct btrfs_fs_devices *seed_devices = NULL;
549 int ret;
550
551 mutex_lock(&uuid_mutex);
552 ret = __btrfs_close_devices(fs_devices);
553 if (!fs_devices->opened) {
554 seed_devices = fs_devices->seed;
555 fs_devices->seed = NULL;
556 }
557 mutex_unlock(&uuid_mutex);
558
559 while (seed_devices) {
560 fs_devices = seed_devices;
561 seed_devices = fs_devices->seed;
562 __btrfs_close_devices(fs_devices);
563 free_fs_devices(fs_devices);
564 }
565 return ret;
566}
567
568static int __btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
569 fmode_t flags, void *holder)
570{
571 struct request_queue *q;
572 struct block_device *bdev;
573 struct list_head *head = &fs_devices->devices;
574 struct btrfs_device *device;
575 struct block_device *latest_bdev = NULL;
576 struct buffer_head *bh;
577 struct btrfs_super_block *disk_super;
578 u64 latest_devid = 0;
579 u64 latest_transid = 0;
580 u64 devid;
581 int seeding = 1;
582 int ret = 0;
583
584 flags |= FMODE_EXCL;
585
586 list_for_each_entry(device, head, dev_list) {
587 if (device->bdev)
588 continue;
589 if (!device->name)
590 continue;
591
592 bdev = blkdev_get_by_path(device->name, flags, holder);
593 if (IS_ERR(bdev)) {
594 printk(KERN_INFO "open %s failed\n", device->name);
595 goto error;
596 }
597 set_blocksize(bdev, 4096);
598
599 bh = btrfs_read_dev_super(bdev);
600 if (!bh) {
601 ret = -EINVAL;
602 goto error_close;
603 }
604
605 disk_super = (struct btrfs_super_block *)bh->b_data;
606 devid = btrfs_stack_device_id(&disk_super->dev_item);
607 if (devid != device->devid)
608 goto error_brelse;
609
610 if (memcmp(device->uuid, disk_super->dev_item.uuid,
611 BTRFS_UUID_SIZE))
612 goto error_brelse;
613
614 device->generation = btrfs_super_generation(disk_super);
615 if (!latest_transid || device->generation > latest_transid) {
616 latest_devid = devid;
617 latest_transid = device->generation;
618 latest_bdev = bdev;
619 }
620
621 if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_SEEDING) {
622 device->writeable = 0;
623 } else {
624 device->writeable = !bdev_read_only(bdev);
625 seeding = 0;
626 }
627
628 q = bdev_get_queue(bdev);
629 if (blk_queue_discard(q)) {
630 device->can_discard = 1;
631 fs_devices->num_can_discard++;
632 }
633
634 device->bdev = bdev;
635 device->in_fs_metadata = 0;
636 device->mode = flags;
637
638 if (!blk_queue_nonrot(bdev_get_queue(bdev)))
639 fs_devices->rotating = 1;
640
641 fs_devices->open_devices++;
642 if (device->writeable) {
643 fs_devices->rw_devices++;
644 list_add(&device->dev_alloc_list,
645 &fs_devices->alloc_list);
646 }
647 brelse(bh);
648 continue;
649
650error_brelse:
651 brelse(bh);
652error_close:
653 blkdev_put(bdev, flags);
654error:
655 continue;
656 }
657 if (fs_devices->open_devices == 0) {
658 ret = -EIO;
659 goto out;
660 }
661 fs_devices->seeding = seeding;
662 fs_devices->opened = 1;
663 fs_devices->latest_bdev = latest_bdev;
664 fs_devices->latest_devid = latest_devid;
665 fs_devices->latest_trans = latest_transid;
666 fs_devices->total_rw_bytes = 0;
667out:
668 return ret;
669}
670
671int btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
672 fmode_t flags, void *holder)
673{
674 int ret;
675
676 mutex_lock(&uuid_mutex);
677 if (fs_devices->opened) {
678 fs_devices->opened++;
679 ret = 0;
680 } else {
681 ret = __btrfs_open_devices(fs_devices, flags, holder);
682 }
683 mutex_unlock(&uuid_mutex);
684 return ret;
685}
686
687int btrfs_scan_one_device(const char *path, fmode_t flags, void *holder,
688 struct btrfs_fs_devices **fs_devices_ret)
689{
690 struct btrfs_super_block *disk_super;
691 struct block_device *bdev;
692 struct buffer_head *bh;
693 int ret;
694 u64 devid;
695 u64 transid;
696
697 mutex_lock(&uuid_mutex);
698
699 flags |= FMODE_EXCL;
700 bdev = blkdev_get_by_path(path, flags, holder);
701
702 if (IS_ERR(bdev)) {
703 ret = PTR_ERR(bdev);
704 goto error;
705 }
706
707 ret = set_blocksize(bdev, 4096);
708 if (ret)
709 goto error_close;
710 bh = btrfs_read_dev_super(bdev);
711 if (!bh) {
712 ret = -EINVAL;
713 goto error_close;
714 }
715 disk_super = (struct btrfs_super_block *)bh->b_data;
716 devid = btrfs_stack_device_id(&disk_super->dev_item);
717 transid = btrfs_super_generation(disk_super);
718 if (disk_super->label[0])
719 printk(KERN_INFO "device label %s ", disk_super->label);
720 else
721 printk(KERN_INFO "device fsid %pU ", disk_super->fsid);
722 printk(KERN_CONT "devid %llu transid %llu %s\n",
723 (unsigned long long)devid, (unsigned long long)transid, path);
724 ret = device_list_add(path, disk_super, devid, fs_devices_ret);
725
726 brelse(bh);
727error_close:
728 blkdev_put(bdev, flags);
729error:
730 mutex_unlock(&uuid_mutex);
731 return ret;
732}
733
734/* helper to account the used device space in the range */
735int btrfs_account_dev_extents_size(struct btrfs_device *device, u64 start,
736 u64 end, u64 *length)
737{
738 struct btrfs_key key;
739 struct btrfs_root *root = device->dev_root;
740 struct btrfs_dev_extent *dev_extent;
741 struct btrfs_path *path;
742 u64 extent_end;
743 int ret;
744 int slot;
745 struct extent_buffer *l;
746
747 *length = 0;
748
749 if (start >= device->total_bytes)
750 return 0;
751
752 path = btrfs_alloc_path();
753 if (!path)
754 return -ENOMEM;
755 path->reada = 2;
756
757 key.objectid = device->devid;
758 key.offset = start;
759 key.type = BTRFS_DEV_EXTENT_KEY;
760
761 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
762 if (ret < 0)
763 goto out;
764 if (ret > 0) {
765 ret = btrfs_previous_item(root, path, key.objectid, key.type);
766 if (ret < 0)
767 goto out;
768 }
769
770 while (1) {
771 l = path->nodes[0];
772 slot = path->slots[0];
773 if (slot >= btrfs_header_nritems(l)) {
774 ret = btrfs_next_leaf(root, path);
775 if (ret == 0)
776 continue;
777 if (ret < 0)
778 goto out;
779
780 break;
781 }
782 btrfs_item_key_to_cpu(l, &key, slot);
783
784 if (key.objectid < device->devid)
785 goto next;
786
787 if (key.objectid > device->devid)
788 break;
789
790 if (btrfs_key_type(&key) != BTRFS_DEV_EXTENT_KEY)
791 goto next;
792
793 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
794 extent_end = key.offset + btrfs_dev_extent_length(l,
795 dev_extent);
796 if (key.offset <= start && extent_end > end) {
797 *length = end - start + 1;
798 break;
799 } else if (key.offset <= start && extent_end > start)
800 *length += extent_end - start;
801 else if (key.offset > start && extent_end <= end)
802 *length += extent_end - key.offset;
803 else if (key.offset > start && key.offset <= end) {
804 *length += end - key.offset + 1;
805 break;
806 } else if (key.offset > end)
807 break;
808
809next:
810 path->slots[0]++;
811 }
812 ret = 0;
813out:
814 btrfs_free_path(path);
815 return ret;
816}
817
818/*
819 * find_free_dev_extent - find free space in the specified device
820 * @trans: transaction handler
821 * @device: the device which we search the free space in
822 * @num_bytes: the size of the free space that we need
823 * @start: store the start of the free space.
824 * @len: the size of the free space. that we find, or the size of the max
825 * free space if we don't find suitable free space
826 *
827 * this uses a pretty simple search, the expectation is that it is
828 * called very infrequently and that a given device has a small number
829 * of extents
830 *
831 * @start is used to store the start of the free space if we find. But if we
832 * don't find suitable free space, it will be used to store the start position
833 * of the max free space.
834 *
835 * @len is used to store the size of the free space that we find.
836 * But if we don't find suitable free space, it is used to store the size of
837 * the max free space.
838 */
839int find_free_dev_extent(struct btrfs_trans_handle *trans,
840 struct btrfs_device *device, u64 num_bytes,
841 u64 *start, u64 *len)
842{
843 struct btrfs_key key;
844 struct btrfs_root *root = device->dev_root;
845 struct btrfs_dev_extent *dev_extent;
846 struct btrfs_path *path;
847 u64 hole_size;
848 u64 max_hole_start;
849 u64 max_hole_size;
850 u64 extent_end;
851 u64 search_start;
852 u64 search_end = device->total_bytes;
853 int ret;
854 int slot;
855 struct extent_buffer *l;
856
857 /* FIXME use last free of some kind */
858
859 /* we don't want to overwrite the superblock on the drive,
860 * so we make sure to start at an offset of at least 1MB
861 */
862 search_start = max(root->fs_info->alloc_start, 1024ull * 1024);
863
864 max_hole_start = search_start;
865 max_hole_size = 0;
866 hole_size = 0;
867
868 if (search_start >= search_end) {
869 ret = -ENOSPC;
870 goto error;
871 }
872
873 path = btrfs_alloc_path();
874 if (!path) {
875 ret = -ENOMEM;
876 goto error;
877 }
878 path->reada = 2;
879
880 key.objectid = device->devid;
881 key.offset = search_start;
882 key.type = BTRFS_DEV_EXTENT_KEY;
883
884 ret = btrfs_search_slot(trans, root, &key, path, 0, 0);
885 if (ret < 0)
886 goto out;
887 if (ret > 0) {
888 ret = btrfs_previous_item(root, path, key.objectid, key.type);
889 if (ret < 0)
890 goto out;
891 }
892
893 while (1) {
894 l = path->nodes[0];
895 slot = path->slots[0];
896 if (slot >= btrfs_header_nritems(l)) {
897 ret = btrfs_next_leaf(root, path);
898 if (ret == 0)
899 continue;
900 if (ret < 0)
901 goto out;
902
903 break;
904 }
905 btrfs_item_key_to_cpu(l, &key, slot);
906
907 if (key.objectid < device->devid)
908 goto next;
909
910 if (key.objectid > device->devid)
911 break;
912
913 if (btrfs_key_type(&key) != BTRFS_DEV_EXTENT_KEY)
914 goto next;
915
916 if (key.offset > search_start) {
917 hole_size = key.offset - search_start;
918
919 if (hole_size > max_hole_size) {
920 max_hole_start = search_start;
921 max_hole_size = hole_size;
922 }
923
924 /*
925 * If this free space is greater than which we need,
926 * it must be the max free space that we have found
927 * until now, so max_hole_start must point to the start
928 * of this free space and the length of this free space
929 * is stored in max_hole_size. Thus, we return
930 * max_hole_start and max_hole_size and go back to the
931 * caller.
932 */
933 if (hole_size >= num_bytes) {
934 ret = 0;
935 goto out;
936 }
937 }
938
939 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
940 extent_end = key.offset + btrfs_dev_extent_length(l,
941 dev_extent);
942 if (extent_end > search_start)
943 search_start = extent_end;
944next:
945 path->slots[0]++;
946 cond_resched();
947 }
948
949 /*
950 * At this point, search_start should be the end of
951 * allocated dev extents, and when shrinking the device,
952 * search_end may be smaller than search_start.
953 */
954 if (search_end > search_start)
955 hole_size = search_end - search_start;
956
957 if (hole_size > max_hole_size) {
958 max_hole_start = search_start;
959 max_hole_size = hole_size;
960 }
961
962 /* See above. */
963 if (hole_size < num_bytes)
964 ret = -ENOSPC;
965 else
966 ret = 0;
967
968out:
969 btrfs_free_path(path);
970error:
971 *start = max_hole_start;
972 if (len)
973 *len = max_hole_size;
974 return ret;
975}
976
977static int btrfs_free_dev_extent(struct btrfs_trans_handle *trans,
978 struct btrfs_device *device,
979 u64 start)
980{
981 int ret;
982 struct btrfs_path *path;
983 struct btrfs_root *root = device->dev_root;
984 struct btrfs_key key;
985 struct btrfs_key found_key;
986 struct extent_buffer *leaf = NULL;
987 struct btrfs_dev_extent *extent = NULL;
988
989 path = btrfs_alloc_path();
990 if (!path)
991 return -ENOMEM;
992
993 key.objectid = device->devid;
994 key.offset = start;
995 key.type = BTRFS_DEV_EXTENT_KEY;
996
997 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
998 if (ret > 0) {
999 ret = btrfs_previous_item(root, path, key.objectid,
1000 BTRFS_DEV_EXTENT_KEY);
1001 if (ret)
1002 goto out;
1003 leaf = path->nodes[0];
1004 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1005 extent = btrfs_item_ptr(leaf, path->slots[0],
1006 struct btrfs_dev_extent);
1007 BUG_ON(found_key.offset > start || found_key.offset +
1008 btrfs_dev_extent_length(leaf, extent) < start);
1009 } else if (ret == 0) {
1010 leaf = path->nodes[0];
1011 extent = btrfs_item_ptr(leaf, path->slots[0],
1012 struct btrfs_dev_extent);
1013 }
1014 BUG_ON(ret);
1015
1016 if (device->bytes_used > 0)
1017 device->bytes_used -= btrfs_dev_extent_length(leaf, extent);
1018 ret = btrfs_del_item(trans, root, path);
1019
1020out:
1021 btrfs_free_path(path);
1022 return ret;
1023}
1024
1025int btrfs_alloc_dev_extent(struct btrfs_trans_handle *trans,
1026 struct btrfs_device *device,
1027 u64 chunk_tree, u64 chunk_objectid,
1028 u64 chunk_offset, u64 start, u64 num_bytes)
1029{
1030 int ret;
1031 struct btrfs_path *path;
1032 struct btrfs_root *root = device->dev_root;
1033 struct btrfs_dev_extent *extent;
1034 struct extent_buffer *leaf;
1035 struct btrfs_key key;
1036
1037 WARN_ON(!device->in_fs_metadata);
1038 path = btrfs_alloc_path();
1039 if (!path)
1040 return -ENOMEM;
1041
1042 key.objectid = device->devid;
1043 key.offset = start;
1044 key.type = BTRFS_DEV_EXTENT_KEY;
1045 ret = btrfs_insert_empty_item(trans, root, path, &key,
1046 sizeof(*extent));
1047 BUG_ON(ret);
1048
1049 leaf = path->nodes[0];
1050 extent = btrfs_item_ptr(leaf, path->slots[0],
1051 struct btrfs_dev_extent);
1052 btrfs_set_dev_extent_chunk_tree(leaf, extent, chunk_tree);
1053 btrfs_set_dev_extent_chunk_objectid(leaf, extent, chunk_objectid);
1054 btrfs_set_dev_extent_chunk_offset(leaf, extent, chunk_offset);
1055
1056 write_extent_buffer(leaf, root->fs_info->chunk_tree_uuid,
1057 (unsigned long)btrfs_dev_extent_chunk_tree_uuid(extent),
1058 BTRFS_UUID_SIZE);
1059
1060 btrfs_set_dev_extent_length(leaf, extent, num_bytes);
1061 btrfs_mark_buffer_dirty(leaf);
1062 btrfs_free_path(path);
1063 return ret;
1064}
1065
1066static noinline int find_next_chunk(struct btrfs_root *root,
1067 u64 objectid, u64 *offset)
1068{
1069 struct btrfs_path *path;
1070 int ret;
1071 struct btrfs_key key;
1072 struct btrfs_chunk *chunk;
1073 struct btrfs_key found_key;
1074
1075 path = btrfs_alloc_path();
1076 if (!path)
1077 return -ENOMEM;
1078
1079 key.objectid = objectid;
1080 key.offset = (u64)-1;
1081 key.type = BTRFS_CHUNK_ITEM_KEY;
1082
1083 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1084 if (ret < 0)
1085 goto error;
1086
1087 BUG_ON(ret == 0);
1088
1089 ret = btrfs_previous_item(root, path, 0, BTRFS_CHUNK_ITEM_KEY);
1090 if (ret) {
1091 *offset = 0;
1092 } else {
1093 btrfs_item_key_to_cpu(path->nodes[0], &found_key,
1094 path->slots[0]);
1095 if (found_key.objectid != objectid)
1096 *offset = 0;
1097 else {
1098 chunk = btrfs_item_ptr(path->nodes[0], path->slots[0],
1099 struct btrfs_chunk);
1100 *offset = found_key.offset +
1101 btrfs_chunk_length(path->nodes[0], chunk);
1102 }
1103 }
1104 ret = 0;
1105error:
1106 btrfs_free_path(path);
1107 return ret;
1108}
1109
1110static noinline int find_next_devid(struct btrfs_root *root, u64 *objectid)
1111{
1112 int ret;
1113 struct btrfs_key key;
1114 struct btrfs_key found_key;
1115 struct btrfs_path *path;
1116
1117 root = root->fs_info->chunk_root;
1118
1119 path = btrfs_alloc_path();
1120 if (!path)
1121 return -ENOMEM;
1122
1123 key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1124 key.type = BTRFS_DEV_ITEM_KEY;
1125 key.offset = (u64)-1;
1126
1127 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1128 if (ret < 0)
1129 goto error;
1130
1131 BUG_ON(ret == 0);
1132
1133 ret = btrfs_previous_item(root, path, BTRFS_DEV_ITEMS_OBJECTID,
1134 BTRFS_DEV_ITEM_KEY);
1135 if (ret) {
1136 *objectid = 1;
1137 } else {
1138 btrfs_item_key_to_cpu(path->nodes[0], &found_key,
1139 path->slots[0]);
1140 *objectid = found_key.offset + 1;
1141 }
1142 ret = 0;
1143error:
1144 btrfs_free_path(path);
1145 return ret;
1146}
1147
1148/*
1149 * the device information is stored in the chunk root
1150 * the btrfs_device struct should be fully filled in
1151 */
1152int btrfs_add_device(struct btrfs_trans_handle *trans,
1153 struct btrfs_root *root,
1154 struct btrfs_device *device)
1155{
1156 int ret;
1157 struct btrfs_path *path;
1158 struct btrfs_dev_item *dev_item;
1159 struct extent_buffer *leaf;
1160 struct btrfs_key key;
1161 unsigned long ptr;
1162
1163 root = root->fs_info->chunk_root;
1164
1165 path = btrfs_alloc_path();
1166 if (!path)
1167 return -ENOMEM;
1168
1169 key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1170 key.type = BTRFS_DEV_ITEM_KEY;
1171 key.offset = device->devid;
1172
1173 ret = btrfs_insert_empty_item(trans, root, path, &key,
1174 sizeof(*dev_item));
1175 if (ret)
1176 goto out;
1177
1178 leaf = path->nodes[0];
1179 dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
1180
1181 btrfs_set_device_id(leaf, dev_item, device->devid);
1182 btrfs_set_device_generation(leaf, dev_item, 0);
1183 btrfs_set_device_type(leaf, dev_item, device->type);
1184 btrfs_set_device_io_align(leaf, dev_item, device->io_align);
1185 btrfs_set_device_io_width(leaf, dev_item, device->io_width);
1186 btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
1187 btrfs_set_device_total_bytes(leaf, dev_item, device->total_bytes);
1188 btrfs_set_device_bytes_used(leaf, dev_item, device->bytes_used);
1189 btrfs_set_device_group(leaf, dev_item, 0);
1190 btrfs_set_device_seek_speed(leaf, dev_item, 0);
1191 btrfs_set_device_bandwidth(leaf, dev_item, 0);
1192 btrfs_set_device_start_offset(leaf, dev_item, 0);
1193
1194 ptr = (unsigned long)btrfs_device_uuid(dev_item);
1195 write_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
1196 ptr = (unsigned long)btrfs_device_fsid(dev_item);
1197 write_extent_buffer(leaf, root->fs_info->fsid, ptr, BTRFS_UUID_SIZE);
1198 btrfs_mark_buffer_dirty(leaf);
1199
1200 ret = 0;
1201out:
1202 btrfs_free_path(path);
1203 return ret;
1204}
1205
1206static int btrfs_rm_dev_item(struct btrfs_root *root,
1207 struct btrfs_device *device)
1208{
1209 int ret;
1210 struct btrfs_path *path;
1211 struct btrfs_key key;
1212 struct btrfs_trans_handle *trans;
1213
1214 root = root->fs_info->chunk_root;
1215
1216 path = btrfs_alloc_path();
1217 if (!path)
1218 return -ENOMEM;
1219
1220 trans = btrfs_start_transaction(root, 0);
1221 if (IS_ERR(trans)) {
1222 btrfs_free_path(path);
1223 return PTR_ERR(trans);
1224 }
1225 key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1226 key.type = BTRFS_DEV_ITEM_KEY;
1227 key.offset = device->devid;
1228 lock_chunks(root);
1229
1230 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1231 if (ret < 0)
1232 goto out;
1233
1234 if (ret > 0) {
1235 ret = -ENOENT;
1236 goto out;
1237 }
1238
1239 ret = btrfs_del_item(trans, root, path);
1240 if (ret)
1241 goto out;
1242out:
1243 btrfs_free_path(path);
1244 unlock_chunks(root);
1245 btrfs_commit_transaction(trans, root);
1246 return ret;
1247}
1248
1249int btrfs_rm_device(struct btrfs_root *root, char *device_path)
1250{
1251 struct btrfs_device *device;
1252 struct btrfs_device *next_device;
1253 struct block_device *bdev;
1254 struct buffer_head *bh = NULL;
1255 struct btrfs_super_block *disk_super;
1256 struct btrfs_fs_devices *cur_devices;
1257 u64 all_avail;
1258 u64 devid;
1259 u64 num_devices;
1260 u8 *dev_uuid;
1261 int ret = 0;
1262 bool clear_super = false;
1263
1264 mutex_lock(&uuid_mutex);
1265 mutex_lock(&root->fs_info->volume_mutex);
1266
1267 all_avail = root->fs_info->avail_data_alloc_bits |
1268 root->fs_info->avail_system_alloc_bits |
1269 root->fs_info->avail_metadata_alloc_bits;
1270
1271 if ((all_avail & BTRFS_BLOCK_GROUP_RAID10) &&
1272 root->fs_info->fs_devices->num_devices <= 4) {
1273 printk(KERN_ERR "btrfs: unable to go below four devices "
1274 "on raid10\n");
1275 ret = -EINVAL;
1276 goto out;
1277 }
1278
1279 if ((all_avail & BTRFS_BLOCK_GROUP_RAID1) &&
1280 root->fs_info->fs_devices->num_devices <= 2) {
1281 printk(KERN_ERR "btrfs: unable to go below two "
1282 "devices on raid1\n");
1283 ret = -EINVAL;
1284 goto out;
1285 }
1286
1287 if (strcmp(device_path, "missing") == 0) {
1288 struct list_head *devices;
1289 struct btrfs_device *tmp;
1290
1291 device = NULL;
1292 devices = &root->fs_info->fs_devices->devices;
1293 /*
1294 * It is safe to read the devices since the volume_mutex
1295 * is held.
1296 */
1297 list_for_each_entry(tmp, devices, dev_list) {
1298 if (tmp->in_fs_metadata && !tmp->bdev) {
1299 device = tmp;
1300 break;
1301 }
1302 }
1303 bdev = NULL;
1304 bh = NULL;
1305 disk_super = NULL;
1306 if (!device) {
1307 printk(KERN_ERR "btrfs: no missing devices found to "
1308 "remove\n");
1309 goto out;
1310 }
1311 } else {
1312 bdev = blkdev_get_by_path(device_path, FMODE_READ | FMODE_EXCL,
1313 root->fs_info->bdev_holder);
1314 if (IS_ERR(bdev)) {
1315 ret = PTR_ERR(bdev);
1316 goto out;
1317 }
1318
1319 set_blocksize(bdev, 4096);
1320 bh = btrfs_read_dev_super(bdev);
1321 if (!bh) {
1322 ret = -EINVAL;
1323 goto error_close;
1324 }
1325 disk_super = (struct btrfs_super_block *)bh->b_data;
1326 devid = btrfs_stack_device_id(&disk_super->dev_item);
1327 dev_uuid = disk_super->dev_item.uuid;
1328 device = btrfs_find_device(root, devid, dev_uuid,
1329 disk_super->fsid);
1330 if (!device) {
1331 ret = -ENOENT;
1332 goto error_brelse;
1333 }
1334 }
1335
1336 if (device->writeable && root->fs_info->fs_devices->rw_devices == 1) {
1337 printk(KERN_ERR "btrfs: unable to remove the only writeable "
1338 "device\n");
1339 ret = -EINVAL;
1340 goto error_brelse;
1341 }
1342
1343 if (device->writeable) {
1344 lock_chunks(root);
1345 list_del_init(&device->dev_alloc_list);
1346 unlock_chunks(root);
1347 root->fs_info->fs_devices->rw_devices--;
1348 clear_super = true;
1349 }
1350
1351 ret = btrfs_shrink_device(device, 0);
1352 if (ret)
1353 goto error_undo;
1354
1355 ret = btrfs_rm_dev_item(root->fs_info->chunk_root, device);
1356 if (ret)
1357 goto error_undo;
1358
1359 device->in_fs_metadata = 0;
1360 btrfs_scrub_cancel_dev(root, device);
1361
1362 /*
1363 * the device list mutex makes sure that we don't change
1364 * the device list while someone else is writing out all
1365 * the device supers.
1366 */
1367
1368 cur_devices = device->fs_devices;
1369 mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
1370 list_del_rcu(&device->dev_list);
1371
1372 device->fs_devices->num_devices--;
1373
1374 if (device->missing)
1375 root->fs_info->fs_devices->missing_devices--;
1376
1377 next_device = list_entry(root->fs_info->fs_devices->devices.next,
1378 struct btrfs_device, dev_list);
1379 if (device->bdev == root->fs_info->sb->s_bdev)
1380 root->fs_info->sb->s_bdev = next_device->bdev;
1381 if (device->bdev == root->fs_info->fs_devices->latest_bdev)
1382 root->fs_info->fs_devices->latest_bdev = next_device->bdev;
1383
1384 if (device->bdev)
1385 device->fs_devices->open_devices--;
1386
1387 call_rcu(&device->rcu, free_device);
1388 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1389
1390 num_devices = btrfs_super_num_devices(&root->fs_info->super_copy) - 1;
1391 btrfs_set_super_num_devices(&root->fs_info->super_copy, num_devices);
1392
1393 if (cur_devices->open_devices == 0) {
1394 struct btrfs_fs_devices *fs_devices;
1395 fs_devices = root->fs_info->fs_devices;
1396 while (fs_devices) {
1397 if (fs_devices->seed == cur_devices)
1398 break;
1399 fs_devices = fs_devices->seed;
1400 }
1401 fs_devices->seed = cur_devices->seed;
1402 cur_devices->seed = NULL;
1403 lock_chunks(root);
1404 __btrfs_close_devices(cur_devices);
1405 unlock_chunks(root);
1406 free_fs_devices(cur_devices);
1407 }
1408
1409 /*
1410 * at this point, the device is zero sized. We want to
1411 * remove it from the devices list and zero out the old super
1412 */
1413 if (clear_super) {
1414 /* make sure this device isn't detected as part of
1415 * the FS anymore
1416 */
1417 memset(&disk_super->magic, 0, sizeof(disk_super->magic));
1418 set_buffer_dirty(bh);
1419 sync_dirty_buffer(bh);
1420 }
1421
1422 ret = 0;
1423
1424error_brelse:
1425 brelse(bh);
1426error_close:
1427 if (bdev)
1428 blkdev_put(bdev, FMODE_READ | FMODE_EXCL);
1429out:
1430 mutex_unlock(&root->fs_info->volume_mutex);
1431 mutex_unlock(&uuid_mutex);
1432 return ret;
1433error_undo:
1434 if (device->writeable) {
1435 lock_chunks(root);
1436 list_add(&device->dev_alloc_list,
1437 &root->fs_info->fs_devices->alloc_list);
1438 unlock_chunks(root);
1439 root->fs_info->fs_devices->rw_devices++;
1440 }
1441 goto error_brelse;
1442}
1443
1444/*
1445 * does all the dirty work required for changing file system's UUID.
1446 */
1447static int btrfs_prepare_sprout(struct btrfs_trans_handle *trans,
1448 struct btrfs_root *root)
1449{
1450 struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
1451 struct btrfs_fs_devices *old_devices;
1452 struct btrfs_fs_devices *seed_devices;
1453 struct btrfs_super_block *disk_super = &root->fs_info->super_copy;
1454 struct btrfs_device *device;
1455 u64 super_flags;
1456
1457 BUG_ON(!mutex_is_locked(&uuid_mutex));
1458 if (!fs_devices->seeding)
1459 return -EINVAL;
1460
1461 seed_devices = kzalloc(sizeof(*fs_devices), GFP_NOFS);
1462 if (!seed_devices)
1463 return -ENOMEM;
1464
1465 old_devices = clone_fs_devices(fs_devices);
1466 if (IS_ERR(old_devices)) {
1467 kfree(seed_devices);
1468 return PTR_ERR(old_devices);
1469 }
1470
1471 list_add(&old_devices->list, &fs_uuids);
1472
1473 memcpy(seed_devices, fs_devices, sizeof(*seed_devices));
1474 seed_devices->opened = 1;
1475 INIT_LIST_HEAD(&seed_devices->devices);
1476 INIT_LIST_HEAD(&seed_devices->alloc_list);
1477 mutex_init(&seed_devices->device_list_mutex);
1478
1479 mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
1480 list_splice_init_rcu(&fs_devices->devices, &seed_devices->devices,
1481 synchronize_rcu);
1482 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1483
1484 list_splice_init(&fs_devices->alloc_list, &seed_devices->alloc_list);
1485 list_for_each_entry(device, &seed_devices->devices, dev_list) {
1486 device->fs_devices = seed_devices;
1487 }
1488
1489 fs_devices->seeding = 0;
1490 fs_devices->num_devices = 0;
1491 fs_devices->open_devices = 0;
1492 fs_devices->seed = seed_devices;
1493
1494 generate_random_uuid(fs_devices->fsid);
1495 memcpy(root->fs_info->fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
1496 memcpy(disk_super->fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
1497 super_flags = btrfs_super_flags(disk_super) &
1498 ~BTRFS_SUPER_FLAG_SEEDING;
1499 btrfs_set_super_flags(disk_super, super_flags);
1500
1501 return 0;
1502}
1503
1504/*
1505 * strore the expected generation for seed devices in device items.
1506 */
1507static int btrfs_finish_sprout(struct btrfs_trans_handle *trans,
1508 struct btrfs_root *root)
1509{
1510 struct btrfs_path *path;
1511 struct extent_buffer *leaf;
1512 struct btrfs_dev_item *dev_item;
1513 struct btrfs_device *device;
1514 struct btrfs_key key;
1515 u8 fs_uuid[BTRFS_UUID_SIZE];
1516 u8 dev_uuid[BTRFS_UUID_SIZE];
1517 u64 devid;
1518 int ret;
1519
1520 path = btrfs_alloc_path();
1521 if (!path)
1522 return -ENOMEM;
1523
1524 root = root->fs_info->chunk_root;
1525 key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1526 key.offset = 0;
1527 key.type = BTRFS_DEV_ITEM_KEY;
1528
1529 while (1) {
1530 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
1531 if (ret < 0)
1532 goto error;
1533
1534 leaf = path->nodes[0];
1535next_slot:
1536 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
1537 ret = btrfs_next_leaf(root, path);
1538 if (ret > 0)
1539 break;
1540 if (ret < 0)
1541 goto error;
1542 leaf = path->nodes[0];
1543 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1544 btrfs_release_path(path);
1545 continue;
1546 }
1547
1548 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1549 if (key.objectid != BTRFS_DEV_ITEMS_OBJECTID ||
1550 key.type != BTRFS_DEV_ITEM_KEY)
1551 break;
1552
1553 dev_item = btrfs_item_ptr(leaf, path->slots[0],
1554 struct btrfs_dev_item);
1555 devid = btrfs_device_id(leaf, dev_item);
1556 read_extent_buffer(leaf, dev_uuid,
1557 (unsigned long)btrfs_device_uuid(dev_item),
1558 BTRFS_UUID_SIZE);
1559 read_extent_buffer(leaf, fs_uuid,
1560 (unsigned long)btrfs_device_fsid(dev_item),
1561 BTRFS_UUID_SIZE);
1562 device = btrfs_find_device(root, devid, dev_uuid, fs_uuid);
1563 BUG_ON(!device);
1564
1565 if (device->fs_devices->seeding) {
1566 btrfs_set_device_generation(leaf, dev_item,
1567 device->generation);
1568 btrfs_mark_buffer_dirty(leaf);
1569 }
1570
1571 path->slots[0]++;
1572 goto next_slot;
1573 }
1574 ret = 0;
1575error:
1576 btrfs_free_path(path);
1577 return ret;
1578}
1579
1580int btrfs_init_new_device(struct btrfs_root *root, char *device_path)
1581{
1582 struct request_queue *q;
1583 struct btrfs_trans_handle *trans;
1584 struct btrfs_device *device;
1585 struct block_device *bdev;
1586 struct list_head *devices;
1587 struct super_block *sb = root->fs_info->sb;
1588 u64 total_bytes;
1589 int seeding_dev = 0;
1590 int ret = 0;
1591
1592 if ((sb->s_flags & MS_RDONLY) && !root->fs_info->fs_devices->seeding)
1593 return -EINVAL;
1594
1595 bdev = blkdev_get_by_path(device_path, FMODE_EXCL,
1596 root->fs_info->bdev_holder);
1597 if (IS_ERR(bdev))
1598 return PTR_ERR(bdev);
1599
1600 if (root->fs_info->fs_devices->seeding) {
1601 seeding_dev = 1;
1602 down_write(&sb->s_umount);
1603 mutex_lock(&uuid_mutex);
1604 }
1605
1606 filemap_write_and_wait(bdev->bd_inode->i_mapping);
1607 mutex_lock(&root->fs_info->volume_mutex);
1608
1609 devices = &root->fs_info->fs_devices->devices;
1610 /*
1611 * we have the volume lock, so we don't need the extra
1612 * device list mutex while reading the list here.
1613 */
1614 list_for_each_entry(device, devices, dev_list) {
1615 if (device->bdev == bdev) {
1616 ret = -EEXIST;
1617 goto error;
1618 }
1619 }
1620
1621 device = kzalloc(sizeof(*device), GFP_NOFS);
1622 if (!device) {
1623 /* we can safely leave the fs_devices entry around */
1624 ret = -ENOMEM;
1625 goto error;
1626 }
1627
1628 device->name = kstrdup(device_path, GFP_NOFS);
1629 if (!device->name) {
1630 kfree(device);
1631 ret = -ENOMEM;
1632 goto error;
1633 }
1634
1635 ret = find_next_devid(root, &device->devid);
1636 if (ret) {
1637 kfree(device->name);
1638 kfree(device);
1639 goto error;
1640 }
1641
1642 trans = btrfs_start_transaction(root, 0);
1643 if (IS_ERR(trans)) {
1644 kfree(device->name);
1645 kfree(device);
1646 ret = PTR_ERR(trans);
1647 goto error;
1648 }
1649
1650 lock_chunks(root);
1651
1652 q = bdev_get_queue(bdev);
1653 if (blk_queue_discard(q))
1654 device->can_discard = 1;
1655 device->writeable = 1;
1656 device->work.func = pending_bios_fn;
1657 generate_random_uuid(device->uuid);
1658 spin_lock_init(&device->io_lock);
1659 device->generation = trans->transid;
1660 device->io_width = root->sectorsize;
1661 device->io_align = root->sectorsize;
1662 device->sector_size = root->sectorsize;
1663 device->total_bytes = i_size_read(bdev->bd_inode);
1664 device->disk_total_bytes = device->total_bytes;
1665 device->dev_root = root->fs_info->dev_root;
1666 device->bdev = bdev;
1667 device->in_fs_metadata = 1;
1668 device->mode = FMODE_EXCL;
1669 set_blocksize(device->bdev, 4096);
1670
1671 if (seeding_dev) {
1672 sb->s_flags &= ~MS_RDONLY;
1673 ret = btrfs_prepare_sprout(trans, root);
1674 BUG_ON(ret);
1675 }
1676
1677 device->fs_devices = root->fs_info->fs_devices;
1678
1679 /*
1680 * we don't want write_supers to jump in here with our device
1681 * half setup
1682 */
1683 mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
1684 list_add_rcu(&device->dev_list, &root->fs_info->fs_devices->devices);
1685 list_add(&device->dev_alloc_list,
1686 &root->fs_info->fs_devices->alloc_list);
1687 root->fs_info->fs_devices->num_devices++;
1688 root->fs_info->fs_devices->open_devices++;
1689 root->fs_info->fs_devices->rw_devices++;
1690 if (device->can_discard)
1691 root->fs_info->fs_devices->num_can_discard++;
1692 root->fs_info->fs_devices->total_rw_bytes += device->total_bytes;
1693
1694 if (!blk_queue_nonrot(bdev_get_queue(bdev)))
1695 root->fs_info->fs_devices->rotating = 1;
1696
1697 total_bytes = btrfs_super_total_bytes(&root->fs_info->super_copy);
1698 btrfs_set_super_total_bytes(&root->fs_info->super_copy,
1699 total_bytes + device->total_bytes);
1700
1701 total_bytes = btrfs_super_num_devices(&root->fs_info->super_copy);
1702 btrfs_set_super_num_devices(&root->fs_info->super_copy,
1703 total_bytes + 1);
1704 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1705
1706 if (seeding_dev) {
1707 ret = init_first_rw_device(trans, root, device);
1708 BUG_ON(ret);
1709 ret = btrfs_finish_sprout(trans, root);
1710 BUG_ON(ret);
1711 } else {
1712 ret = btrfs_add_device(trans, root, device);
1713 }
1714
1715 /*
1716 * we've got more storage, clear any full flags on the space
1717 * infos
1718 */
1719 btrfs_clear_space_info_full(root->fs_info);
1720
1721 unlock_chunks(root);
1722 btrfs_commit_transaction(trans, root);
1723
1724 if (seeding_dev) {
1725 mutex_unlock(&uuid_mutex);
1726 up_write(&sb->s_umount);
1727
1728 ret = btrfs_relocate_sys_chunks(root);
1729 BUG_ON(ret);
1730 }
1731out:
1732 mutex_unlock(&root->fs_info->volume_mutex);
1733 return ret;
1734error:
1735 blkdev_put(bdev, FMODE_EXCL);
1736 if (seeding_dev) {
1737 mutex_unlock(&uuid_mutex);
1738 up_write(&sb->s_umount);
1739 }
1740 goto out;
1741}
1742
1743static noinline int btrfs_update_device(struct btrfs_trans_handle *trans,
1744 struct btrfs_device *device)
1745{
1746 int ret;
1747 struct btrfs_path *path;
1748 struct btrfs_root *root;
1749 struct btrfs_dev_item *dev_item;
1750 struct extent_buffer *leaf;
1751 struct btrfs_key key;
1752
1753 root = device->dev_root->fs_info->chunk_root;
1754
1755 path = btrfs_alloc_path();
1756 if (!path)
1757 return -ENOMEM;
1758
1759 key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1760 key.type = BTRFS_DEV_ITEM_KEY;
1761 key.offset = device->devid;
1762
1763 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
1764 if (ret < 0)
1765 goto out;
1766
1767 if (ret > 0) {
1768 ret = -ENOENT;
1769 goto out;
1770 }
1771
1772 leaf = path->nodes[0];
1773 dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
1774
1775 btrfs_set_device_id(leaf, dev_item, device->devid);
1776 btrfs_set_device_type(leaf, dev_item, device->type);
1777 btrfs_set_device_io_align(leaf, dev_item, device->io_align);
1778 btrfs_set_device_io_width(leaf, dev_item, device->io_width);
1779 btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
1780 btrfs_set_device_total_bytes(leaf, dev_item, device->disk_total_bytes);
1781 btrfs_set_device_bytes_used(leaf, dev_item, device->bytes_used);
1782 btrfs_mark_buffer_dirty(leaf);
1783
1784out:
1785 btrfs_free_path(path);
1786 return ret;
1787}
1788
1789static int __btrfs_grow_device(struct btrfs_trans_handle *trans,
1790 struct btrfs_device *device, u64 new_size)
1791{
1792 struct btrfs_super_block *super_copy =
1793 &device->dev_root->fs_info->super_copy;
1794 u64 old_total = btrfs_super_total_bytes(super_copy);
1795 u64 diff = new_size - device->total_bytes;
1796
1797 if (!device->writeable)
1798 return -EACCES;
1799 if (new_size <= device->total_bytes)
1800 return -EINVAL;
1801
1802 btrfs_set_super_total_bytes(super_copy, old_total + diff);
1803 device->fs_devices->total_rw_bytes += diff;
1804
1805 device->total_bytes = new_size;
1806 device->disk_total_bytes = new_size;
1807 btrfs_clear_space_info_full(device->dev_root->fs_info);
1808
1809 return btrfs_update_device(trans, device);
1810}
1811
1812int btrfs_grow_device(struct btrfs_trans_handle *trans,
1813 struct btrfs_device *device, u64 new_size)
1814{
1815 int ret;
1816 lock_chunks(device->dev_root);
1817 ret = __btrfs_grow_device(trans, device, new_size);
1818 unlock_chunks(device->dev_root);
1819 return ret;
1820}
1821
1822static int btrfs_free_chunk(struct btrfs_trans_handle *trans,
1823 struct btrfs_root *root,
1824 u64 chunk_tree, u64 chunk_objectid,
1825 u64 chunk_offset)
1826{
1827 int ret;
1828 struct btrfs_path *path;
1829 struct btrfs_key key;
1830
1831 root = root->fs_info->chunk_root;
1832 path = btrfs_alloc_path();
1833 if (!path)
1834 return -ENOMEM;
1835
1836 key.objectid = chunk_objectid;
1837 key.offset = chunk_offset;
1838 key.type = BTRFS_CHUNK_ITEM_KEY;
1839
1840 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1841 BUG_ON(ret);
1842
1843 ret = btrfs_del_item(trans, root, path);
1844
1845 btrfs_free_path(path);
1846 return ret;
1847}
1848
1849static int btrfs_del_sys_chunk(struct btrfs_root *root, u64 chunk_objectid, u64
1850 chunk_offset)
1851{
1852 struct btrfs_super_block *super_copy = &root->fs_info->super_copy;
1853 struct btrfs_disk_key *disk_key;
1854 struct btrfs_chunk *chunk;
1855 u8 *ptr;
1856 int ret = 0;
1857 u32 num_stripes;
1858 u32 array_size;
1859 u32 len = 0;
1860 u32 cur;
1861 struct btrfs_key key;
1862
1863 array_size = btrfs_super_sys_array_size(super_copy);
1864
1865 ptr = super_copy->sys_chunk_array;
1866 cur = 0;
1867
1868 while (cur < array_size) {
1869 disk_key = (struct btrfs_disk_key *)ptr;
1870 btrfs_disk_key_to_cpu(&key, disk_key);
1871
1872 len = sizeof(*disk_key);
1873
1874 if (key.type == BTRFS_CHUNK_ITEM_KEY) {
1875 chunk = (struct btrfs_chunk *)(ptr + len);
1876 num_stripes = btrfs_stack_chunk_num_stripes(chunk);
1877 len += btrfs_chunk_item_size(num_stripes);
1878 } else {
1879 ret = -EIO;
1880 break;
1881 }
1882 if (key.objectid == chunk_objectid &&
1883 key.offset == chunk_offset) {
1884 memmove(ptr, ptr + len, array_size - (cur + len));
1885 array_size -= len;
1886 btrfs_set_super_sys_array_size(super_copy, array_size);
1887 } else {
1888 ptr += len;
1889 cur += len;
1890 }
1891 }
1892 return ret;
1893}
1894
1895static int btrfs_relocate_chunk(struct btrfs_root *root,
1896 u64 chunk_tree, u64 chunk_objectid,
1897 u64 chunk_offset)
1898{
1899 struct extent_map_tree *em_tree;
1900 struct btrfs_root *extent_root;
1901 struct btrfs_trans_handle *trans;
1902 struct extent_map *em;
1903 struct map_lookup *map;
1904 int ret;
1905 int i;
1906
1907 root = root->fs_info->chunk_root;
1908 extent_root = root->fs_info->extent_root;
1909 em_tree = &root->fs_info->mapping_tree.map_tree;
1910
1911 ret = btrfs_can_relocate(extent_root, chunk_offset);
1912 if (ret)
1913 return -ENOSPC;
1914
1915 /* step one, relocate all the extents inside this chunk */
1916 ret = btrfs_relocate_block_group(extent_root, chunk_offset);
1917 if (ret)
1918 return ret;
1919
1920 trans = btrfs_start_transaction(root, 0);
1921 BUG_ON(IS_ERR(trans));
1922
1923 lock_chunks(root);
1924
1925 /*
1926 * step two, delete the device extents and the
1927 * chunk tree entries
1928 */
1929 read_lock(&em_tree->lock);
1930 em = lookup_extent_mapping(em_tree, chunk_offset, 1);
1931 read_unlock(&em_tree->lock);
1932
1933 BUG_ON(em->start > chunk_offset ||
1934 em->start + em->len < chunk_offset);
1935 map = (struct map_lookup *)em->bdev;
1936
1937 for (i = 0; i < map->num_stripes; i++) {
1938 ret = btrfs_free_dev_extent(trans, map->stripes[i].dev,
1939 map->stripes[i].physical);
1940 BUG_ON(ret);
1941
1942 if (map->stripes[i].dev) {
1943 ret = btrfs_update_device(trans, map->stripes[i].dev);
1944 BUG_ON(ret);
1945 }
1946 }
1947 ret = btrfs_free_chunk(trans, root, chunk_tree, chunk_objectid,
1948 chunk_offset);
1949
1950 BUG_ON(ret);
1951
1952 trace_btrfs_chunk_free(root, map, chunk_offset, em->len);
1953
1954 if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
1955 ret = btrfs_del_sys_chunk(root, chunk_objectid, chunk_offset);
1956 BUG_ON(ret);
1957 }
1958
1959 ret = btrfs_remove_block_group(trans, extent_root, chunk_offset);
1960 BUG_ON(ret);
1961
1962 write_lock(&em_tree->lock);
1963 remove_extent_mapping(em_tree, em);
1964 write_unlock(&em_tree->lock);
1965
1966 kfree(map);
1967 em->bdev = NULL;
1968
1969 /* once for the tree */
1970 free_extent_map(em);
1971 /* once for us */
1972 free_extent_map(em);
1973
1974 unlock_chunks(root);
1975 btrfs_end_transaction(trans, root);
1976 return 0;
1977}
1978
1979static int btrfs_relocate_sys_chunks(struct btrfs_root *root)
1980{
1981 struct btrfs_root *chunk_root = root->fs_info->chunk_root;
1982 struct btrfs_path *path;
1983 struct extent_buffer *leaf;
1984 struct btrfs_chunk *chunk;
1985 struct btrfs_key key;
1986 struct btrfs_key found_key;
1987 u64 chunk_tree = chunk_root->root_key.objectid;
1988 u64 chunk_type;
1989 bool retried = false;
1990 int failed = 0;
1991 int ret;
1992
1993 path = btrfs_alloc_path();
1994 if (!path)
1995 return -ENOMEM;
1996
1997again:
1998 key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
1999 key.offset = (u64)-1;
2000 key.type = BTRFS_CHUNK_ITEM_KEY;
2001
2002 while (1) {
2003 ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
2004 if (ret < 0)
2005 goto error;
2006 BUG_ON(ret == 0);
2007
2008 ret = btrfs_previous_item(chunk_root, path, key.objectid,
2009 key.type);
2010 if (ret < 0)
2011 goto error;
2012 if (ret > 0)
2013 break;
2014
2015 leaf = path->nodes[0];
2016 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
2017
2018 chunk = btrfs_item_ptr(leaf, path->slots[0],
2019 struct btrfs_chunk);
2020 chunk_type = btrfs_chunk_type(leaf, chunk);
2021 btrfs_release_path(path);
2022
2023 if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) {
2024 ret = btrfs_relocate_chunk(chunk_root, chunk_tree,
2025 found_key.objectid,
2026 found_key.offset);
2027 if (ret == -ENOSPC)
2028 failed++;
2029 else if (ret)
2030 BUG();
2031 }
2032
2033 if (found_key.offset == 0)
2034 break;
2035 key.offset = found_key.offset - 1;
2036 }
2037 ret = 0;
2038 if (failed && !retried) {
2039 failed = 0;
2040 retried = true;
2041 goto again;
2042 } else if (failed && retried) {
2043 WARN_ON(1);
2044 ret = -ENOSPC;
2045 }
2046error:
2047 btrfs_free_path(path);
2048 return ret;
2049}
2050
2051static u64 div_factor(u64 num, int factor)
2052{
2053 if (factor == 10)
2054 return num;
2055 num *= factor;
2056 do_div(num, 10);
2057 return num;
2058}
2059
2060int btrfs_balance(struct btrfs_root *dev_root)
2061{
2062 int ret;
2063 struct list_head *devices = &dev_root->fs_info->fs_devices->devices;
2064 struct btrfs_device *device;
2065 u64 old_size;
2066 u64 size_to_free;
2067 struct btrfs_path *path;
2068 struct btrfs_key key;
2069 struct btrfs_root *chunk_root = dev_root->fs_info->chunk_root;
2070 struct btrfs_trans_handle *trans;
2071 struct btrfs_key found_key;
2072
2073 if (dev_root->fs_info->sb->s_flags & MS_RDONLY)
2074 return -EROFS;
2075
2076 if (!capable(CAP_SYS_ADMIN))
2077 return -EPERM;
2078
2079 mutex_lock(&dev_root->fs_info->volume_mutex);
2080 dev_root = dev_root->fs_info->dev_root;
2081
2082 /* step one make some room on all the devices */
2083 list_for_each_entry(device, devices, dev_list) {
2084 old_size = device->total_bytes;
2085 size_to_free = div_factor(old_size, 1);
2086 size_to_free = min(size_to_free, (u64)1 * 1024 * 1024);
2087 if (!device->writeable ||
2088 device->total_bytes - device->bytes_used > size_to_free)
2089 continue;
2090
2091 ret = btrfs_shrink_device(device, old_size - size_to_free);
2092 if (ret == -ENOSPC)
2093 break;
2094 BUG_ON(ret);
2095
2096 trans = btrfs_start_transaction(dev_root, 0);
2097 BUG_ON(IS_ERR(trans));
2098
2099 ret = btrfs_grow_device(trans, device, old_size);
2100 BUG_ON(ret);
2101
2102 btrfs_end_transaction(trans, dev_root);
2103 }
2104
2105 /* step two, relocate all the chunks */
2106 path = btrfs_alloc_path();
2107 if (!path) {
2108 ret = -ENOMEM;
2109 goto error;
2110 }
2111 key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
2112 key.offset = (u64)-1;
2113 key.type = BTRFS_CHUNK_ITEM_KEY;
2114
2115 while (1) {
2116 ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
2117 if (ret < 0)
2118 goto error;
2119
2120 /*
2121 * this shouldn't happen, it means the last relocate
2122 * failed
2123 */
2124 if (ret == 0)
2125 break;
2126
2127 ret = btrfs_previous_item(chunk_root, path, 0,
2128 BTRFS_CHUNK_ITEM_KEY);
2129 if (ret)
2130 break;
2131
2132 btrfs_item_key_to_cpu(path->nodes[0], &found_key,
2133 path->slots[0]);
2134 if (found_key.objectid != key.objectid)
2135 break;
2136
2137 /* chunk zero is special */
2138 if (found_key.offset == 0)
2139 break;
2140
2141 btrfs_release_path(path);
2142 ret = btrfs_relocate_chunk(chunk_root,
2143 chunk_root->root_key.objectid,
2144 found_key.objectid,
2145 found_key.offset);
2146 if (ret && ret != -ENOSPC)
2147 goto error;
2148 key.offset = found_key.offset - 1;
2149 }
2150 ret = 0;
2151error:
2152 btrfs_free_path(path);
2153 mutex_unlock(&dev_root->fs_info->volume_mutex);
2154 return ret;
2155}
2156
2157/*
2158 * shrinking a device means finding all of the device extents past
2159 * the new size, and then following the back refs to the chunks.
2160 * The chunk relocation code actually frees the device extent
2161 */
2162int btrfs_shrink_device(struct btrfs_device *device, u64 new_size)
2163{
2164 struct btrfs_trans_handle *trans;
2165 struct btrfs_root *root = device->dev_root;
2166 struct btrfs_dev_extent *dev_extent = NULL;
2167 struct btrfs_path *path;
2168 u64 length;
2169 u64 chunk_tree;
2170 u64 chunk_objectid;
2171 u64 chunk_offset;
2172 int ret;
2173 int slot;
2174 int failed = 0;
2175 bool retried = false;
2176 struct extent_buffer *l;
2177 struct btrfs_key key;
2178 struct btrfs_super_block *super_copy = &root->fs_info->super_copy;
2179 u64 old_total = btrfs_super_total_bytes(super_copy);
2180 u64 old_size = device->total_bytes;
2181 u64 diff = device->total_bytes - new_size;
2182
2183 if (new_size >= device->total_bytes)
2184 return -EINVAL;
2185
2186 path = btrfs_alloc_path();
2187 if (!path)
2188 return -ENOMEM;
2189
2190 path->reada = 2;
2191
2192 lock_chunks(root);
2193
2194 device->total_bytes = new_size;
2195 if (device->writeable)
2196 device->fs_devices->total_rw_bytes -= diff;
2197 unlock_chunks(root);
2198
2199again:
2200 key.objectid = device->devid;
2201 key.offset = (u64)-1;
2202 key.type = BTRFS_DEV_EXTENT_KEY;
2203
2204 while (1) {
2205 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2206 if (ret < 0)
2207 goto done;
2208
2209 ret = btrfs_previous_item(root, path, 0, key.type);
2210 if (ret < 0)
2211 goto done;
2212 if (ret) {
2213 ret = 0;
2214 btrfs_release_path(path);
2215 break;
2216 }
2217
2218 l = path->nodes[0];
2219 slot = path->slots[0];
2220 btrfs_item_key_to_cpu(l, &key, path->slots[0]);
2221
2222 if (key.objectid != device->devid) {
2223 btrfs_release_path(path);
2224 break;
2225 }
2226
2227 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
2228 length = btrfs_dev_extent_length(l, dev_extent);
2229
2230 if (key.offset + length <= new_size) {
2231 btrfs_release_path(path);
2232 break;
2233 }
2234
2235 chunk_tree = btrfs_dev_extent_chunk_tree(l, dev_extent);
2236 chunk_objectid = btrfs_dev_extent_chunk_objectid(l, dev_extent);
2237 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
2238 btrfs_release_path(path);
2239
2240 ret = btrfs_relocate_chunk(root, chunk_tree, chunk_objectid,
2241 chunk_offset);
2242 if (ret && ret != -ENOSPC)
2243 goto done;
2244 if (ret == -ENOSPC)
2245 failed++;
2246 key.offset -= 1;
2247 }
2248
2249 if (failed && !retried) {
2250 failed = 0;
2251 retried = true;
2252 goto again;
2253 } else if (failed && retried) {
2254 ret = -ENOSPC;
2255 lock_chunks(root);
2256
2257 device->total_bytes = old_size;
2258 if (device->writeable)
2259 device->fs_devices->total_rw_bytes += diff;
2260 unlock_chunks(root);
2261 goto done;
2262 }
2263
2264 /* Shrinking succeeded, else we would be at "done". */
2265 trans = btrfs_start_transaction(root, 0);
2266 if (IS_ERR(trans)) {
2267 ret = PTR_ERR(trans);
2268 goto done;
2269 }
2270
2271 lock_chunks(root);
2272
2273 device->disk_total_bytes = new_size;
2274 /* Now btrfs_update_device() will change the on-disk size. */
2275 ret = btrfs_update_device(trans, device);
2276 if (ret) {
2277 unlock_chunks(root);
2278 btrfs_end_transaction(trans, root);
2279 goto done;
2280 }
2281 WARN_ON(diff > old_total);
2282 btrfs_set_super_total_bytes(super_copy, old_total - diff);
2283 unlock_chunks(root);
2284 btrfs_end_transaction(trans, root);
2285done:
2286 btrfs_free_path(path);
2287 return ret;
2288}
2289
2290static int btrfs_add_system_chunk(struct btrfs_trans_handle *trans,
2291 struct btrfs_root *root,
2292 struct btrfs_key *key,
2293 struct btrfs_chunk *chunk, int item_size)
2294{
2295 struct btrfs_super_block *super_copy = &root->fs_info->super_copy;
2296 struct btrfs_disk_key disk_key;
2297 u32 array_size;
2298 u8 *ptr;
2299
2300 array_size = btrfs_super_sys_array_size(super_copy);
2301 if (array_size + item_size > BTRFS_SYSTEM_CHUNK_ARRAY_SIZE)
2302 return -EFBIG;
2303
2304 ptr = super_copy->sys_chunk_array + array_size;
2305 btrfs_cpu_key_to_disk(&disk_key, key);
2306 memcpy(ptr, &disk_key, sizeof(disk_key));
2307 ptr += sizeof(disk_key);
2308 memcpy(ptr, chunk, item_size);
2309 item_size += sizeof(disk_key);
2310 btrfs_set_super_sys_array_size(super_copy, array_size + item_size);
2311 return 0;
2312}
2313
2314/*
2315 * sort the devices in descending order by max_avail, total_avail
2316 */
2317static int btrfs_cmp_device_info(const void *a, const void *b)
2318{
2319 const struct btrfs_device_info *di_a = a;
2320 const struct btrfs_device_info *di_b = b;
2321
2322 if (di_a->max_avail > di_b->max_avail)
2323 return -1;
2324 if (di_a->max_avail < di_b->max_avail)
2325 return 1;
2326 if (di_a->total_avail > di_b->total_avail)
2327 return -1;
2328 if (di_a->total_avail < di_b->total_avail)
2329 return 1;
2330 return 0;
2331}
2332
2333static int __btrfs_alloc_chunk(struct btrfs_trans_handle *trans,
2334 struct btrfs_root *extent_root,
2335 struct map_lookup **map_ret,
2336 u64 *num_bytes_out, u64 *stripe_size_out,
2337 u64 start, u64 type)
2338{
2339 struct btrfs_fs_info *info = extent_root->fs_info;
2340 struct btrfs_fs_devices *fs_devices = info->fs_devices;
2341 struct list_head *cur;
2342 struct map_lookup *map = NULL;
2343 struct extent_map_tree *em_tree;
2344 struct extent_map *em;
2345 struct btrfs_device_info *devices_info = NULL;
2346 u64 total_avail;
2347 int num_stripes; /* total number of stripes to allocate */
2348 int sub_stripes; /* sub_stripes info for map */
2349 int dev_stripes; /* stripes per dev */
2350 int devs_max; /* max devs to use */
2351 int devs_min; /* min devs needed */
2352 int devs_increment; /* ndevs has to be a multiple of this */
2353 int ncopies; /* how many copies to data has */
2354 int ret;
2355 u64 max_stripe_size;
2356 u64 max_chunk_size;
2357 u64 stripe_size;
2358 u64 num_bytes;
2359 int ndevs;
2360 int i;
2361 int j;
2362
2363 if ((type & BTRFS_BLOCK_GROUP_RAID1) &&
2364 (type & BTRFS_BLOCK_GROUP_DUP)) {
2365 WARN_ON(1);
2366 type &= ~BTRFS_BLOCK_GROUP_DUP;
2367 }
2368
2369 if (list_empty(&fs_devices->alloc_list))
2370 return -ENOSPC;
2371
2372 sub_stripes = 1;
2373 dev_stripes = 1;
2374 devs_increment = 1;
2375 ncopies = 1;
2376 devs_max = 0; /* 0 == as many as possible */
2377 devs_min = 1;
2378
2379 /*
2380 * define the properties of each RAID type.
2381 * FIXME: move this to a global table and use it in all RAID
2382 * calculation code
2383 */
2384 if (type & (BTRFS_BLOCK_GROUP_DUP)) {
2385 dev_stripes = 2;
2386 ncopies = 2;
2387 devs_max = 1;
2388 } else if (type & (BTRFS_BLOCK_GROUP_RAID0)) {
2389 devs_min = 2;
2390 } else if (type & (BTRFS_BLOCK_GROUP_RAID1)) {
2391 devs_increment = 2;
2392 ncopies = 2;
2393 devs_max = 2;
2394 devs_min = 2;
2395 } else if (type & (BTRFS_BLOCK_GROUP_RAID10)) {
2396 sub_stripes = 2;
2397 devs_increment = 2;
2398 ncopies = 2;
2399 devs_min = 4;
2400 } else {
2401 devs_max = 1;
2402 }
2403
2404 if (type & BTRFS_BLOCK_GROUP_DATA) {
2405 max_stripe_size = 1024 * 1024 * 1024;
2406 max_chunk_size = 10 * max_stripe_size;
2407 } else if (type & BTRFS_BLOCK_GROUP_METADATA) {
2408 max_stripe_size = 256 * 1024 * 1024;
2409 max_chunk_size = max_stripe_size;
2410 } else if (type & BTRFS_BLOCK_GROUP_SYSTEM) {
2411 max_stripe_size = 8 * 1024 * 1024;
2412 max_chunk_size = 2 * max_stripe_size;
2413 } else {
2414 printk(KERN_ERR "btrfs: invalid chunk type 0x%llx requested\n",
2415 type);
2416 BUG_ON(1);
2417 }
2418
2419 /* we don't want a chunk larger than 10% of writeable space */
2420 max_chunk_size = min(div_factor(fs_devices->total_rw_bytes, 1),
2421 max_chunk_size);
2422
2423 devices_info = kzalloc(sizeof(*devices_info) * fs_devices->rw_devices,
2424 GFP_NOFS);
2425 if (!devices_info)
2426 return -ENOMEM;
2427
2428 cur = fs_devices->alloc_list.next;
2429
2430 /*
2431 * in the first pass through the devices list, we gather information
2432 * about the available holes on each device.
2433 */
2434 ndevs = 0;
2435 while (cur != &fs_devices->alloc_list) {
2436 struct btrfs_device *device;
2437 u64 max_avail;
2438 u64 dev_offset;
2439
2440 device = list_entry(cur, struct btrfs_device, dev_alloc_list);
2441
2442 cur = cur->next;
2443
2444 if (!device->writeable) {
2445 printk(KERN_ERR
2446 "btrfs: read-only device in alloc_list\n");
2447 WARN_ON(1);
2448 continue;
2449 }
2450
2451 if (!device->in_fs_metadata)
2452 continue;
2453
2454 if (device->total_bytes > device->bytes_used)
2455 total_avail = device->total_bytes - device->bytes_used;
2456 else
2457 total_avail = 0;
2458
2459 /* If there is no space on this device, skip it. */
2460 if (total_avail == 0)
2461 continue;
2462
2463 ret = find_free_dev_extent(trans, device,
2464 max_stripe_size * dev_stripes,
2465 &dev_offset, &max_avail);
2466 if (ret && ret != -ENOSPC)
2467 goto error;
2468
2469 if (ret == 0)
2470 max_avail = max_stripe_size * dev_stripes;
2471
2472 if (max_avail < BTRFS_STRIPE_LEN * dev_stripes)
2473 continue;
2474
2475 devices_info[ndevs].dev_offset = dev_offset;
2476 devices_info[ndevs].max_avail = max_avail;
2477 devices_info[ndevs].total_avail = total_avail;
2478 devices_info[ndevs].dev = device;
2479 ++ndevs;
2480 }
2481
2482 /*
2483 * now sort the devices by hole size / available space
2484 */
2485 sort(devices_info, ndevs, sizeof(struct btrfs_device_info),
2486 btrfs_cmp_device_info, NULL);
2487
2488 /* round down to number of usable stripes */
2489 ndevs -= ndevs % devs_increment;
2490
2491 if (ndevs < devs_increment * sub_stripes || ndevs < devs_min) {
2492 ret = -ENOSPC;
2493 goto error;
2494 }
2495
2496 if (devs_max && ndevs > devs_max)
2497 ndevs = devs_max;
2498 /*
2499 * the primary goal is to maximize the number of stripes, so use as many
2500 * devices as possible, even if the stripes are not maximum sized.
2501 */
2502 stripe_size = devices_info[ndevs-1].max_avail;
2503 num_stripes = ndevs * dev_stripes;
2504
2505 if (stripe_size * num_stripes > max_chunk_size * ncopies) {
2506 stripe_size = max_chunk_size * ncopies;
2507 do_div(stripe_size, num_stripes);
2508 }
2509
2510 do_div(stripe_size, dev_stripes);
2511 do_div(stripe_size, BTRFS_STRIPE_LEN);
2512 stripe_size *= BTRFS_STRIPE_LEN;
2513
2514 map = kmalloc(map_lookup_size(num_stripes), GFP_NOFS);
2515 if (!map) {
2516 ret = -ENOMEM;
2517 goto error;
2518 }
2519 map->num_stripes = num_stripes;
2520
2521 for (i = 0; i < ndevs; ++i) {
2522 for (j = 0; j < dev_stripes; ++j) {
2523 int s = i * dev_stripes + j;
2524 map->stripes[s].dev = devices_info[i].dev;
2525 map->stripes[s].physical = devices_info[i].dev_offset +
2526 j * stripe_size;
2527 }
2528 }
2529 map->sector_size = extent_root->sectorsize;
2530 map->stripe_len = BTRFS_STRIPE_LEN;
2531 map->io_align = BTRFS_STRIPE_LEN;
2532 map->io_width = BTRFS_STRIPE_LEN;
2533 map->type = type;
2534 map->sub_stripes = sub_stripes;
2535
2536 *map_ret = map;
2537 num_bytes = stripe_size * (num_stripes / ncopies);
2538
2539 *stripe_size_out = stripe_size;
2540 *num_bytes_out = num_bytes;
2541
2542 trace_btrfs_chunk_alloc(info->chunk_root, map, start, num_bytes);
2543
2544 em = alloc_extent_map();
2545 if (!em) {
2546 ret = -ENOMEM;
2547 goto error;
2548 }
2549 em->bdev = (struct block_device *)map;
2550 em->start = start;
2551 em->len = num_bytes;
2552 em->block_start = 0;
2553 em->block_len = em->len;
2554
2555 em_tree = &extent_root->fs_info->mapping_tree.map_tree;
2556 write_lock(&em_tree->lock);
2557 ret = add_extent_mapping(em_tree, em);
2558 write_unlock(&em_tree->lock);
2559 BUG_ON(ret);
2560 free_extent_map(em);
2561
2562 ret = btrfs_make_block_group(trans, extent_root, 0, type,
2563 BTRFS_FIRST_CHUNK_TREE_OBJECTID,
2564 start, num_bytes);
2565 BUG_ON(ret);
2566
2567 for (i = 0; i < map->num_stripes; ++i) {
2568 struct btrfs_device *device;
2569 u64 dev_offset;
2570
2571 device = map->stripes[i].dev;
2572 dev_offset = map->stripes[i].physical;
2573
2574 ret = btrfs_alloc_dev_extent(trans, device,
2575 info->chunk_root->root_key.objectid,
2576 BTRFS_FIRST_CHUNK_TREE_OBJECTID,
2577 start, dev_offset, stripe_size);
2578 BUG_ON(ret);
2579 }
2580
2581 kfree(devices_info);
2582 return 0;
2583
2584error:
2585 kfree(map);
2586 kfree(devices_info);
2587 return ret;
2588}
2589
2590static int __finish_chunk_alloc(struct btrfs_trans_handle *trans,
2591 struct btrfs_root *extent_root,
2592 struct map_lookup *map, u64 chunk_offset,
2593 u64 chunk_size, u64 stripe_size)
2594{
2595 u64 dev_offset;
2596 struct btrfs_key key;
2597 struct btrfs_root *chunk_root = extent_root->fs_info->chunk_root;
2598 struct btrfs_device *device;
2599 struct btrfs_chunk *chunk;
2600 struct btrfs_stripe *stripe;
2601 size_t item_size = btrfs_chunk_item_size(map->num_stripes);
2602 int index = 0;
2603 int ret;
2604
2605 chunk = kzalloc(item_size, GFP_NOFS);
2606 if (!chunk)
2607 return -ENOMEM;
2608
2609 index = 0;
2610 while (index < map->num_stripes) {
2611 device = map->stripes[index].dev;
2612 device->bytes_used += stripe_size;
2613 ret = btrfs_update_device(trans, device);
2614 BUG_ON(ret);
2615 index++;
2616 }
2617
2618 index = 0;
2619 stripe = &chunk->stripe;
2620 while (index < map->num_stripes) {
2621 device = map->stripes[index].dev;
2622 dev_offset = map->stripes[index].physical;
2623
2624 btrfs_set_stack_stripe_devid(stripe, device->devid);
2625 btrfs_set_stack_stripe_offset(stripe, dev_offset);
2626 memcpy(stripe->dev_uuid, device->uuid, BTRFS_UUID_SIZE);
2627 stripe++;
2628 index++;
2629 }
2630
2631 btrfs_set_stack_chunk_length(chunk, chunk_size);
2632 btrfs_set_stack_chunk_owner(chunk, extent_root->root_key.objectid);
2633 btrfs_set_stack_chunk_stripe_len(chunk, map->stripe_len);
2634 btrfs_set_stack_chunk_type(chunk, map->type);
2635 btrfs_set_stack_chunk_num_stripes(chunk, map->num_stripes);
2636 btrfs_set_stack_chunk_io_align(chunk, map->stripe_len);
2637 btrfs_set_stack_chunk_io_width(chunk, map->stripe_len);
2638 btrfs_set_stack_chunk_sector_size(chunk, extent_root->sectorsize);
2639 btrfs_set_stack_chunk_sub_stripes(chunk, map->sub_stripes);
2640
2641 key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
2642 key.type = BTRFS_CHUNK_ITEM_KEY;
2643 key.offset = chunk_offset;
2644
2645 ret = btrfs_insert_item(trans, chunk_root, &key, chunk, item_size);
2646 BUG_ON(ret);
2647
2648 if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
2649 ret = btrfs_add_system_chunk(trans, chunk_root, &key, chunk,
2650 item_size);
2651 BUG_ON(ret);
2652 }
2653
2654 kfree(chunk);
2655 return 0;
2656}
2657
2658/*
2659 * Chunk allocation falls into two parts. The first part does works
2660 * that make the new allocated chunk useable, but not do any operation
2661 * that modifies the chunk tree. The second part does the works that
2662 * require modifying the chunk tree. This division is important for the
2663 * bootstrap process of adding storage to a seed btrfs.
2664 */
2665int btrfs_alloc_chunk(struct btrfs_trans_handle *trans,
2666 struct btrfs_root *extent_root, u64 type)
2667{
2668 u64 chunk_offset;
2669 u64 chunk_size;
2670 u64 stripe_size;
2671 struct map_lookup *map;
2672 struct btrfs_root *chunk_root = extent_root->fs_info->chunk_root;
2673 int ret;
2674
2675 ret = find_next_chunk(chunk_root, BTRFS_FIRST_CHUNK_TREE_OBJECTID,
2676 &chunk_offset);
2677 if (ret)
2678 return ret;
2679
2680 ret = __btrfs_alloc_chunk(trans, extent_root, &map, &chunk_size,
2681 &stripe_size, chunk_offset, type);
2682 if (ret)
2683 return ret;
2684
2685 ret = __finish_chunk_alloc(trans, extent_root, map, chunk_offset,
2686 chunk_size, stripe_size);
2687 BUG_ON(ret);
2688 return 0;
2689}
2690
2691static noinline int init_first_rw_device(struct btrfs_trans_handle *trans,
2692 struct btrfs_root *root,
2693 struct btrfs_device *device)
2694{
2695 u64 chunk_offset;
2696 u64 sys_chunk_offset;
2697 u64 chunk_size;
2698 u64 sys_chunk_size;
2699 u64 stripe_size;
2700 u64 sys_stripe_size;
2701 u64 alloc_profile;
2702 struct map_lookup *map;
2703 struct map_lookup *sys_map;
2704 struct btrfs_fs_info *fs_info = root->fs_info;
2705 struct btrfs_root *extent_root = fs_info->extent_root;
2706 int ret;
2707
2708 ret = find_next_chunk(fs_info->chunk_root,
2709 BTRFS_FIRST_CHUNK_TREE_OBJECTID, &chunk_offset);
2710 if (ret)
2711 return ret;
2712
2713 alloc_profile = BTRFS_BLOCK_GROUP_METADATA |
2714 (fs_info->metadata_alloc_profile &
2715 fs_info->avail_metadata_alloc_bits);
2716 alloc_profile = btrfs_reduce_alloc_profile(root, alloc_profile);
2717
2718 ret = __btrfs_alloc_chunk(trans, extent_root, &map, &chunk_size,
2719 &stripe_size, chunk_offset, alloc_profile);
2720 BUG_ON(ret);
2721
2722 sys_chunk_offset = chunk_offset + chunk_size;
2723
2724 alloc_profile = BTRFS_BLOCK_GROUP_SYSTEM |
2725 (fs_info->system_alloc_profile &
2726 fs_info->avail_system_alloc_bits);
2727 alloc_profile = btrfs_reduce_alloc_profile(root, alloc_profile);
2728
2729 ret = __btrfs_alloc_chunk(trans, extent_root, &sys_map,
2730 &sys_chunk_size, &sys_stripe_size,
2731 sys_chunk_offset, alloc_profile);
2732 BUG_ON(ret);
2733
2734 ret = btrfs_add_device(trans, fs_info->chunk_root, device);
2735 BUG_ON(ret);
2736
2737 /*
2738 * Modifying chunk tree needs allocating new blocks from both
2739 * system block group and metadata block group. So we only can
2740 * do operations require modifying the chunk tree after both
2741 * block groups were created.
2742 */
2743 ret = __finish_chunk_alloc(trans, extent_root, map, chunk_offset,
2744 chunk_size, stripe_size);
2745 BUG_ON(ret);
2746
2747 ret = __finish_chunk_alloc(trans, extent_root, sys_map,
2748 sys_chunk_offset, sys_chunk_size,
2749 sys_stripe_size);
2750 BUG_ON(ret);
2751 return 0;
2752}
2753
2754int btrfs_chunk_readonly(struct btrfs_root *root, u64 chunk_offset)
2755{
2756 struct extent_map *em;
2757 struct map_lookup *map;
2758 struct btrfs_mapping_tree *map_tree = &root->fs_info->mapping_tree;
2759 int readonly = 0;
2760 int i;
2761
2762 read_lock(&map_tree->map_tree.lock);
2763 em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
2764 read_unlock(&map_tree->map_tree.lock);
2765 if (!em)
2766 return 1;
2767
2768 if (btrfs_test_opt(root, DEGRADED)) {
2769 free_extent_map(em);
2770 return 0;
2771 }
2772
2773 map = (struct map_lookup *)em->bdev;
2774 for (i = 0; i < map->num_stripes; i++) {
2775 if (!map->stripes[i].dev->writeable) {
2776 readonly = 1;
2777 break;
2778 }
2779 }
2780 free_extent_map(em);
2781 return readonly;
2782}
2783
2784void btrfs_mapping_init(struct btrfs_mapping_tree *tree)
2785{
2786 extent_map_tree_init(&tree->map_tree);
2787}
2788
2789void btrfs_mapping_tree_free(struct btrfs_mapping_tree *tree)
2790{
2791 struct extent_map *em;
2792
2793 while (1) {
2794 write_lock(&tree->map_tree.lock);
2795 em = lookup_extent_mapping(&tree->map_tree, 0, (u64)-1);
2796 if (em)
2797 remove_extent_mapping(&tree->map_tree, em);
2798 write_unlock(&tree->map_tree.lock);
2799 if (!em)
2800 break;
2801 kfree(em->bdev);
2802 /* once for us */
2803 free_extent_map(em);
2804 /* once for the tree */
2805 free_extent_map(em);
2806 }
2807}
2808
2809int btrfs_num_copies(struct btrfs_mapping_tree *map_tree, u64 logical, u64 len)
2810{
2811 struct extent_map *em;
2812 struct map_lookup *map;
2813 struct extent_map_tree *em_tree = &map_tree->map_tree;
2814 int ret;
2815
2816 read_lock(&em_tree->lock);
2817 em = lookup_extent_mapping(em_tree, logical, len);
2818 read_unlock(&em_tree->lock);
2819 BUG_ON(!em);
2820
2821 BUG_ON(em->start > logical || em->start + em->len < logical);
2822 map = (struct map_lookup *)em->bdev;
2823 if (map->type & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1))
2824 ret = map->num_stripes;
2825 else if (map->type & BTRFS_BLOCK_GROUP_RAID10)
2826 ret = map->sub_stripes;
2827 else
2828 ret = 1;
2829 free_extent_map(em);
2830 return ret;
2831}
2832
2833static int find_live_mirror(struct map_lookup *map, int first, int num,
2834 int optimal)
2835{
2836 int i;
2837 if (map->stripes[optimal].dev->bdev)
2838 return optimal;
2839 for (i = first; i < first + num; i++) {
2840 if (map->stripes[i].dev->bdev)
2841 return i;
2842 }
2843 /* we couldn't find one that doesn't fail. Just return something
2844 * and the io error handling code will clean up eventually
2845 */
2846 return optimal;
2847}
2848
2849static int __btrfs_map_block(struct btrfs_mapping_tree *map_tree, int rw,
2850 u64 logical, u64 *length,
2851 struct btrfs_multi_bio **multi_ret,
2852 int mirror_num)
2853{
2854 struct extent_map *em;
2855 struct map_lookup *map;
2856 struct extent_map_tree *em_tree = &map_tree->map_tree;
2857 u64 offset;
2858 u64 stripe_offset;
2859 u64 stripe_end_offset;
2860 u64 stripe_nr;
2861 u64 stripe_nr_orig;
2862 u64 stripe_nr_end;
2863 int stripes_allocated = 8;
2864 int stripes_required = 1;
2865 int stripe_index;
2866 int i;
2867 int num_stripes;
2868 int max_errors = 0;
2869 struct btrfs_multi_bio *multi = NULL;
2870
2871 if (multi_ret && !(rw & (REQ_WRITE | REQ_DISCARD)))
2872 stripes_allocated = 1;
2873again:
2874 if (multi_ret) {
2875 multi = kzalloc(btrfs_multi_bio_size(stripes_allocated),
2876 GFP_NOFS);
2877 if (!multi)
2878 return -ENOMEM;
2879
2880 atomic_set(&multi->error, 0);
2881 }
2882
2883 read_lock(&em_tree->lock);
2884 em = lookup_extent_mapping(em_tree, logical, *length);
2885 read_unlock(&em_tree->lock);
2886
2887 if (!em) {
2888 printk(KERN_CRIT "unable to find logical %llu len %llu\n",
2889 (unsigned long long)logical,
2890 (unsigned long long)*length);
2891 BUG();
2892 }
2893
2894 BUG_ON(em->start > logical || em->start + em->len < logical);
2895 map = (struct map_lookup *)em->bdev;
2896 offset = logical - em->start;
2897
2898 if (mirror_num > map->num_stripes)
2899 mirror_num = 0;
2900
2901 /* if our multi bio struct is too small, back off and try again */
2902 if (rw & REQ_WRITE) {
2903 if (map->type & (BTRFS_BLOCK_GROUP_RAID1 |
2904 BTRFS_BLOCK_GROUP_DUP)) {
2905 stripes_required = map->num_stripes;
2906 max_errors = 1;
2907 } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
2908 stripes_required = map->sub_stripes;
2909 max_errors = 1;
2910 }
2911 }
2912 if (rw & REQ_DISCARD) {
2913 if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
2914 BTRFS_BLOCK_GROUP_RAID1 |
2915 BTRFS_BLOCK_GROUP_DUP |
2916 BTRFS_BLOCK_GROUP_RAID10)) {
2917 stripes_required = map->num_stripes;
2918 }
2919 }
2920 if (multi_ret && (rw & (REQ_WRITE | REQ_DISCARD)) &&
2921 stripes_allocated < stripes_required) {
2922 stripes_allocated = map->num_stripes;
2923 free_extent_map(em);
2924 kfree(multi);
2925 goto again;
2926 }
2927 stripe_nr = offset;
2928 /*
2929 * stripe_nr counts the total number of stripes we have to stride
2930 * to get to this block
2931 */
2932 do_div(stripe_nr, map->stripe_len);
2933
2934 stripe_offset = stripe_nr * map->stripe_len;
2935 BUG_ON(offset < stripe_offset);
2936
2937 /* stripe_offset is the offset of this block in its stripe*/
2938 stripe_offset = offset - stripe_offset;
2939
2940 if (rw & REQ_DISCARD)
2941 *length = min_t(u64, em->len - offset, *length);
2942 else if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
2943 BTRFS_BLOCK_GROUP_RAID1 |
2944 BTRFS_BLOCK_GROUP_RAID10 |
2945 BTRFS_BLOCK_GROUP_DUP)) {
2946 /* we limit the length of each bio to what fits in a stripe */
2947 *length = min_t(u64, em->len - offset,
2948 map->stripe_len - stripe_offset);
2949 } else {
2950 *length = em->len - offset;
2951 }
2952
2953 if (!multi_ret)
2954 goto out;
2955
2956 num_stripes = 1;
2957 stripe_index = 0;
2958 stripe_nr_orig = stripe_nr;
2959 stripe_nr_end = (offset + *length + map->stripe_len - 1) &
2960 (~(map->stripe_len - 1));
2961 do_div(stripe_nr_end, map->stripe_len);
2962 stripe_end_offset = stripe_nr_end * map->stripe_len -
2963 (offset + *length);
2964 if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
2965 if (rw & REQ_DISCARD)
2966 num_stripes = min_t(u64, map->num_stripes,
2967 stripe_nr_end - stripe_nr_orig);
2968 stripe_index = do_div(stripe_nr, map->num_stripes);
2969 } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
2970 if (rw & (REQ_WRITE | REQ_DISCARD))
2971 num_stripes = map->num_stripes;
2972 else if (mirror_num)
2973 stripe_index = mirror_num - 1;
2974 else {
2975 stripe_index = find_live_mirror(map, 0,
2976 map->num_stripes,
2977 current->pid % map->num_stripes);
2978 }
2979
2980 } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
2981 if (rw & (REQ_WRITE | REQ_DISCARD))
2982 num_stripes = map->num_stripes;
2983 else if (mirror_num)
2984 stripe_index = mirror_num - 1;
2985
2986 } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
2987 int factor = map->num_stripes / map->sub_stripes;
2988
2989 stripe_index = do_div(stripe_nr, factor);
2990 stripe_index *= map->sub_stripes;
2991
2992 if (rw & REQ_WRITE)
2993 num_stripes = map->sub_stripes;
2994 else if (rw & REQ_DISCARD)
2995 num_stripes = min_t(u64, map->sub_stripes *
2996 (stripe_nr_end - stripe_nr_orig),
2997 map->num_stripes);
2998 else if (mirror_num)
2999 stripe_index += mirror_num - 1;
3000 else {
3001 stripe_index = find_live_mirror(map, stripe_index,
3002 map->sub_stripes, stripe_index +
3003 current->pid % map->sub_stripes);
3004 }
3005 } else {
3006 /*
3007 * after this do_div call, stripe_nr is the number of stripes
3008 * on this device we have to walk to find the data, and
3009 * stripe_index is the number of our device in the stripe array
3010 */
3011 stripe_index = do_div(stripe_nr, map->num_stripes);
3012 }
3013 BUG_ON(stripe_index >= map->num_stripes);
3014
3015 if (rw & REQ_DISCARD) {
3016 for (i = 0; i < num_stripes; i++) {
3017 multi->stripes[i].physical =
3018 map->stripes[stripe_index].physical +
3019 stripe_offset + stripe_nr * map->stripe_len;
3020 multi->stripes[i].dev = map->stripes[stripe_index].dev;
3021
3022 if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
3023 u64 stripes;
3024 u32 last_stripe = 0;
3025 int j;
3026
3027 div_u64_rem(stripe_nr_end - 1,
3028 map->num_stripes,
3029 &last_stripe);
3030
3031 for (j = 0; j < map->num_stripes; j++) {
3032 u32 test;
3033
3034 div_u64_rem(stripe_nr_end - 1 - j,
3035 map->num_stripes, &test);
3036 if (test == stripe_index)
3037 break;
3038 }
3039 stripes = stripe_nr_end - 1 - j;
3040 do_div(stripes, map->num_stripes);
3041 multi->stripes[i].length = map->stripe_len *
3042 (stripes - stripe_nr + 1);
3043
3044 if (i == 0) {
3045 multi->stripes[i].length -=
3046 stripe_offset;
3047 stripe_offset = 0;
3048 }
3049 if (stripe_index == last_stripe)
3050 multi->stripes[i].length -=
3051 stripe_end_offset;
3052 } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
3053 u64 stripes;
3054 int j;
3055 int factor = map->num_stripes /
3056 map->sub_stripes;
3057 u32 last_stripe = 0;
3058
3059 div_u64_rem(stripe_nr_end - 1,
3060 factor, &last_stripe);
3061 last_stripe *= map->sub_stripes;
3062
3063 for (j = 0; j < factor; j++) {
3064 u32 test;
3065
3066 div_u64_rem(stripe_nr_end - 1 - j,
3067 factor, &test);
3068
3069 if (test ==
3070 stripe_index / map->sub_stripes)
3071 break;
3072 }
3073 stripes = stripe_nr_end - 1 - j;
3074 do_div(stripes, factor);
3075 multi->stripes[i].length = map->stripe_len *
3076 (stripes - stripe_nr + 1);
3077
3078 if (i < map->sub_stripes) {
3079 multi->stripes[i].length -=
3080 stripe_offset;
3081 if (i == map->sub_stripes - 1)
3082 stripe_offset = 0;
3083 }
3084 if (stripe_index >= last_stripe &&
3085 stripe_index <= (last_stripe +
3086 map->sub_stripes - 1)) {
3087 multi->stripes[i].length -=
3088 stripe_end_offset;
3089 }
3090 } else
3091 multi->stripes[i].length = *length;
3092
3093 stripe_index++;
3094 if (stripe_index == map->num_stripes) {
3095 /* This could only happen for RAID0/10 */
3096 stripe_index = 0;
3097 stripe_nr++;
3098 }
3099 }
3100 } else {
3101 for (i = 0; i < num_stripes; i++) {
3102 multi->stripes[i].physical =
3103 map->stripes[stripe_index].physical +
3104 stripe_offset +
3105 stripe_nr * map->stripe_len;
3106 multi->stripes[i].dev =
3107 map->stripes[stripe_index].dev;
3108 stripe_index++;
3109 }
3110 }
3111 if (multi_ret) {
3112 *multi_ret = multi;
3113 multi->num_stripes = num_stripes;
3114 multi->max_errors = max_errors;
3115 }
3116out:
3117 free_extent_map(em);
3118 return 0;
3119}
3120
3121int btrfs_map_block(struct btrfs_mapping_tree *map_tree, int rw,
3122 u64 logical, u64 *length,
3123 struct btrfs_multi_bio **multi_ret, int mirror_num)
3124{
3125 return __btrfs_map_block(map_tree, rw, logical, length, multi_ret,
3126 mirror_num);
3127}
3128
3129int btrfs_rmap_block(struct btrfs_mapping_tree *map_tree,
3130 u64 chunk_start, u64 physical, u64 devid,
3131 u64 **logical, int *naddrs, int *stripe_len)
3132{
3133 struct extent_map_tree *em_tree = &map_tree->map_tree;
3134 struct extent_map *em;
3135 struct map_lookup *map;
3136 u64 *buf;
3137 u64 bytenr;
3138 u64 length;
3139 u64 stripe_nr;
3140 int i, j, nr = 0;
3141
3142 read_lock(&em_tree->lock);
3143 em = lookup_extent_mapping(em_tree, chunk_start, 1);
3144 read_unlock(&em_tree->lock);
3145
3146 BUG_ON(!em || em->start != chunk_start);
3147 map = (struct map_lookup *)em->bdev;
3148
3149 length = em->len;
3150 if (map->type & BTRFS_BLOCK_GROUP_RAID10)
3151 do_div(length, map->num_stripes / map->sub_stripes);
3152 else if (map->type & BTRFS_BLOCK_GROUP_RAID0)
3153 do_div(length, map->num_stripes);
3154
3155 buf = kzalloc(sizeof(u64) * map->num_stripes, GFP_NOFS);
3156 BUG_ON(!buf);
3157
3158 for (i = 0; i < map->num_stripes; i++) {
3159 if (devid && map->stripes[i].dev->devid != devid)
3160 continue;
3161 if (map->stripes[i].physical > physical ||
3162 map->stripes[i].physical + length <= physical)
3163 continue;
3164
3165 stripe_nr = physical - map->stripes[i].physical;
3166 do_div(stripe_nr, map->stripe_len);
3167
3168 if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
3169 stripe_nr = stripe_nr * map->num_stripes + i;
3170 do_div(stripe_nr, map->sub_stripes);
3171 } else if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
3172 stripe_nr = stripe_nr * map->num_stripes + i;
3173 }
3174 bytenr = chunk_start + stripe_nr * map->stripe_len;
3175 WARN_ON(nr >= map->num_stripes);
3176 for (j = 0; j < nr; j++) {
3177 if (buf[j] == bytenr)
3178 break;
3179 }
3180 if (j == nr) {
3181 WARN_ON(nr >= map->num_stripes);
3182 buf[nr++] = bytenr;
3183 }
3184 }
3185
3186 *logical = buf;
3187 *naddrs = nr;
3188 *stripe_len = map->stripe_len;
3189
3190 free_extent_map(em);
3191 return 0;
3192}
3193
3194static void end_bio_multi_stripe(struct bio *bio, int err)
3195{
3196 struct btrfs_multi_bio *multi = bio->bi_private;
3197 int is_orig_bio = 0;
3198
3199 if (err)
3200 atomic_inc(&multi->error);
3201
3202 if (bio == multi->orig_bio)
3203 is_orig_bio = 1;
3204
3205 if (atomic_dec_and_test(&multi->stripes_pending)) {
3206 if (!is_orig_bio) {
3207 bio_put(bio);
3208 bio = multi->orig_bio;
3209 }
3210 bio->bi_private = multi->private;
3211 bio->bi_end_io = multi->end_io;
3212 /* only send an error to the higher layers if it is
3213 * beyond the tolerance of the multi-bio
3214 */
3215 if (atomic_read(&multi->error) > multi->max_errors) {
3216 err = -EIO;
3217 } else if (err) {
3218 /*
3219 * this bio is actually up to date, we didn't
3220 * go over the max number of errors
3221 */
3222 set_bit(BIO_UPTODATE, &bio->bi_flags);
3223 err = 0;
3224 }
3225 kfree(multi);
3226
3227 bio_endio(bio, err);
3228 } else if (!is_orig_bio) {
3229 bio_put(bio);
3230 }
3231}
3232
3233struct async_sched {
3234 struct bio *bio;
3235 int rw;
3236 struct btrfs_fs_info *info;
3237 struct btrfs_work work;
3238};
3239
3240/*
3241 * see run_scheduled_bios for a description of why bios are collected for
3242 * async submit.
3243 *
3244 * This will add one bio to the pending list for a device and make sure
3245 * the work struct is scheduled.
3246 */
3247static noinline int schedule_bio(struct btrfs_root *root,
3248 struct btrfs_device *device,
3249 int rw, struct bio *bio)
3250{
3251 int should_queue = 1;
3252 struct btrfs_pending_bios *pending_bios;
3253
3254 /* don't bother with additional async steps for reads, right now */
3255 if (!(rw & REQ_WRITE)) {
3256 bio_get(bio);
3257 submit_bio(rw, bio);
3258 bio_put(bio);
3259 return 0;
3260 }
3261
3262 /*
3263 * nr_async_bios allows us to reliably return congestion to the
3264 * higher layers. Otherwise, the async bio makes it appear we have
3265 * made progress against dirty pages when we've really just put it
3266 * on a queue for later
3267 */
3268 atomic_inc(&root->fs_info->nr_async_bios);
3269 WARN_ON(bio->bi_next);
3270 bio->bi_next = NULL;
3271 bio->bi_rw |= rw;
3272
3273 spin_lock(&device->io_lock);
3274 if (bio->bi_rw & REQ_SYNC)
3275 pending_bios = &device->pending_sync_bios;
3276 else
3277 pending_bios = &device->pending_bios;
3278
3279 if (pending_bios->tail)
3280 pending_bios->tail->bi_next = bio;
3281
3282 pending_bios->tail = bio;
3283 if (!pending_bios->head)
3284 pending_bios->head = bio;
3285 if (device->running_pending)
3286 should_queue = 0;
3287
3288 spin_unlock(&device->io_lock);
3289
3290 if (should_queue)
3291 btrfs_queue_worker(&root->fs_info->submit_workers,
3292 &device->work);
3293 return 0;
3294}
3295
3296int btrfs_map_bio(struct btrfs_root *root, int rw, struct bio *bio,
3297 int mirror_num, int async_submit)
3298{
3299 struct btrfs_mapping_tree *map_tree;
3300 struct btrfs_device *dev;
3301 struct bio *first_bio = bio;
3302 u64 logical = (u64)bio->bi_sector << 9;
3303 u64 length = 0;
3304 u64 map_length;
3305 struct btrfs_multi_bio *multi = NULL;
3306 int ret;
3307 int dev_nr = 0;
3308 int total_devs = 1;
3309
3310 length = bio->bi_size;
3311 map_tree = &root->fs_info->mapping_tree;
3312 map_length = length;
3313
3314 ret = btrfs_map_block(map_tree, rw, logical, &map_length, &multi,
3315 mirror_num);
3316 BUG_ON(ret);
3317
3318 total_devs = multi->num_stripes;
3319 if (map_length < length) {
3320 printk(KERN_CRIT "mapping failed logical %llu bio len %llu "
3321 "len %llu\n", (unsigned long long)logical,
3322 (unsigned long long)length,
3323 (unsigned long long)map_length);
3324 BUG();
3325 }
3326 multi->end_io = first_bio->bi_end_io;
3327 multi->private = first_bio->bi_private;
3328 multi->orig_bio = first_bio;
3329 atomic_set(&multi->stripes_pending, multi->num_stripes);
3330
3331 while (dev_nr < total_devs) {
3332 if (total_devs > 1) {
3333 if (dev_nr < total_devs - 1) {
3334 bio = bio_clone(first_bio, GFP_NOFS);
3335 BUG_ON(!bio);
3336 } else {
3337 bio = first_bio;
3338 }
3339 bio->bi_private = multi;
3340 bio->bi_end_io = end_bio_multi_stripe;
3341 }
3342 bio->bi_sector = multi->stripes[dev_nr].physical >> 9;
3343 dev = multi->stripes[dev_nr].dev;
3344 if (dev && dev->bdev && (rw != WRITE || dev->writeable)) {
3345 bio->bi_bdev = dev->bdev;
3346 if (async_submit)
3347 schedule_bio(root, dev, rw, bio);
3348 else
3349 submit_bio(rw, bio);
3350 } else {
3351 bio->bi_bdev = root->fs_info->fs_devices->latest_bdev;
3352 bio->bi_sector = logical >> 9;
3353 bio_endio(bio, -EIO);
3354 }
3355 dev_nr++;
3356 }
3357 if (total_devs == 1)
3358 kfree(multi);
3359 return 0;
3360}
3361
3362struct btrfs_device *btrfs_find_device(struct btrfs_root *root, u64 devid,
3363 u8 *uuid, u8 *fsid)
3364{
3365 struct btrfs_device *device;
3366 struct btrfs_fs_devices *cur_devices;
3367
3368 cur_devices = root->fs_info->fs_devices;
3369 while (cur_devices) {
3370 if (!fsid ||
3371 !memcmp(cur_devices->fsid, fsid, BTRFS_UUID_SIZE)) {
3372 device = __find_device(&cur_devices->devices,
3373 devid, uuid);
3374 if (device)
3375 return device;
3376 }
3377 cur_devices = cur_devices->seed;
3378 }
3379 return NULL;
3380}
3381
3382static struct btrfs_device *add_missing_dev(struct btrfs_root *root,
3383 u64 devid, u8 *dev_uuid)
3384{
3385 struct btrfs_device *device;
3386 struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
3387
3388 device = kzalloc(sizeof(*device), GFP_NOFS);
3389 if (!device)
3390 return NULL;
3391 list_add(&device->dev_list,
3392 &fs_devices->devices);
3393 device->dev_root = root->fs_info->dev_root;
3394 device->devid = devid;
3395 device->work.func = pending_bios_fn;
3396 device->fs_devices = fs_devices;
3397 device->missing = 1;
3398 fs_devices->num_devices++;
3399 fs_devices->missing_devices++;
3400 spin_lock_init(&device->io_lock);
3401 INIT_LIST_HEAD(&device->dev_alloc_list);
3402 memcpy(device->uuid, dev_uuid, BTRFS_UUID_SIZE);
3403 return device;
3404}
3405
3406static int read_one_chunk(struct btrfs_root *root, struct btrfs_key *key,
3407 struct extent_buffer *leaf,
3408 struct btrfs_chunk *chunk)
3409{
3410 struct btrfs_mapping_tree *map_tree = &root->fs_info->mapping_tree;
3411 struct map_lookup *map;
3412 struct extent_map *em;
3413 u64 logical;
3414 u64 length;
3415 u64 devid;
3416 u8 uuid[BTRFS_UUID_SIZE];
3417 int num_stripes;
3418 int ret;
3419 int i;
3420
3421 logical = key->offset;
3422 length = btrfs_chunk_length(leaf, chunk);
3423
3424 read_lock(&map_tree->map_tree.lock);
3425 em = lookup_extent_mapping(&map_tree->map_tree, logical, 1);
3426 read_unlock(&map_tree->map_tree.lock);
3427
3428 /* already mapped? */
3429 if (em && em->start <= logical && em->start + em->len > logical) {
3430 free_extent_map(em);
3431 return 0;
3432 } else if (em) {
3433 free_extent_map(em);
3434 }
3435
3436 em = alloc_extent_map();
3437 if (!em)
3438 return -ENOMEM;
3439 num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
3440 map = kmalloc(map_lookup_size(num_stripes), GFP_NOFS);
3441 if (!map) {
3442 free_extent_map(em);
3443 return -ENOMEM;
3444 }
3445
3446 em->bdev = (struct block_device *)map;
3447 em->start = logical;
3448 em->len = length;
3449 em->block_start = 0;
3450 em->block_len = em->len;
3451
3452 map->num_stripes = num_stripes;
3453 map->io_width = btrfs_chunk_io_width(leaf, chunk);
3454 map->io_align = btrfs_chunk_io_align(leaf, chunk);
3455 map->sector_size = btrfs_chunk_sector_size(leaf, chunk);
3456 map->stripe_len = btrfs_chunk_stripe_len(leaf, chunk);
3457 map->type = btrfs_chunk_type(leaf, chunk);
3458 map->sub_stripes = btrfs_chunk_sub_stripes(leaf, chunk);
3459 for (i = 0; i < num_stripes; i++) {
3460 map->stripes[i].physical =
3461 btrfs_stripe_offset_nr(leaf, chunk, i);
3462 devid = btrfs_stripe_devid_nr(leaf, chunk, i);
3463 read_extent_buffer(leaf, uuid, (unsigned long)
3464 btrfs_stripe_dev_uuid_nr(chunk, i),
3465 BTRFS_UUID_SIZE);
3466 map->stripes[i].dev = btrfs_find_device(root, devid, uuid,
3467 NULL);
3468 if (!map->stripes[i].dev && !btrfs_test_opt(root, DEGRADED)) {
3469 kfree(map);
3470 free_extent_map(em);
3471 return -EIO;
3472 }
3473 if (!map->stripes[i].dev) {
3474 map->stripes[i].dev =
3475 add_missing_dev(root, devid, uuid);
3476 if (!map->stripes[i].dev) {
3477 kfree(map);
3478 free_extent_map(em);
3479 return -EIO;
3480 }
3481 }
3482 map->stripes[i].dev->in_fs_metadata = 1;
3483 }
3484
3485 write_lock(&map_tree->map_tree.lock);
3486 ret = add_extent_mapping(&map_tree->map_tree, em);
3487 write_unlock(&map_tree->map_tree.lock);
3488 BUG_ON(ret);
3489 free_extent_map(em);
3490
3491 return 0;
3492}
3493
3494static int fill_device_from_item(struct extent_buffer *leaf,
3495 struct btrfs_dev_item *dev_item,
3496 struct btrfs_device *device)
3497{
3498 unsigned long ptr;
3499
3500 device->devid = btrfs_device_id(leaf, dev_item);
3501 device->disk_total_bytes = btrfs_device_total_bytes(leaf, dev_item);
3502 device->total_bytes = device->disk_total_bytes;
3503 device->bytes_used = btrfs_device_bytes_used(leaf, dev_item);
3504 device->type = btrfs_device_type(leaf, dev_item);
3505 device->io_align = btrfs_device_io_align(leaf, dev_item);
3506 device->io_width = btrfs_device_io_width(leaf, dev_item);
3507 device->sector_size = btrfs_device_sector_size(leaf, dev_item);
3508
3509 ptr = (unsigned long)btrfs_device_uuid(dev_item);
3510 read_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
3511
3512 return 0;
3513}
3514
3515static int open_seed_devices(struct btrfs_root *root, u8 *fsid)
3516{
3517 struct btrfs_fs_devices *fs_devices;
3518 int ret;
3519
3520 mutex_lock(&uuid_mutex);
3521
3522 fs_devices = root->fs_info->fs_devices->seed;
3523 while (fs_devices) {
3524 if (!memcmp(fs_devices->fsid, fsid, BTRFS_UUID_SIZE)) {
3525 ret = 0;
3526 goto out;
3527 }
3528 fs_devices = fs_devices->seed;
3529 }
3530
3531 fs_devices = find_fsid(fsid);
3532 if (!fs_devices) {
3533 ret = -ENOENT;
3534 goto out;
3535 }
3536
3537 fs_devices = clone_fs_devices(fs_devices);
3538 if (IS_ERR(fs_devices)) {
3539 ret = PTR_ERR(fs_devices);
3540 goto out;
3541 }
3542
3543 ret = __btrfs_open_devices(fs_devices, FMODE_READ,
3544 root->fs_info->bdev_holder);
3545 if (ret)
3546 goto out;
3547
3548 if (!fs_devices->seeding) {
3549 __btrfs_close_devices(fs_devices);
3550 free_fs_devices(fs_devices);
3551 ret = -EINVAL;
3552 goto out;
3553 }
3554
3555 fs_devices->seed = root->fs_info->fs_devices->seed;
3556 root->fs_info->fs_devices->seed = fs_devices;
3557out:
3558 mutex_unlock(&uuid_mutex);
3559 return ret;
3560}
3561
3562static int read_one_dev(struct btrfs_root *root,
3563 struct extent_buffer *leaf,
3564 struct btrfs_dev_item *dev_item)
3565{
3566 struct btrfs_device *device;
3567 u64 devid;
3568 int ret;
3569 u8 fs_uuid[BTRFS_UUID_SIZE];
3570 u8 dev_uuid[BTRFS_UUID_SIZE];
3571
3572 devid = btrfs_device_id(leaf, dev_item);
3573 read_extent_buffer(leaf, dev_uuid,
3574 (unsigned long)btrfs_device_uuid(dev_item),
3575 BTRFS_UUID_SIZE);
3576 read_extent_buffer(leaf, fs_uuid,
3577 (unsigned long)btrfs_device_fsid(dev_item),
3578 BTRFS_UUID_SIZE);
3579
3580 if (memcmp(fs_uuid, root->fs_info->fsid, BTRFS_UUID_SIZE)) {
3581 ret = open_seed_devices(root, fs_uuid);
3582 if (ret && !btrfs_test_opt(root, DEGRADED))
3583 return ret;
3584 }
3585
3586 device = btrfs_find_device(root, devid, dev_uuid, fs_uuid);
3587 if (!device || !device->bdev) {
3588 if (!btrfs_test_opt(root, DEGRADED))
3589 return -EIO;
3590
3591 if (!device) {
3592 printk(KERN_WARNING "warning devid %llu missing\n",
3593 (unsigned long long)devid);
3594 device = add_missing_dev(root, devid, dev_uuid);
3595 if (!device)
3596 return -ENOMEM;
3597 } else if (!device->missing) {
3598 /*
3599 * this happens when a device that was properly setup
3600 * in the device info lists suddenly goes bad.
3601 * device->bdev is NULL, and so we have to set
3602 * device->missing to one here
3603 */
3604 root->fs_info->fs_devices->missing_devices++;
3605 device->missing = 1;
3606 }
3607 }
3608
3609 if (device->fs_devices != root->fs_info->fs_devices) {
3610 BUG_ON(device->writeable);
3611 if (device->generation !=
3612 btrfs_device_generation(leaf, dev_item))
3613 return -EINVAL;
3614 }
3615
3616 fill_device_from_item(leaf, dev_item, device);
3617 device->dev_root = root->fs_info->dev_root;
3618 device->in_fs_metadata = 1;
3619 if (device->writeable)
3620 device->fs_devices->total_rw_bytes += device->total_bytes;
3621 ret = 0;
3622 return ret;
3623}
3624
3625int btrfs_read_sys_array(struct btrfs_root *root)
3626{
3627 struct btrfs_super_block *super_copy = &root->fs_info->super_copy;
3628 struct extent_buffer *sb;
3629 struct btrfs_disk_key *disk_key;
3630 struct btrfs_chunk *chunk;
3631 u8 *ptr;
3632 unsigned long sb_ptr;
3633 int ret = 0;
3634 u32 num_stripes;
3635 u32 array_size;
3636 u32 len = 0;
3637 u32 cur;
3638 struct btrfs_key key;
3639
3640 sb = btrfs_find_create_tree_block(root, BTRFS_SUPER_INFO_OFFSET,
3641 BTRFS_SUPER_INFO_SIZE);
3642 if (!sb)
3643 return -ENOMEM;
3644 btrfs_set_buffer_uptodate(sb);
3645 btrfs_set_buffer_lockdep_class(root->root_key.objectid, sb, 0);
3646
3647 write_extent_buffer(sb, super_copy, 0, BTRFS_SUPER_INFO_SIZE);
3648 array_size = btrfs_super_sys_array_size(super_copy);
3649
3650 ptr = super_copy->sys_chunk_array;
3651 sb_ptr = offsetof(struct btrfs_super_block, sys_chunk_array);
3652 cur = 0;
3653
3654 while (cur < array_size) {
3655 disk_key = (struct btrfs_disk_key *)ptr;
3656 btrfs_disk_key_to_cpu(&key, disk_key);
3657
3658 len = sizeof(*disk_key); ptr += len;
3659 sb_ptr += len;
3660 cur += len;
3661
3662 if (key.type == BTRFS_CHUNK_ITEM_KEY) {
3663 chunk = (struct btrfs_chunk *)sb_ptr;
3664 ret = read_one_chunk(root, &key, sb, chunk);
3665 if (ret)
3666 break;
3667 num_stripes = btrfs_chunk_num_stripes(sb, chunk);
3668 len = btrfs_chunk_item_size(num_stripes);
3669 } else {
3670 ret = -EIO;
3671 break;
3672 }
3673 ptr += len;
3674 sb_ptr += len;
3675 cur += len;
3676 }
3677 free_extent_buffer(sb);
3678 return ret;
3679}
3680
3681int btrfs_read_chunk_tree(struct btrfs_root *root)
3682{
3683 struct btrfs_path *path;
3684 struct extent_buffer *leaf;
3685 struct btrfs_key key;
3686 struct btrfs_key found_key;
3687 int ret;
3688 int slot;
3689
3690 root = root->fs_info->chunk_root;
3691
3692 path = btrfs_alloc_path();
3693 if (!path)
3694 return -ENOMEM;
3695
3696 /* first we search for all of the device items, and then we
3697 * read in all of the chunk items. This way we can create chunk
3698 * mappings that reference all of the devices that are afound
3699 */
3700 key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
3701 key.offset = 0;
3702 key.type = 0;
3703again:
3704 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3705 if (ret < 0)
3706 goto error;
3707 while (1) {
3708 leaf = path->nodes[0];
3709 slot = path->slots[0];
3710 if (slot >= btrfs_header_nritems(leaf)) {
3711 ret = btrfs_next_leaf(root, path);
3712 if (ret == 0)
3713 continue;
3714 if (ret < 0)
3715 goto error;
3716 break;
3717 }
3718 btrfs_item_key_to_cpu(leaf, &found_key, slot);
3719 if (key.objectid == BTRFS_DEV_ITEMS_OBJECTID) {
3720 if (found_key.objectid != BTRFS_DEV_ITEMS_OBJECTID)
3721 break;
3722 if (found_key.type == BTRFS_DEV_ITEM_KEY) {
3723 struct btrfs_dev_item *dev_item;
3724 dev_item = btrfs_item_ptr(leaf, slot,
3725 struct btrfs_dev_item);
3726 ret = read_one_dev(root, leaf, dev_item);
3727 if (ret)
3728 goto error;
3729 }
3730 } else if (found_key.type == BTRFS_CHUNK_ITEM_KEY) {
3731 struct btrfs_chunk *chunk;
3732 chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
3733 ret = read_one_chunk(root, &found_key, leaf, chunk);
3734 if (ret)
3735 goto error;
3736 }
3737 path->slots[0]++;
3738 }
3739 if (key.objectid == BTRFS_DEV_ITEMS_OBJECTID) {
3740 key.objectid = 0;
3741 btrfs_release_path(path);
3742 goto again;
3743 }
3744 ret = 0;
3745error:
3746 btrfs_free_path(path);
3747 return ret;
3748}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2007 Oracle. All rights reserved.
4 */
5
6#include <linux/sched.h>
7#include <linux/sched/mm.h>
8#include <linux/slab.h>
9#include <linux/ratelimit.h>
10#include <linux/kthread.h>
11#include <linux/semaphore.h>
12#include <linux/uuid.h>
13#include <linux/list_sort.h>
14#include <linux/namei.h>
15#include "misc.h"
16#include "ctree.h"
17#include "disk-io.h"
18#include "transaction.h"
19#include "volumes.h"
20#include "raid56.h"
21#include "rcu-string.h"
22#include "dev-replace.h"
23#include "sysfs.h"
24#include "tree-checker.h"
25#include "space-info.h"
26#include "block-group.h"
27#include "discard.h"
28#include "zoned.h"
29#include "fs.h"
30#include "accessors.h"
31#include "uuid-tree.h"
32#include "ioctl.h"
33#include "relocation.h"
34#include "scrub.h"
35#include "super.h"
36#include "raid-stripe-tree.h"
37
38#define BTRFS_BLOCK_GROUP_STRIPE_MASK (BTRFS_BLOCK_GROUP_RAID0 | \
39 BTRFS_BLOCK_GROUP_RAID10 | \
40 BTRFS_BLOCK_GROUP_RAID56_MASK)
41
42struct btrfs_io_geometry {
43 u32 stripe_index;
44 u32 stripe_nr;
45 int mirror_num;
46 int num_stripes;
47 u64 stripe_offset;
48 u64 raid56_full_stripe_start;
49 int max_errors;
50 enum btrfs_map_op op;
51};
52
53const struct btrfs_raid_attr btrfs_raid_array[BTRFS_NR_RAID_TYPES] = {
54 [BTRFS_RAID_RAID10] = {
55 .sub_stripes = 2,
56 .dev_stripes = 1,
57 .devs_max = 0, /* 0 == as many as possible */
58 .devs_min = 2,
59 .tolerated_failures = 1,
60 .devs_increment = 2,
61 .ncopies = 2,
62 .nparity = 0,
63 .raid_name = "raid10",
64 .bg_flag = BTRFS_BLOCK_GROUP_RAID10,
65 .mindev_error = BTRFS_ERROR_DEV_RAID10_MIN_NOT_MET,
66 },
67 [BTRFS_RAID_RAID1] = {
68 .sub_stripes = 1,
69 .dev_stripes = 1,
70 .devs_max = 2,
71 .devs_min = 2,
72 .tolerated_failures = 1,
73 .devs_increment = 2,
74 .ncopies = 2,
75 .nparity = 0,
76 .raid_name = "raid1",
77 .bg_flag = BTRFS_BLOCK_GROUP_RAID1,
78 .mindev_error = BTRFS_ERROR_DEV_RAID1_MIN_NOT_MET,
79 },
80 [BTRFS_RAID_RAID1C3] = {
81 .sub_stripes = 1,
82 .dev_stripes = 1,
83 .devs_max = 3,
84 .devs_min = 3,
85 .tolerated_failures = 2,
86 .devs_increment = 3,
87 .ncopies = 3,
88 .nparity = 0,
89 .raid_name = "raid1c3",
90 .bg_flag = BTRFS_BLOCK_GROUP_RAID1C3,
91 .mindev_error = BTRFS_ERROR_DEV_RAID1C3_MIN_NOT_MET,
92 },
93 [BTRFS_RAID_RAID1C4] = {
94 .sub_stripes = 1,
95 .dev_stripes = 1,
96 .devs_max = 4,
97 .devs_min = 4,
98 .tolerated_failures = 3,
99 .devs_increment = 4,
100 .ncopies = 4,
101 .nparity = 0,
102 .raid_name = "raid1c4",
103 .bg_flag = BTRFS_BLOCK_GROUP_RAID1C4,
104 .mindev_error = BTRFS_ERROR_DEV_RAID1C4_MIN_NOT_MET,
105 },
106 [BTRFS_RAID_DUP] = {
107 .sub_stripes = 1,
108 .dev_stripes = 2,
109 .devs_max = 1,
110 .devs_min = 1,
111 .tolerated_failures = 0,
112 .devs_increment = 1,
113 .ncopies = 2,
114 .nparity = 0,
115 .raid_name = "dup",
116 .bg_flag = BTRFS_BLOCK_GROUP_DUP,
117 .mindev_error = 0,
118 },
119 [BTRFS_RAID_RAID0] = {
120 .sub_stripes = 1,
121 .dev_stripes = 1,
122 .devs_max = 0,
123 .devs_min = 1,
124 .tolerated_failures = 0,
125 .devs_increment = 1,
126 .ncopies = 1,
127 .nparity = 0,
128 .raid_name = "raid0",
129 .bg_flag = BTRFS_BLOCK_GROUP_RAID0,
130 .mindev_error = 0,
131 },
132 [BTRFS_RAID_SINGLE] = {
133 .sub_stripes = 1,
134 .dev_stripes = 1,
135 .devs_max = 1,
136 .devs_min = 1,
137 .tolerated_failures = 0,
138 .devs_increment = 1,
139 .ncopies = 1,
140 .nparity = 0,
141 .raid_name = "single",
142 .bg_flag = 0,
143 .mindev_error = 0,
144 },
145 [BTRFS_RAID_RAID5] = {
146 .sub_stripes = 1,
147 .dev_stripes = 1,
148 .devs_max = 0,
149 .devs_min = 2,
150 .tolerated_failures = 1,
151 .devs_increment = 1,
152 .ncopies = 1,
153 .nparity = 1,
154 .raid_name = "raid5",
155 .bg_flag = BTRFS_BLOCK_GROUP_RAID5,
156 .mindev_error = BTRFS_ERROR_DEV_RAID5_MIN_NOT_MET,
157 },
158 [BTRFS_RAID_RAID6] = {
159 .sub_stripes = 1,
160 .dev_stripes = 1,
161 .devs_max = 0,
162 .devs_min = 3,
163 .tolerated_failures = 2,
164 .devs_increment = 1,
165 .ncopies = 1,
166 .nparity = 2,
167 .raid_name = "raid6",
168 .bg_flag = BTRFS_BLOCK_GROUP_RAID6,
169 .mindev_error = BTRFS_ERROR_DEV_RAID6_MIN_NOT_MET,
170 },
171};
172
173/*
174 * Convert block group flags (BTRFS_BLOCK_GROUP_*) to btrfs_raid_types, which
175 * can be used as index to access btrfs_raid_array[].
176 */
177enum btrfs_raid_types __attribute_const__ btrfs_bg_flags_to_raid_index(u64 flags)
178{
179 const u64 profile = (flags & BTRFS_BLOCK_GROUP_PROFILE_MASK);
180
181 if (!profile)
182 return BTRFS_RAID_SINGLE;
183
184 return BTRFS_BG_FLAG_TO_INDEX(profile);
185}
186
187const char *btrfs_bg_type_to_raid_name(u64 flags)
188{
189 const int index = btrfs_bg_flags_to_raid_index(flags);
190
191 if (index >= BTRFS_NR_RAID_TYPES)
192 return NULL;
193
194 return btrfs_raid_array[index].raid_name;
195}
196
197int btrfs_nr_parity_stripes(u64 type)
198{
199 enum btrfs_raid_types index = btrfs_bg_flags_to_raid_index(type);
200
201 return btrfs_raid_array[index].nparity;
202}
203
204/*
205 * Fill @buf with textual description of @bg_flags, no more than @size_buf
206 * bytes including terminating null byte.
207 */
208void btrfs_describe_block_groups(u64 bg_flags, char *buf, u32 size_buf)
209{
210 int i;
211 int ret;
212 char *bp = buf;
213 u64 flags = bg_flags;
214 u32 size_bp = size_buf;
215
216 if (!flags) {
217 strcpy(bp, "NONE");
218 return;
219 }
220
221#define DESCRIBE_FLAG(flag, desc) \
222 do { \
223 if (flags & (flag)) { \
224 ret = snprintf(bp, size_bp, "%s|", (desc)); \
225 if (ret < 0 || ret >= size_bp) \
226 goto out_overflow; \
227 size_bp -= ret; \
228 bp += ret; \
229 flags &= ~(flag); \
230 } \
231 } while (0)
232
233 DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_DATA, "data");
234 DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_SYSTEM, "system");
235 DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_METADATA, "metadata");
236
237 DESCRIBE_FLAG(BTRFS_AVAIL_ALLOC_BIT_SINGLE, "single");
238 for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
239 DESCRIBE_FLAG(btrfs_raid_array[i].bg_flag,
240 btrfs_raid_array[i].raid_name);
241#undef DESCRIBE_FLAG
242
243 if (flags) {
244 ret = snprintf(bp, size_bp, "0x%llx|", flags);
245 size_bp -= ret;
246 }
247
248 if (size_bp < size_buf)
249 buf[size_buf - size_bp - 1] = '\0'; /* remove last | */
250
251 /*
252 * The text is trimmed, it's up to the caller to provide sufficiently
253 * large buffer
254 */
255out_overflow:;
256}
257
258static int init_first_rw_device(struct btrfs_trans_handle *trans);
259static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info);
260static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);
261
262/*
263 * Device locking
264 * ==============
265 *
266 * There are several mutexes that protect manipulation of devices and low-level
267 * structures like chunks but not block groups, extents or files
268 *
269 * uuid_mutex (global lock)
270 * ------------------------
271 * protects the fs_uuids list that tracks all per-fs fs_devices, resulting from
272 * the SCAN_DEV ioctl registration or from mount either implicitly (the first
273 * device) or requested by the device= mount option
274 *
275 * the mutex can be very coarse and can cover long-running operations
276 *
277 * protects: updates to fs_devices counters like missing devices, rw devices,
278 * seeding, structure cloning, opening/closing devices at mount/umount time
279 *
280 * global::fs_devs - add, remove, updates to the global list
281 *
282 * does not protect: manipulation of the fs_devices::devices list in general
283 * but in mount context it could be used to exclude list modifications by eg.
284 * scan ioctl
285 *
286 * btrfs_device::name - renames (write side), read is RCU
287 *
288 * fs_devices::device_list_mutex (per-fs, with RCU)
289 * ------------------------------------------------
290 * protects updates to fs_devices::devices, ie. adding and deleting
291 *
292 * simple list traversal with read-only actions can be done with RCU protection
293 *
294 * may be used to exclude some operations from running concurrently without any
295 * modifications to the list (see write_all_supers)
296 *
297 * Is not required at mount and close times, because our device list is
298 * protected by the uuid_mutex at that point.
299 *
300 * balance_mutex
301 * -------------
302 * protects balance structures (status, state) and context accessed from
303 * several places (internally, ioctl)
304 *
305 * chunk_mutex
306 * -----------
307 * protects chunks, adding or removing during allocation, trim or when a new
308 * device is added/removed. Additionally it also protects post_commit_list of
309 * individual devices, since they can be added to the transaction's
310 * post_commit_list only with chunk_mutex held.
311 *
312 * cleaner_mutex
313 * -------------
314 * a big lock that is held by the cleaner thread and prevents running subvolume
315 * cleaning together with relocation or delayed iputs
316 *
317 *
318 * Lock nesting
319 * ============
320 *
321 * uuid_mutex
322 * device_list_mutex
323 * chunk_mutex
324 * balance_mutex
325 *
326 *
327 * Exclusive operations
328 * ====================
329 *
330 * Maintains the exclusivity of the following operations that apply to the
331 * whole filesystem and cannot run in parallel.
332 *
333 * - Balance (*)
334 * - Device add
335 * - Device remove
336 * - Device replace (*)
337 * - Resize
338 *
339 * The device operations (as above) can be in one of the following states:
340 *
341 * - Running state
342 * - Paused state
343 * - Completed state
344 *
345 * Only device operations marked with (*) can go into the Paused state for the
346 * following reasons:
347 *
348 * - ioctl (only Balance can be Paused through ioctl)
349 * - filesystem remounted as read-only
350 * - filesystem unmounted and mounted as read-only
351 * - system power-cycle and filesystem mounted as read-only
352 * - filesystem or device errors leading to forced read-only
353 *
354 * The status of exclusive operation is set and cleared atomically.
355 * During the course of Paused state, fs_info::exclusive_operation remains set.
356 * A device operation in Paused or Running state can be canceled or resumed
357 * either by ioctl (Balance only) or when remounted as read-write.
358 * The exclusive status is cleared when the device operation is canceled or
359 * completed.
360 */
361
362DEFINE_MUTEX(uuid_mutex);
363static LIST_HEAD(fs_uuids);
364struct list_head * __attribute_const__ btrfs_get_fs_uuids(void)
365{
366 return &fs_uuids;
367}
368
369/*
370 * Allocate new btrfs_fs_devices structure identified by a fsid.
371 *
372 * @fsid: if not NULL, copy the UUID to fs_devices::fsid and to
373 * fs_devices::metadata_fsid
374 *
375 * Return a pointer to a new struct btrfs_fs_devices on success, or ERR_PTR().
376 * The returned struct is not linked onto any lists and can be destroyed with
377 * kfree() right away.
378 */
379static struct btrfs_fs_devices *alloc_fs_devices(const u8 *fsid)
380{
381 struct btrfs_fs_devices *fs_devs;
382
383 fs_devs = kzalloc(sizeof(*fs_devs), GFP_KERNEL);
384 if (!fs_devs)
385 return ERR_PTR(-ENOMEM);
386
387 mutex_init(&fs_devs->device_list_mutex);
388
389 INIT_LIST_HEAD(&fs_devs->devices);
390 INIT_LIST_HEAD(&fs_devs->alloc_list);
391 INIT_LIST_HEAD(&fs_devs->fs_list);
392 INIT_LIST_HEAD(&fs_devs->seed_list);
393
394 if (fsid) {
395 memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE);
396 memcpy(fs_devs->metadata_uuid, fsid, BTRFS_FSID_SIZE);
397 }
398
399 return fs_devs;
400}
401
402static void btrfs_free_device(struct btrfs_device *device)
403{
404 WARN_ON(!list_empty(&device->post_commit_list));
405 rcu_string_free(device->name);
406 extent_io_tree_release(&device->alloc_state);
407 btrfs_destroy_dev_zone_info(device);
408 kfree(device);
409}
410
411static void free_fs_devices(struct btrfs_fs_devices *fs_devices)
412{
413 struct btrfs_device *device;
414
415 WARN_ON(fs_devices->opened);
416 while (!list_empty(&fs_devices->devices)) {
417 device = list_entry(fs_devices->devices.next,
418 struct btrfs_device, dev_list);
419 list_del(&device->dev_list);
420 btrfs_free_device(device);
421 }
422 kfree(fs_devices);
423}
424
425void __exit btrfs_cleanup_fs_uuids(void)
426{
427 struct btrfs_fs_devices *fs_devices;
428
429 while (!list_empty(&fs_uuids)) {
430 fs_devices = list_entry(fs_uuids.next,
431 struct btrfs_fs_devices, fs_list);
432 list_del(&fs_devices->fs_list);
433 free_fs_devices(fs_devices);
434 }
435}
436
437static bool match_fsid_fs_devices(const struct btrfs_fs_devices *fs_devices,
438 const u8 *fsid, const u8 *metadata_fsid)
439{
440 if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) != 0)
441 return false;
442
443 if (!metadata_fsid)
444 return true;
445
446 if (memcmp(metadata_fsid, fs_devices->metadata_uuid, BTRFS_FSID_SIZE) != 0)
447 return false;
448
449 return true;
450}
451
452static noinline struct btrfs_fs_devices *find_fsid(
453 const u8 *fsid, const u8 *metadata_fsid)
454{
455 struct btrfs_fs_devices *fs_devices;
456
457 ASSERT(fsid);
458
459 /* Handle non-split brain cases */
460 list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
461 if (match_fsid_fs_devices(fs_devices, fsid, metadata_fsid))
462 return fs_devices;
463 }
464 return NULL;
465}
466
467static int
468btrfs_get_bdev_and_sb(const char *device_path, blk_mode_t flags, void *holder,
469 int flush, struct file **bdev_file,
470 struct btrfs_super_block **disk_super)
471{
472 struct block_device *bdev;
473 int ret;
474
475 *bdev_file = bdev_file_open_by_path(device_path, flags, holder, NULL);
476
477 if (IS_ERR(*bdev_file)) {
478 ret = PTR_ERR(*bdev_file);
479 btrfs_err(NULL, "failed to open device for path %s with flags 0x%x: %d",
480 device_path, flags, ret);
481 goto error;
482 }
483 bdev = file_bdev(*bdev_file);
484
485 if (flush)
486 sync_blockdev(bdev);
487 if (holder) {
488 ret = set_blocksize(*bdev_file, BTRFS_BDEV_BLOCKSIZE);
489 if (ret) {
490 fput(*bdev_file);
491 goto error;
492 }
493 }
494 invalidate_bdev(bdev);
495 *disk_super = btrfs_read_dev_super(bdev);
496 if (IS_ERR(*disk_super)) {
497 ret = PTR_ERR(*disk_super);
498 fput(*bdev_file);
499 goto error;
500 }
501
502 return 0;
503
504error:
505 *disk_super = NULL;
506 *bdev_file = NULL;
507 return ret;
508}
509
510/*
511 * Search and remove all stale devices (which are not mounted). When both
512 * inputs are NULL, it will search and release all stale devices.
513 *
514 * @devt: Optional. When provided will it release all unmounted devices
515 * matching this devt only.
516 * @skip_device: Optional. Will skip this device when searching for the stale
517 * devices.
518 *
519 * Return: 0 for success or if @devt is 0.
520 * -EBUSY if @devt is a mounted device.
521 * -ENOENT if @devt does not match any device in the list.
522 */
523static int btrfs_free_stale_devices(dev_t devt, struct btrfs_device *skip_device)
524{
525 struct btrfs_fs_devices *fs_devices, *tmp_fs_devices;
526 struct btrfs_device *device, *tmp_device;
527 int ret;
528 bool freed = false;
529
530 lockdep_assert_held(&uuid_mutex);
531
532 /* Return good status if there is no instance of devt. */
533 ret = 0;
534 list_for_each_entry_safe(fs_devices, tmp_fs_devices, &fs_uuids, fs_list) {
535
536 mutex_lock(&fs_devices->device_list_mutex);
537 list_for_each_entry_safe(device, tmp_device,
538 &fs_devices->devices, dev_list) {
539 if (skip_device && skip_device == device)
540 continue;
541 if (devt && devt != device->devt)
542 continue;
543 if (fs_devices->opened) {
544 if (devt)
545 ret = -EBUSY;
546 break;
547 }
548
549 /* delete the stale device */
550 fs_devices->num_devices--;
551 list_del(&device->dev_list);
552 btrfs_free_device(device);
553
554 freed = true;
555 }
556 mutex_unlock(&fs_devices->device_list_mutex);
557
558 if (fs_devices->num_devices == 0) {
559 btrfs_sysfs_remove_fsid(fs_devices);
560 list_del(&fs_devices->fs_list);
561 free_fs_devices(fs_devices);
562 }
563 }
564
565 /* If there is at least one freed device return 0. */
566 if (freed)
567 return 0;
568
569 return ret;
570}
571
572static struct btrfs_fs_devices *find_fsid_by_device(
573 struct btrfs_super_block *disk_super,
574 dev_t devt, bool *same_fsid_diff_dev)
575{
576 struct btrfs_fs_devices *fsid_fs_devices;
577 struct btrfs_fs_devices *devt_fs_devices;
578 const bool has_metadata_uuid = (btrfs_super_incompat_flags(disk_super) &
579 BTRFS_FEATURE_INCOMPAT_METADATA_UUID);
580 bool found_by_devt = false;
581
582 /* Find the fs_device by the usual method, if found use it. */
583 fsid_fs_devices = find_fsid(disk_super->fsid,
584 has_metadata_uuid ? disk_super->metadata_uuid : NULL);
585
586 /* The temp_fsid feature is supported only with single device filesystem. */
587 if (btrfs_super_num_devices(disk_super) != 1)
588 return fsid_fs_devices;
589
590 /*
591 * A seed device is an integral component of the sprout device, which
592 * functions as a multi-device filesystem. So, temp-fsid feature is
593 * not supported.
594 */
595 if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_SEEDING)
596 return fsid_fs_devices;
597
598 /* Try to find a fs_devices by matching devt. */
599 list_for_each_entry(devt_fs_devices, &fs_uuids, fs_list) {
600 struct btrfs_device *device;
601
602 list_for_each_entry(device, &devt_fs_devices->devices, dev_list) {
603 if (device->devt == devt) {
604 found_by_devt = true;
605 break;
606 }
607 }
608 if (found_by_devt)
609 break;
610 }
611
612 if (found_by_devt) {
613 /* Existing device. */
614 if (fsid_fs_devices == NULL) {
615 if (devt_fs_devices->opened == 0) {
616 /* Stale device. */
617 return NULL;
618 } else {
619 /* temp_fsid is mounting a subvol. */
620 return devt_fs_devices;
621 }
622 } else {
623 /* Regular or temp_fsid device mounting a subvol. */
624 return devt_fs_devices;
625 }
626 } else {
627 /* New device. */
628 if (fsid_fs_devices == NULL) {
629 return NULL;
630 } else {
631 /* sb::fsid is already used create a new temp_fsid. */
632 *same_fsid_diff_dev = true;
633 return NULL;
634 }
635 }
636
637 /* Not reached. */
638}
639
640/*
641 * This is only used on mount, and we are protected from competing things
642 * messing with our fs_devices by the uuid_mutex, thus we do not need the
643 * fs_devices->device_list_mutex here.
644 */
645static int btrfs_open_one_device(struct btrfs_fs_devices *fs_devices,
646 struct btrfs_device *device, blk_mode_t flags,
647 void *holder)
648{
649 struct file *bdev_file;
650 struct btrfs_super_block *disk_super;
651 u64 devid;
652 int ret;
653
654 if (device->bdev)
655 return -EINVAL;
656 if (!device->name)
657 return -EINVAL;
658
659 ret = btrfs_get_bdev_and_sb(device->name->str, flags, holder, 1,
660 &bdev_file, &disk_super);
661 if (ret)
662 return ret;
663
664 devid = btrfs_stack_device_id(&disk_super->dev_item);
665 if (devid != device->devid)
666 goto error_free_page;
667
668 if (memcmp(device->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE))
669 goto error_free_page;
670
671 device->generation = btrfs_super_generation(disk_super);
672
673 if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_SEEDING) {
674 if (btrfs_super_incompat_flags(disk_super) &
675 BTRFS_FEATURE_INCOMPAT_METADATA_UUID) {
676 pr_err(
677 "BTRFS: Invalid seeding and uuid-changed device detected\n");
678 goto error_free_page;
679 }
680
681 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
682 fs_devices->seeding = true;
683 } else {
684 if (bdev_read_only(file_bdev(bdev_file)))
685 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
686 else
687 set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
688 }
689
690 if (!bdev_nonrot(file_bdev(bdev_file)))
691 fs_devices->rotating = true;
692
693 if (bdev_max_discard_sectors(file_bdev(bdev_file)))
694 fs_devices->discardable = true;
695
696 device->bdev_file = bdev_file;
697 device->bdev = file_bdev(bdev_file);
698 clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
699
700 if (device->devt != device->bdev->bd_dev) {
701 btrfs_warn(NULL,
702 "device %s maj:min changed from %d:%d to %d:%d",
703 device->name->str, MAJOR(device->devt),
704 MINOR(device->devt), MAJOR(device->bdev->bd_dev),
705 MINOR(device->bdev->bd_dev));
706
707 device->devt = device->bdev->bd_dev;
708 }
709
710 fs_devices->open_devices++;
711 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
712 device->devid != BTRFS_DEV_REPLACE_DEVID) {
713 fs_devices->rw_devices++;
714 list_add_tail(&device->dev_alloc_list, &fs_devices->alloc_list);
715 }
716 btrfs_release_disk_super(disk_super);
717
718 return 0;
719
720error_free_page:
721 btrfs_release_disk_super(disk_super);
722 fput(bdev_file);
723
724 return -EINVAL;
725}
726
727const u8 *btrfs_sb_fsid_ptr(const struct btrfs_super_block *sb)
728{
729 bool has_metadata_uuid = (btrfs_super_incompat_flags(sb) &
730 BTRFS_FEATURE_INCOMPAT_METADATA_UUID);
731
732 return has_metadata_uuid ? sb->metadata_uuid : sb->fsid;
733}
734
735/*
736 * We can have very weird soft links passed in.
737 * One example is "/proc/self/fd/<fd>", which can be a soft link to
738 * a block device.
739 *
740 * But it's never a good idea to use those weird names.
741 * Here we check if the path (not following symlinks) is a good one inside
742 * "/dev/".
743 */
744static bool is_good_dev_path(const char *dev_path)
745{
746 struct path path = { .mnt = NULL, .dentry = NULL };
747 char *path_buf = NULL;
748 char *resolved_path;
749 bool is_good = false;
750 int ret;
751
752 if (!dev_path)
753 goto out;
754
755 path_buf = kmalloc(PATH_MAX, GFP_KERNEL);
756 if (!path_buf)
757 goto out;
758
759 /*
760 * Do not follow soft link, just check if the original path is inside
761 * "/dev/".
762 */
763 ret = kern_path(dev_path, 0, &path);
764 if (ret)
765 goto out;
766 resolved_path = d_path(&path, path_buf, PATH_MAX);
767 if (IS_ERR(resolved_path))
768 goto out;
769 if (strncmp(resolved_path, "/dev/", strlen("/dev/")))
770 goto out;
771 is_good = true;
772out:
773 kfree(path_buf);
774 path_put(&path);
775 return is_good;
776}
777
778static int get_canonical_dev_path(const char *dev_path, char *canonical)
779{
780 struct path path = { .mnt = NULL, .dentry = NULL };
781 char *path_buf = NULL;
782 char *resolved_path;
783 int ret;
784
785 if (!dev_path) {
786 ret = -EINVAL;
787 goto out;
788 }
789
790 path_buf = kmalloc(PATH_MAX, GFP_KERNEL);
791 if (!path_buf) {
792 ret = -ENOMEM;
793 goto out;
794 }
795
796 ret = kern_path(dev_path, LOOKUP_FOLLOW, &path);
797 if (ret)
798 goto out;
799 resolved_path = d_path(&path, path_buf, PATH_MAX);
800 if (IS_ERR(resolved_path)) {
801 ret = PTR_ERR(resolved_path);
802 goto out;
803 }
804 ret = strscpy(canonical, resolved_path, PATH_MAX);
805out:
806 kfree(path_buf);
807 path_put(&path);
808 return ret;
809}
810
811static bool is_same_device(struct btrfs_device *device, const char *new_path)
812{
813 struct path old = { .mnt = NULL, .dentry = NULL };
814 struct path new = { .mnt = NULL, .dentry = NULL };
815 char *old_path = NULL;
816 bool is_same = false;
817 int ret;
818
819 if (!device->name)
820 goto out;
821
822 old_path = kzalloc(PATH_MAX, GFP_NOFS);
823 if (!old_path)
824 goto out;
825
826 rcu_read_lock();
827 ret = strscpy(old_path, rcu_str_deref(device->name), PATH_MAX);
828 rcu_read_unlock();
829 if (ret < 0)
830 goto out;
831
832 ret = kern_path(old_path, LOOKUP_FOLLOW, &old);
833 if (ret)
834 goto out;
835 ret = kern_path(new_path, LOOKUP_FOLLOW, &new);
836 if (ret)
837 goto out;
838 if (path_equal(&old, &new))
839 is_same = true;
840out:
841 kfree(old_path);
842 path_put(&old);
843 path_put(&new);
844 return is_same;
845}
846
847/*
848 * Add new device to list of registered devices
849 *
850 * Returns:
851 * device pointer which was just added or updated when successful
852 * error pointer when failed
853 */
854static noinline struct btrfs_device *device_list_add(const char *path,
855 struct btrfs_super_block *disk_super,
856 bool *new_device_added)
857{
858 struct btrfs_device *device;
859 struct btrfs_fs_devices *fs_devices = NULL;
860 struct rcu_string *name;
861 u64 found_transid = btrfs_super_generation(disk_super);
862 u64 devid = btrfs_stack_device_id(&disk_super->dev_item);
863 dev_t path_devt;
864 int error;
865 bool same_fsid_diff_dev = false;
866 bool has_metadata_uuid = (btrfs_super_incompat_flags(disk_super) &
867 BTRFS_FEATURE_INCOMPAT_METADATA_UUID);
868
869 if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_CHANGING_FSID_V2) {
870 btrfs_err(NULL,
871"device %s has incomplete metadata_uuid change, please use btrfstune to complete",
872 path);
873 return ERR_PTR(-EAGAIN);
874 }
875
876 error = lookup_bdev(path, &path_devt);
877 if (error) {
878 btrfs_err(NULL, "failed to lookup block device for path %s: %d",
879 path, error);
880 return ERR_PTR(error);
881 }
882
883 fs_devices = find_fsid_by_device(disk_super, path_devt, &same_fsid_diff_dev);
884
885 if (!fs_devices) {
886 fs_devices = alloc_fs_devices(disk_super->fsid);
887 if (IS_ERR(fs_devices))
888 return ERR_CAST(fs_devices);
889
890 if (has_metadata_uuid)
891 memcpy(fs_devices->metadata_uuid,
892 disk_super->metadata_uuid, BTRFS_FSID_SIZE);
893
894 if (same_fsid_diff_dev) {
895 generate_random_uuid(fs_devices->fsid);
896 fs_devices->temp_fsid = true;
897 pr_info("BTRFS: device %s (%d:%d) using temp-fsid %pU\n",
898 path, MAJOR(path_devt), MINOR(path_devt),
899 fs_devices->fsid);
900 }
901
902 mutex_lock(&fs_devices->device_list_mutex);
903 list_add(&fs_devices->fs_list, &fs_uuids);
904
905 device = NULL;
906 } else {
907 struct btrfs_dev_lookup_args args = {
908 .devid = devid,
909 .uuid = disk_super->dev_item.uuid,
910 };
911
912 mutex_lock(&fs_devices->device_list_mutex);
913 device = btrfs_find_device(fs_devices, &args);
914
915 if (found_transid > fs_devices->latest_generation) {
916 memcpy(fs_devices->fsid, disk_super->fsid,
917 BTRFS_FSID_SIZE);
918 memcpy(fs_devices->metadata_uuid,
919 btrfs_sb_fsid_ptr(disk_super), BTRFS_FSID_SIZE);
920 }
921 }
922
923 if (!device) {
924 unsigned int nofs_flag;
925
926 if (fs_devices->opened) {
927 btrfs_err(NULL,
928"device %s (%d:%d) belongs to fsid %pU, and the fs is already mounted, scanned by %s (%d)",
929 path, MAJOR(path_devt), MINOR(path_devt),
930 fs_devices->fsid, current->comm,
931 task_pid_nr(current));
932 mutex_unlock(&fs_devices->device_list_mutex);
933 return ERR_PTR(-EBUSY);
934 }
935
936 nofs_flag = memalloc_nofs_save();
937 device = btrfs_alloc_device(NULL, &devid,
938 disk_super->dev_item.uuid, path);
939 memalloc_nofs_restore(nofs_flag);
940 if (IS_ERR(device)) {
941 mutex_unlock(&fs_devices->device_list_mutex);
942 /* we can safely leave the fs_devices entry around */
943 return device;
944 }
945
946 device->devt = path_devt;
947
948 list_add_rcu(&device->dev_list, &fs_devices->devices);
949 fs_devices->num_devices++;
950
951 device->fs_devices = fs_devices;
952 *new_device_added = true;
953
954 if (disk_super->label[0])
955 pr_info(
956"BTRFS: device label %s devid %llu transid %llu %s (%d:%d) scanned by %s (%d)\n",
957 disk_super->label, devid, found_transid, path,
958 MAJOR(path_devt), MINOR(path_devt),
959 current->comm, task_pid_nr(current));
960 else
961 pr_info(
962"BTRFS: device fsid %pU devid %llu transid %llu %s (%d:%d) scanned by %s (%d)\n",
963 disk_super->fsid, devid, found_transid, path,
964 MAJOR(path_devt), MINOR(path_devt),
965 current->comm, task_pid_nr(current));
966
967 } else if (!device->name || !is_same_device(device, path)) {
968 /*
969 * When FS is already mounted.
970 * 1. If you are here and if the device->name is NULL that
971 * means this device was missing at time of FS mount.
972 * 2. If you are here and if the device->name is different
973 * from 'path' that means either
974 * a. The same device disappeared and reappeared with
975 * different name. or
976 * b. The missing-disk-which-was-replaced, has
977 * reappeared now.
978 *
979 * We must allow 1 and 2a above. But 2b would be a spurious
980 * and unintentional.
981 *
982 * Further in case of 1 and 2a above, the disk at 'path'
983 * would have missed some transaction when it was away and
984 * in case of 2a the stale bdev has to be updated as well.
985 * 2b must not be allowed at all time.
986 */
987
988 /*
989 * For now, we do allow update to btrfs_fs_device through the
990 * btrfs dev scan cli after FS has been mounted. We're still
991 * tracking a problem where systems fail mount by subvolume id
992 * when we reject replacement on a mounted FS.
993 */
994 if (!fs_devices->opened && found_transid < device->generation) {
995 /*
996 * That is if the FS is _not_ mounted and if you
997 * are here, that means there is more than one
998 * disk with same uuid and devid.We keep the one
999 * with larger generation number or the last-in if
1000 * generation are equal.
1001 */
1002 mutex_unlock(&fs_devices->device_list_mutex);
1003 btrfs_err(NULL,
1004"device %s already registered with a higher generation, found %llu expect %llu",
1005 path, found_transid, device->generation);
1006 return ERR_PTR(-EEXIST);
1007 }
1008
1009 /*
1010 * We are going to replace the device path for a given devid,
1011 * make sure it's the same device if the device is mounted
1012 *
1013 * NOTE: the device->fs_info may not be reliable here so pass
1014 * in a NULL to message helpers instead. This avoids a possible
1015 * use-after-free when the fs_info and fs_info->sb are already
1016 * torn down.
1017 */
1018 if (device->bdev) {
1019 if (device->devt != path_devt) {
1020 mutex_unlock(&fs_devices->device_list_mutex);
1021 btrfs_warn_in_rcu(NULL,
1022 "duplicate device %s devid %llu generation %llu scanned by %s (%d)",
1023 path, devid, found_transid,
1024 current->comm,
1025 task_pid_nr(current));
1026 return ERR_PTR(-EEXIST);
1027 }
1028 btrfs_info_in_rcu(NULL,
1029 "devid %llu device path %s changed to %s scanned by %s (%d)",
1030 devid, btrfs_dev_name(device),
1031 path, current->comm,
1032 task_pid_nr(current));
1033 }
1034
1035 name = rcu_string_strdup(path, GFP_NOFS);
1036 if (!name) {
1037 mutex_unlock(&fs_devices->device_list_mutex);
1038 return ERR_PTR(-ENOMEM);
1039 }
1040 rcu_string_free(device->name);
1041 rcu_assign_pointer(device->name, name);
1042 if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
1043 fs_devices->missing_devices--;
1044 clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
1045 }
1046 device->devt = path_devt;
1047 }
1048
1049 /*
1050 * Unmount does not free the btrfs_device struct but would zero
1051 * generation along with most of the other members. So just update
1052 * it back. We need it to pick the disk with largest generation
1053 * (as above).
1054 */
1055 if (!fs_devices->opened) {
1056 device->generation = found_transid;
1057 fs_devices->latest_generation = max_t(u64, found_transid,
1058 fs_devices->latest_generation);
1059 }
1060
1061 fs_devices->total_devices = btrfs_super_num_devices(disk_super);
1062
1063 mutex_unlock(&fs_devices->device_list_mutex);
1064 return device;
1065}
1066
1067static struct btrfs_fs_devices *clone_fs_devices(struct btrfs_fs_devices *orig)
1068{
1069 struct btrfs_fs_devices *fs_devices;
1070 struct btrfs_device *device;
1071 struct btrfs_device *orig_dev;
1072 int ret = 0;
1073
1074 lockdep_assert_held(&uuid_mutex);
1075
1076 fs_devices = alloc_fs_devices(orig->fsid);
1077 if (IS_ERR(fs_devices))
1078 return fs_devices;
1079
1080 fs_devices->total_devices = orig->total_devices;
1081
1082 list_for_each_entry(orig_dev, &orig->devices, dev_list) {
1083 const char *dev_path = NULL;
1084
1085 /*
1086 * This is ok to do without RCU read locked because we hold the
1087 * uuid mutex so nothing we touch in here is going to disappear.
1088 */
1089 if (orig_dev->name)
1090 dev_path = orig_dev->name->str;
1091
1092 device = btrfs_alloc_device(NULL, &orig_dev->devid,
1093 orig_dev->uuid, dev_path);
1094 if (IS_ERR(device)) {
1095 ret = PTR_ERR(device);
1096 goto error;
1097 }
1098
1099 if (orig_dev->zone_info) {
1100 struct btrfs_zoned_device_info *zone_info;
1101
1102 zone_info = btrfs_clone_dev_zone_info(orig_dev);
1103 if (!zone_info) {
1104 btrfs_free_device(device);
1105 ret = -ENOMEM;
1106 goto error;
1107 }
1108 device->zone_info = zone_info;
1109 }
1110
1111 list_add(&device->dev_list, &fs_devices->devices);
1112 device->fs_devices = fs_devices;
1113 fs_devices->num_devices++;
1114 }
1115 return fs_devices;
1116error:
1117 free_fs_devices(fs_devices);
1118 return ERR_PTR(ret);
1119}
1120
1121static void __btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices,
1122 struct btrfs_device **latest_dev)
1123{
1124 struct btrfs_device *device, *next;
1125
1126 /* This is the initialized path, it is safe to release the devices. */
1127 list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) {
1128 if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state)) {
1129 if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
1130 &device->dev_state) &&
1131 !test_bit(BTRFS_DEV_STATE_MISSING,
1132 &device->dev_state) &&
1133 (!*latest_dev ||
1134 device->generation > (*latest_dev)->generation)) {
1135 *latest_dev = device;
1136 }
1137 continue;
1138 }
1139
1140 /*
1141 * We have already validated the presence of BTRFS_DEV_REPLACE_DEVID,
1142 * in btrfs_init_dev_replace() so just continue.
1143 */
1144 if (device->devid == BTRFS_DEV_REPLACE_DEVID)
1145 continue;
1146
1147 if (device->bdev_file) {
1148 fput(device->bdev_file);
1149 device->bdev = NULL;
1150 device->bdev_file = NULL;
1151 fs_devices->open_devices--;
1152 }
1153 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1154 list_del_init(&device->dev_alloc_list);
1155 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
1156 fs_devices->rw_devices--;
1157 }
1158 list_del_init(&device->dev_list);
1159 fs_devices->num_devices--;
1160 btrfs_free_device(device);
1161 }
1162
1163}
1164
1165/*
1166 * After we have read the system tree and know devids belonging to this
1167 * filesystem, remove the device which does not belong there.
1168 */
1169void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices)
1170{
1171 struct btrfs_device *latest_dev = NULL;
1172 struct btrfs_fs_devices *seed_dev;
1173
1174 mutex_lock(&uuid_mutex);
1175 __btrfs_free_extra_devids(fs_devices, &latest_dev);
1176
1177 list_for_each_entry(seed_dev, &fs_devices->seed_list, seed_list)
1178 __btrfs_free_extra_devids(seed_dev, &latest_dev);
1179
1180 fs_devices->latest_dev = latest_dev;
1181
1182 mutex_unlock(&uuid_mutex);
1183}
1184
1185static void btrfs_close_bdev(struct btrfs_device *device)
1186{
1187 if (!device->bdev)
1188 return;
1189
1190 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1191 sync_blockdev(device->bdev);
1192 invalidate_bdev(device->bdev);
1193 }
1194
1195 fput(device->bdev_file);
1196}
1197
1198static void btrfs_close_one_device(struct btrfs_device *device)
1199{
1200 struct btrfs_fs_devices *fs_devices = device->fs_devices;
1201
1202 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
1203 device->devid != BTRFS_DEV_REPLACE_DEVID) {
1204 list_del_init(&device->dev_alloc_list);
1205 fs_devices->rw_devices--;
1206 }
1207
1208 if (device->devid == BTRFS_DEV_REPLACE_DEVID)
1209 clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
1210
1211 if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
1212 clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
1213 fs_devices->missing_devices--;
1214 }
1215
1216 btrfs_close_bdev(device);
1217 if (device->bdev) {
1218 fs_devices->open_devices--;
1219 device->bdev = NULL;
1220 device->bdev_file = NULL;
1221 }
1222 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
1223 btrfs_destroy_dev_zone_info(device);
1224
1225 device->fs_info = NULL;
1226 atomic_set(&device->dev_stats_ccnt, 0);
1227 extent_io_tree_release(&device->alloc_state);
1228
1229 /*
1230 * Reset the flush error record. We might have a transient flush error
1231 * in this mount, and if so we aborted the current transaction and set
1232 * the fs to an error state, guaranteeing no super blocks can be further
1233 * committed. However that error might be transient and if we unmount the
1234 * filesystem and mount it again, we should allow the mount to succeed
1235 * (btrfs_check_rw_degradable() should not fail) - if after mounting the
1236 * filesystem again we still get flush errors, then we will again abort
1237 * any transaction and set the error state, guaranteeing no commits of
1238 * unsafe super blocks.
1239 */
1240 device->last_flush_error = 0;
1241
1242 /* Verify the device is back in a pristine state */
1243 WARN_ON(test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state));
1244 WARN_ON(test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state));
1245 WARN_ON(!list_empty(&device->dev_alloc_list));
1246 WARN_ON(!list_empty(&device->post_commit_list));
1247}
1248
1249static void close_fs_devices(struct btrfs_fs_devices *fs_devices)
1250{
1251 struct btrfs_device *device, *tmp;
1252
1253 lockdep_assert_held(&uuid_mutex);
1254
1255 if (--fs_devices->opened > 0)
1256 return;
1257
1258 list_for_each_entry_safe(device, tmp, &fs_devices->devices, dev_list)
1259 btrfs_close_one_device(device);
1260
1261 WARN_ON(fs_devices->open_devices);
1262 WARN_ON(fs_devices->rw_devices);
1263 fs_devices->opened = 0;
1264 fs_devices->seeding = false;
1265 fs_devices->fs_info = NULL;
1266}
1267
1268void btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
1269{
1270 LIST_HEAD(list);
1271 struct btrfs_fs_devices *tmp;
1272
1273 mutex_lock(&uuid_mutex);
1274 close_fs_devices(fs_devices);
1275 if (!fs_devices->opened) {
1276 list_splice_init(&fs_devices->seed_list, &list);
1277
1278 /*
1279 * If the struct btrfs_fs_devices is not assembled with any
1280 * other device, it can be re-initialized during the next mount
1281 * without the needing device-scan step. Therefore, it can be
1282 * fully freed.
1283 */
1284 if (fs_devices->num_devices == 1) {
1285 list_del(&fs_devices->fs_list);
1286 free_fs_devices(fs_devices);
1287 }
1288 }
1289
1290
1291 list_for_each_entry_safe(fs_devices, tmp, &list, seed_list) {
1292 close_fs_devices(fs_devices);
1293 list_del(&fs_devices->seed_list);
1294 free_fs_devices(fs_devices);
1295 }
1296 mutex_unlock(&uuid_mutex);
1297}
1298
1299static int open_fs_devices(struct btrfs_fs_devices *fs_devices,
1300 blk_mode_t flags, void *holder)
1301{
1302 struct btrfs_device *device;
1303 struct btrfs_device *latest_dev = NULL;
1304 struct btrfs_device *tmp_device;
1305 int ret = 0;
1306
1307 list_for_each_entry_safe(device, tmp_device, &fs_devices->devices,
1308 dev_list) {
1309 int ret2;
1310
1311 ret2 = btrfs_open_one_device(fs_devices, device, flags, holder);
1312 if (ret2 == 0 &&
1313 (!latest_dev || device->generation > latest_dev->generation)) {
1314 latest_dev = device;
1315 } else if (ret2 == -ENODATA) {
1316 fs_devices->num_devices--;
1317 list_del(&device->dev_list);
1318 btrfs_free_device(device);
1319 }
1320 if (ret == 0 && ret2 != 0)
1321 ret = ret2;
1322 }
1323
1324 if (fs_devices->open_devices == 0) {
1325 if (ret)
1326 return ret;
1327 return -EINVAL;
1328 }
1329
1330 fs_devices->opened = 1;
1331 fs_devices->latest_dev = latest_dev;
1332 fs_devices->total_rw_bytes = 0;
1333 fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_REGULAR;
1334 fs_devices->read_policy = BTRFS_READ_POLICY_PID;
1335
1336 return 0;
1337}
1338
1339static int devid_cmp(void *priv, const struct list_head *a,
1340 const struct list_head *b)
1341{
1342 const struct btrfs_device *dev1, *dev2;
1343
1344 dev1 = list_entry(a, struct btrfs_device, dev_list);
1345 dev2 = list_entry(b, struct btrfs_device, dev_list);
1346
1347 if (dev1->devid < dev2->devid)
1348 return -1;
1349 else if (dev1->devid > dev2->devid)
1350 return 1;
1351 return 0;
1352}
1353
1354int btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
1355 blk_mode_t flags, void *holder)
1356{
1357 int ret;
1358
1359 lockdep_assert_held(&uuid_mutex);
1360 /*
1361 * The device_list_mutex cannot be taken here in case opening the
1362 * underlying device takes further locks like open_mutex.
1363 *
1364 * We also don't need the lock here as this is called during mount and
1365 * exclusion is provided by uuid_mutex
1366 */
1367
1368 if (fs_devices->opened) {
1369 fs_devices->opened++;
1370 ret = 0;
1371 } else {
1372 list_sort(NULL, &fs_devices->devices, devid_cmp);
1373 ret = open_fs_devices(fs_devices, flags, holder);
1374 }
1375
1376 return ret;
1377}
1378
1379void btrfs_release_disk_super(struct btrfs_super_block *super)
1380{
1381 struct page *page = virt_to_page(super);
1382
1383 put_page(page);
1384}
1385
1386static struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev,
1387 u64 bytenr, u64 bytenr_orig)
1388{
1389 struct btrfs_super_block *disk_super;
1390 struct page *page;
1391 void *p;
1392 pgoff_t index;
1393
1394 /* make sure our super fits in the device */
1395 if (bytenr + PAGE_SIZE >= bdev_nr_bytes(bdev))
1396 return ERR_PTR(-EINVAL);
1397
1398 /* make sure our super fits in the page */
1399 if (sizeof(*disk_super) > PAGE_SIZE)
1400 return ERR_PTR(-EINVAL);
1401
1402 /* make sure our super doesn't straddle pages on disk */
1403 index = bytenr >> PAGE_SHIFT;
1404 if ((bytenr + sizeof(*disk_super) - 1) >> PAGE_SHIFT != index)
1405 return ERR_PTR(-EINVAL);
1406
1407 /* pull in the page with our super */
1408 page = read_cache_page_gfp(bdev->bd_mapping, index, GFP_KERNEL);
1409
1410 if (IS_ERR(page))
1411 return ERR_CAST(page);
1412
1413 p = page_address(page);
1414
1415 /* align our pointer to the offset of the super block */
1416 disk_super = p + offset_in_page(bytenr);
1417
1418 if (btrfs_super_bytenr(disk_super) != bytenr_orig ||
1419 btrfs_super_magic(disk_super) != BTRFS_MAGIC) {
1420 btrfs_release_disk_super(p);
1421 return ERR_PTR(-EINVAL);
1422 }
1423
1424 if (disk_super->label[0] && disk_super->label[BTRFS_LABEL_SIZE - 1])
1425 disk_super->label[BTRFS_LABEL_SIZE - 1] = 0;
1426
1427 return disk_super;
1428}
1429
1430int btrfs_forget_devices(dev_t devt)
1431{
1432 int ret;
1433
1434 mutex_lock(&uuid_mutex);
1435 ret = btrfs_free_stale_devices(devt, NULL);
1436 mutex_unlock(&uuid_mutex);
1437
1438 return ret;
1439}
1440
1441static bool btrfs_skip_registration(struct btrfs_super_block *disk_super,
1442 const char *path, dev_t devt,
1443 bool mount_arg_dev)
1444{
1445 struct btrfs_fs_devices *fs_devices;
1446
1447 /*
1448 * Do not skip device registration for mounted devices with matching
1449 * maj:min but different paths. Booting without initrd relies on
1450 * /dev/root initially, later replaced with the actual root device.
1451 * A successful scan ensures grub2-probe selects the correct device.
1452 */
1453 list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
1454 struct btrfs_device *device;
1455
1456 mutex_lock(&fs_devices->device_list_mutex);
1457
1458 if (!fs_devices->opened) {
1459 mutex_unlock(&fs_devices->device_list_mutex);
1460 continue;
1461 }
1462
1463 list_for_each_entry(device, &fs_devices->devices, dev_list) {
1464 if (device->bdev && (device->bdev->bd_dev == devt) &&
1465 strcmp(device->name->str, path) != 0) {
1466 mutex_unlock(&fs_devices->device_list_mutex);
1467
1468 /* Do not skip registration. */
1469 return false;
1470 }
1471 }
1472 mutex_unlock(&fs_devices->device_list_mutex);
1473 }
1474
1475 if (!mount_arg_dev && btrfs_super_num_devices(disk_super) == 1 &&
1476 !(btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_SEEDING))
1477 return true;
1478
1479 return false;
1480}
1481
1482/*
1483 * Look for a btrfs signature on a device. This may be called out of the mount path
1484 * and we are not allowed to call set_blocksize during the scan. The superblock
1485 * is read via pagecache.
1486 *
1487 * With @mount_arg_dev it's a scan during mount time that will always register
1488 * the device or return an error. Multi-device and seeding devices are registered
1489 * in both cases.
1490 */
1491struct btrfs_device *btrfs_scan_one_device(const char *path, blk_mode_t flags,
1492 bool mount_arg_dev)
1493{
1494 struct btrfs_super_block *disk_super;
1495 bool new_device_added = false;
1496 struct btrfs_device *device = NULL;
1497 struct file *bdev_file;
1498 char *canonical_path = NULL;
1499 u64 bytenr;
1500 dev_t devt;
1501 int ret;
1502
1503 lockdep_assert_held(&uuid_mutex);
1504
1505 if (!is_good_dev_path(path)) {
1506 canonical_path = kmalloc(PATH_MAX, GFP_KERNEL);
1507 if (canonical_path) {
1508 ret = get_canonical_dev_path(path, canonical_path);
1509 if (ret < 0) {
1510 kfree(canonical_path);
1511 canonical_path = NULL;
1512 }
1513 }
1514 }
1515 /*
1516 * Avoid an exclusive open here, as the systemd-udev may initiate the
1517 * device scan which may race with the user's mount or mkfs command,
1518 * resulting in failure.
1519 * Since the device scan is solely for reading purposes, there is no
1520 * need for an exclusive open. Additionally, the devices are read again
1521 * during the mount process. It is ok to get some inconsistent
1522 * values temporarily, as the device paths of the fsid are the only
1523 * required information for assembling the volume.
1524 */
1525 bdev_file = bdev_file_open_by_path(path, flags, NULL, NULL);
1526 if (IS_ERR(bdev_file))
1527 return ERR_CAST(bdev_file);
1528
1529 /*
1530 * We would like to check all the super blocks, but doing so would
1531 * allow a mount to succeed after a mkfs from a different filesystem.
1532 * Currently, recovery from a bad primary btrfs superblock is done
1533 * using the userspace command 'btrfs check --super'.
1534 */
1535 ret = btrfs_sb_log_location_bdev(file_bdev(bdev_file), 0, READ, &bytenr);
1536 if (ret) {
1537 device = ERR_PTR(ret);
1538 goto error_bdev_put;
1539 }
1540
1541 disk_super = btrfs_read_disk_super(file_bdev(bdev_file), bytenr,
1542 btrfs_sb_offset(0));
1543 if (IS_ERR(disk_super)) {
1544 device = ERR_CAST(disk_super);
1545 goto error_bdev_put;
1546 }
1547
1548 devt = file_bdev(bdev_file)->bd_dev;
1549 if (btrfs_skip_registration(disk_super, path, devt, mount_arg_dev)) {
1550 pr_debug("BTRFS: skip registering single non-seed device %s (%d:%d)\n",
1551 path, MAJOR(devt), MINOR(devt));
1552
1553 btrfs_free_stale_devices(devt, NULL);
1554
1555 device = NULL;
1556 goto free_disk_super;
1557 }
1558
1559 device = device_list_add(canonical_path ? : path, disk_super,
1560 &new_device_added);
1561 if (!IS_ERR(device) && new_device_added)
1562 btrfs_free_stale_devices(device->devt, device);
1563
1564free_disk_super:
1565 btrfs_release_disk_super(disk_super);
1566
1567error_bdev_put:
1568 fput(bdev_file);
1569 kfree(canonical_path);
1570
1571 return device;
1572}
1573
1574/*
1575 * Try to find a chunk that intersects [start, start + len] range and when one
1576 * such is found, record the end of it in *start
1577 */
1578static bool contains_pending_extent(struct btrfs_device *device, u64 *start,
1579 u64 len)
1580{
1581 u64 physical_start, physical_end;
1582
1583 lockdep_assert_held(&device->fs_info->chunk_mutex);
1584
1585 if (find_first_extent_bit(&device->alloc_state, *start,
1586 &physical_start, &physical_end,
1587 CHUNK_ALLOCATED, NULL)) {
1588
1589 if (in_range(physical_start, *start, len) ||
1590 in_range(*start, physical_start,
1591 physical_end + 1 - physical_start)) {
1592 *start = physical_end + 1;
1593 return true;
1594 }
1595 }
1596 return false;
1597}
1598
1599static u64 dev_extent_search_start(struct btrfs_device *device)
1600{
1601 switch (device->fs_devices->chunk_alloc_policy) {
1602 case BTRFS_CHUNK_ALLOC_REGULAR:
1603 return BTRFS_DEVICE_RANGE_RESERVED;
1604 case BTRFS_CHUNK_ALLOC_ZONED:
1605 /*
1606 * We don't care about the starting region like regular
1607 * allocator, because we anyway use/reserve the first two zones
1608 * for superblock logging.
1609 */
1610 return 0;
1611 default:
1612 BUG();
1613 }
1614}
1615
1616static bool dev_extent_hole_check_zoned(struct btrfs_device *device,
1617 u64 *hole_start, u64 *hole_size,
1618 u64 num_bytes)
1619{
1620 u64 zone_size = device->zone_info->zone_size;
1621 u64 pos;
1622 int ret;
1623 bool changed = false;
1624
1625 ASSERT(IS_ALIGNED(*hole_start, zone_size));
1626
1627 while (*hole_size > 0) {
1628 pos = btrfs_find_allocatable_zones(device, *hole_start,
1629 *hole_start + *hole_size,
1630 num_bytes);
1631 if (pos != *hole_start) {
1632 *hole_size = *hole_start + *hole_size - pos;
1633 *hole_start = pos;
1634 changed = true;
1635 if (*hole_size < num_bytes)
1636 break;
1637 }
1638
1639 ret = btrfs_ensure_empty_zones(device, pos, num_bytes);
1640
1641 /* Range is ensured to be empty */
1642 if (!ret)
1643 return changed;
1644
1645 /* Given hole range was invalid (outside of device) */
1646 if (ret == -ERANGE) {
1647 *hole_start += *hole_size;
1648 *hole_size = 0;
1649 return true;
1650 }
1651
1652 *hole_start += zone_size;
1653 *hole_size -= zone_size;
1654 changed = true;
1655 }
1656
1657 return changed;
1658}
1659
1660/*
1661 * Check if specified hole is suitable for allocation.
1662 *
1663 * @device: the device which we have the hole
1664 * @hole_start: starting position of the hole
1665 * @hole_size: the size of the hole
1666 * @num_bytes: the size of the free space that we need
1667 *
1668 * This function may modify @hole_start and @hole_size to reflect the suitable
1669 * position for allocation. Returns 1 if hole position is updated, 0 otherwise.
1670 */
1671static bool dev_extent_hole_check(struct btrfs_device *device, u64 *hole_start,
1672 u64 *hole_size, u64 num_bytes)
1673{
1674 bool changed = false;
1675 u64 hole_end = *hole_start + *hole_size;
1676
1677 for (;;) {
1678 /*
1679 * Check before we set max_hole_start, otherwise we could end up
1680 * sending back this offset anyway.
1681 */
1682 if (contains_pending_extent(device, hole_start, *hole_size)) {
1683 if (hole_end >= *hole_start)
1684 *hole_size = hole_end - *hole_start;
1685 else
1686 *hole_size = 0;
1687 changed = true;
1688 }
1689
1690 switch (device->fs_devices->chunk_alloc_policy) {
1691 case BTRFS_CHUNK_ALLOC_REGULAR:
1692 /* No extra check */
1693 break;
1694 case BTRFS_CHUNK_ALLOC_ZONED:
1695 if (dev_extent_hole_check_zoned(device, hole_start,
1696 hole_size, num_bytes)) {
1697 changed = true;
1698 /*
1699 * The changed hole can contain pending extent.
1700 * Loop again to check that.
1701 */
1702 continue;
1703 }
1704 break;
1705 default:
1706 BUG();
1707 }
1708
1709 break;
1710 }
1711
1712 return changed;
1713}
1714
1715/*
1716 * Find free space in the specified device.
1717 *
1718 * @device: the device which we search the free space in
1719 * @num_bytes: the size of the free space that we need
1720 * @search_start: the position from which to begin the search
1721 * @start: store the start of the free space.
1722 * @len: the size of the free space. that we find, or the size
1723 * of the max free space if we don't find suitable free space
1724 *
1725 * This does a pretty simple search, the expectation is that it is called very
1726 * infrequently and that a given device has a small number of extents.
1727 *
1728 * @start is used to store the start of the free space if we find. But if we
1729 * don't find suitable free space, it will be used to store the start position
1730 * of the max free space.
1731 *
1732 * @len is used to store the size of the free space that we find.
1733 * But if we don't find suitable free space, it is used to store the size of
1734 * the max free space.
1735 *
1736 * NOTE: This function will search *commit* root of device tree, and does extra
1737 * check to ensure dev extents are not double allocated.
1738 * This makes the function safe to allocate dev extents but may not report
1739 * correct usable device space, as device extent freed in current transaction
1740 * is not reported as available.
1741 */
1742static int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes,
1743 u64 *start, u64 *len)
1744{
1745 struct btrfs_fs_info *fs_info = device->fs_info;
1746 struct btrfs_root *root = fs_info->dev_root;
1747 struct btrfs_key key;
1748 struct btrfs_dev_extent *dev_extent;
1749 struct btrfs_path *path;
1750 u64 search_start;
1751 u64 hole_size;
1752 u64 max_hole_start;
1753 u64 max_hole_size = 0;
1754 u64 extent_end;
1755 u64 search_end = device->total_bytes;
1756 int ret;
1757 int slot;
1758 struct extent_buffer *l;
1759
1760 search_start = dev_extent_search_start(device);
1761 max_hole_start = search_start;
1762
1763 WARN_ON(device->zone_info &&
1764 !IS_ALIGNED(num_bytes, device->zone_info->zone_size));
1765
1766 path = btrfs_alloc_path();
1767 if (!path) {
1768 ret = -ENOMEM;
1769 goto out;
1770 }
1771again:
1772 if (search_start >= search_end ||
1773 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
1774 ret = -ENOSPC;
1775 goto out;
1776 }
1777
1778 path->reada = READA_FORWARD;
1779 path->search_commit_root = 1;
1780 path->skip_locking = 1;
1781
1782 key.objectid = device->devid;
1783 key.offset = search_start;
1784 key.type = BTRFS_DEV_EXTENT_KEY;
1785
1786 ret = btrfs_search_backwards(root, &key, path);
1787 if (ret < 0)
1788 goto out;
1789
1790 while (search_start < search_end) {
1791 l = path->nodes[0];
1792 slot = path->slots[0];
1793 if (slot >= btrfs_header_nritems(l)) {
1794 ret = btrfs_next_leaf(root, path);
1795 if (ret == 0)
1796 continue;
1797 if (ret < 0)
1798 goto out;
1799
1800 break;
1801 }
1802 btrfs_item_key_to_cpu(l, &key, slot);
1803
1804 if (key.objectid < device->devid)
1805 goto next;
1806
1807 if (key.objectid > device->devid)
1808 break;
1809
1810 if (key.type != BTRFS_DEV_EXTENT_KEY)
1811 goto next;
1812
1813 if (key.offset > search_end)
1814 break;
1815
1816 if (key.offset > search_start) {
1817 hole_size = key.offset - search_start;
1818 dev_extent_hole_check(device, &search_start, &hole_size,
1819 num_bytes);
1820
1821 if (hole_size > max_hole_size) {
1822 max_hole_start = search_start;
1823 max_hole_size = hole_size;
1824 }
1825
1826 /*
1827 * If this free space is greater than which we need,
1828 * it must be the max free space that we have found
1829 * until now, so max_hole_start must point to the start
1830 * of this free space and the length of this free space
1831 * is stored in max_hole_size. Thus, we return
1832 * max_hole_start and max_hole_size and go back to the
1833 * caller.
1834 */
1835 if (hole_size >= num_bytes) {
1836 ret = 0;
1837 goto out;
1838 }
1839 }
1840
1841 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
1842 extent_end = key.offset + btrfs_dev_extent_length(l,
1843 dev_extent);
1844 if (extent_end > search_start)
1845 search_start = extent_end;
1846next:
1847 path->slots[0]++;
1848 cond_resched();
1849 }
1850
1851 /*
1852 * At this point, search_start should be the end of
1853 * allocated dev extents, and when shrinking the device,
1854 * search_end may be smaller than search_start.
1855 */
1856 if (search_end > search_start) {
1857 hole_size = search_end - search_start;
1858 if (dev_extent_hole_check(device, &search_start, &hole_size,
1859 num_bytes)) {
1860 btrfs_release_path(path);
1861 goto again;
1862 }
1863
1864 if (hole_size > max_hole_size) {
1865 max_hole_start = search_start;
1866 max_hole_size = hole_size;
1867 }
1868 }
1869
1870 /* See above. */
1871 if (max_hole_size < num_bytes)
1872 ret = -ENOSPC;
1873 else
1874 ret = 0;
1875
1876 ASSERT(max_hole_start + max_hole_size <= search_end);
1877out:
1878 btrfs_free_path(path);
1879 *start = max_hole_start;
1880 if (len)
1881 *len = max_hole_size;
1882 return ret;
1883}
1884
1885static int btrfs_free_dev_extent(struct btrfs_trans_handle *trans,
1886 struct btrfs_device *device,
1887 u64 start, u64 *dev_extent_len)
1888{
1889 struct btrfs_fs_info *fs_info = device->fs_info;
1890 struct btrfs_root *root = fs_info->dev_root;
1891 int ret;
1892 struct btrfs_path *path;
1893 struct btrfs_key key;
1894 struct btrfs_key found_key;
1895 struct extent_buffer *leaf = NULL;
1896 struct btrfs_dev_extent *extent = NULL;
1897
1898 path = btrfs_alloc_path();
1899 if (!path)
1900 return -ENOMEM;
1901
1902 key.objectid = device->devid;
1903 key.offset = start;
1904 key.type = BTRFS_DEV_EXTENT_KEY;
1905again:
1906 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1907 if (ret > 0) {
1908 ret = btrfs_previous_item(root, path, key.objectid,
1909 BTRFS_DEV_EXTENT_KEY);
1910 if (ret)
1911 goto out;
1912 leaf = path->nodes[0];
1913 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1914 extent = btrfs_item_ptr(leaf, path->slots[0],
1915 struct btrfs_dev_extent);
1916 BUG_ON(found_key.offset > start || found_key.offset +
1917 btrfs_dev_extent_length(leaf, extent) < start);
1918 key = found_key;
1919 btrfs_release_path(path);
1920 goto again;
1921 } else if (ret == 0) {
1922 leaf = path->nodes[0];
1923 extent = btrfs_item_ptr(leaf, path->slots[0],
1924 struct btrfs_dev_extent);
1925 } else {
1926 goto out;
1927 }
1928
1929 *dev_extent_len = btrfs_dev_extent_length(leaf, extent);
1930
1931 ret = btrfs_del_item(trans, root, path);
1932 if (ret == 0)
1933 set_bit(BTRFS_TRANS_HAVE_FREE_BGS, &trans->transaction->flags);
1934out:
1935 btrfs_free_path(path);
1936 return ret;
1937}
1938
1939static u64 find_next_chunk(struct btrfs_fs_info *fs_info)
1940{
1941 struct rb_node *n;
1942 u64 ret = 0;
1943
1944 read_lock(&fs_info->mapping_tree_lock);
1945 n = rb_last(&fs_info->mapping_tree.rb_root);
1946 if (n) {
1947 struct btrfs_chunk_map *map;
1948
1949 map = rb_entry(n, struct btrfs_chunk_map, rb_node);
1950 ret = map->start + map->chunk_len;
1951 }
1952 read_unlock(&fs_info->mapping_tree_lock);
1953
1954 return ret;
1955}
1956
1957static noinline int find_next_devid(struct btrfs_fs_info *fs_info,
1958 u64 *devid_ret)
1959{
1960 int ret;
1961 struct btrfs_key key;
1962 struct btrfs_key found_key;
1963 struct btrfs_path *path;
1964
1965 path = btrfs_alloc_path();
1966 if (!path)
1967 return -ENOMEM;
1968
1969 key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1970 key.type = BTRFS_DEV_ITEM_KEY;
1971 key.offset = (u64)-1;
1972
1973 ret = btrfs_search_slot(NULL, fs_info->chunk_root, &key, path, 0, 0);
1974 if (ret < 0)
1975 goto error;
1976
1977 if (ret == 0) {
1978 /* Corruption */
1979 btrfs_err(fs_info, "corrupted chunk tree devid -1 matched");
1980 ret = -EUCLEAN;
1981 goto error;
1982 }
1983
1984 ret = btrfs_previous_item(fs_info->chunk_root, path,
1985 BTRFS_DEV_ITEMS_OBJECTID,
1986 BTRFS_DEV_ITEM_KEY);
1987 if (ret) {
1988 *devid_ret = 1;
1989 } else {
1990 btrfs_item_key_to_cpu(path->nodes[0], &found_key,
1991 path->slots[0]);
1992 *devid_ret = found_key.offset + 1;
1993 }
1994 ret = 0;
1995error:
1996 btrfs_free_path(path);
1997 return ret;
1998}
1999
2000/*
2001 * the device information is stored in the chunk root
2002 * the btrfs_device struct should be fully filled in
2003 */
2004static int btrfs_add_dev_item(struct btrfs_trans_handle *trans,
2005 struct btrfs_device *device)
2006{
2007 int ret;
2008 struct btrfs_path *path;
2009 struct btrfs_dev_item *dev_item;
2010 struct extent_buffer *leaf;
2011 struct btrfs_key key;
2012 unsigned long ptr;
2013
2014 path = btrfs_alloc_path();
2015 if (!path)
2016 return -ENOMEM;
2017
2018 key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2019 key.type = BTRFS_DEV_ITEM_KEY;
2020 key.offset = device->devid;
2021
2022 btrfs_reserve_chunk_metadata(trans, true);
2023 ret = btrfs_insert_empty_item(trans, trans->fs_info->chunk_root, path,
2024 &key, sizeof(*dev_item));
2025 btrfs_trans_release_chunk_metadata(trans);
2026 if (ret)
2027 goto out;
2028
2029 leaf = path->nodes[0];
2030 dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
2031
2032 btrfs_set_device_id(leaf, dev_item, device->devid);
2033 btrfs_set_device_generation(leaf, dev_item, 0);
2034 btrfs_set_device_type(leaf, dev_item, device->type);
2035 btrfs_set_device_io_align(leaf, dev_item, device->io_align);
2036 btrfs_set_device_io_width(leaf, dev_item, device->io_width);
2037 btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
2038 btrfs_set_device_total_bytes(leaf, dev_item,
2039 btrfs_device_get_disk_total_bytes(device));
2040 btrfs_set_device_bytes_used(leaf, dev_item,
2041 btrfs_device_get_bytes_used(device));
2042 btrfs_set_device_group(leaf, dev_item, 0);
2043 btrfs_set_device_seek_speed(leaf, dev_item, 0);
2044 btrfs_set_device_bandwidth(leaf, dev_item, 0);
2045 btrfs_set_device_start_offset(leaf, dev_item, 0);
2046
2047 ptr = btrfs_device_uuid(dev_item);
2048 write_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
2049 ptr = btrfs_device_fsid(dev_item);
2050 write_extent_buffer(leaf, trans->fs_info->fs_devices->metadata_uuid,
2051 ptr, BTRFS_FSID_SIZE);
2052 btrfs_mark_buffer_dirty(trans, leaf);
2053
2054 ret = 0;
2055out:
2056 btrfs_free_path(path);
2057 return ret;
2058}
2059
2060/*
2061 * Function to update ctime/mtime for a given device path.
2062 * Mainly used for ctime/mtime based probe like libblkid.
2063 *
2064 * We don't care about errors here, this is just to be kind to userspace.
2065 */
2066static void update_dev_time(const char *device_path)
2067{
2068 struct path path;
2069 int ret;
2070
2071 ret = kern_path(device_path, LOOKUP_FOLLOW, &path);
2072 if (ret)
2073 return;
2074
2075 inode_update_time(d_inode(path.dentry), S_MTIME | S_CTIME | S_VERSION);
2076 path_put(&path);
2077}
2078
2079static int btrfs_rm_dev_item(struct btrfs_trans_handle *trans,
2080 struct btrfs_device *device)
2081{
2082 struct btrfs_root *root = device->fs_info->chunk_root;
2083 int ret;
2084 struct btrfs_path *path;
2085 struct btrfs_key key;
2086
2087 path = btrfs_alloc_path();
2088 if (!path)
2089 return -ENOMEM;
2090
2091 key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2092 key.type = BTRFS_DEV_ITEM_KEY;
2093 key.offset = device->devid;
2094
2095 btrfs_reserve_chunk_metadata(trans, false);
2096 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
2097 btrfs_trans_release_chunk_metadata(trans);
2098 if (ret) {
2099 if (ret > 0)
2100 ret = -ENOENT;
2101 goto out;
2102 }
2103
2104 ret = btrfs_del_item(trans, root, path);
2105out:
2106 btrfs_free_path(path);
2107 return ret;
2108}
2109
2110/*
2111 * Verify that @num_devices satisfies the RAID profile constraints in the whole
2112 * filesystem. It's up to the caller to adjust that number regarding eg. device
2113 * replace.
2114 */
2115static int btrfs_check_raid_min_devices(struct btrfs_fs_info *fs_info,
2116 u64 num_devices)
2117{
2118 u64 all_avail;
2119 unsigned seq;
2120 int i;
2121
2122 do {
2123 seq = read_seqbegin(&fs_info->profiles_lock);
2124
2125 all_avail = fs_info->avail_data_alloc_bits |
2126 fs_info->avail_system_alloc_bits |
2127 fs_info->avail_metadata_alloc_bits;
2128 } while (read_seqretry(&fs_info->profiles_lock, seq));
2129
2130 for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
2131 if (!(all_avail & btrfs_raid_array[i].bg_flag))
2132 continue;
2133
2134 if (num_devices < btrfs_raid_array[i].devs_min)
2135 return btrfs_raid_array[i].mindev_error;
2136 }
2137
2138 return 0;
2139}
2140
2141static struct btrfs_device * btrfs_find_next_active_device(
2142 struct btrfs_fs_devices *fs_devs, struct btrfs_device *device)
2143{
2144 struct btrfs_device *next_device;
2145
2146 list_for_each_entry(next_device, &fs_devs->devices, dev_list) {
2147 if (next_device != device &&
2148 !test_bit(BTRFS_DEV_STATE_MISSING, &next_device->dev_state)
2149 && next_device->bdev)
2150 return next_device;
2151 }
2152
2153 return NULL;
2154}
2155
2156/*
2157 * Helper function to check if the given device is part of s_bdev / latest_dev
2158 * and replace it with the provided or the next active device, in the context
2159 * where this function called, there should be always be another device (or
2160 * this_dev) which is active.
2161 */
2162void __cold btrfs_assign_next_active_device(struct btrfs_device *device,
2163 struct btrfs_device *next_device)
2164{
2165 struct btrfs_fs_info *fs_info = device->fs_info;
2166
2167 if (!next_device)
2168 next_device = btrfs_find_next_active_device(fs_info->fs_devices,
2169 device);
2170 ASSERT(next_device);
2171
2172 if (fs_info->sb->s_bdev &&
2173 (fs_info->sb->s_bdev == device->bdev))
2174 fs_info->sb->s_bdev = next_device->bdev;
2175
2176 if (fs_info->fs_devices->latest_dev->bdev == device->bdev)
2177 fs_info->fs_devices->latest_dev = next_device;
2178}
2179
2180/*
2181 * Return btrfs_fs_devices::num_devices excluding the device that's being
2182 * currently replaced.
2183 */
2184static u64 btrfs_num_devices(struct btrfs_fs_info *fs_info)
2185{
2186 u64 num_devices = fs_info->fs_devices->num_devices;
2187
2188 down_read(&fs_info->dev_replace.rwsem);
2189 if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace)) {
2190 ASSERT(num_devices > 1);
2191 num_devices--;
2192 }
2193 up_read(&fs_info->dev_replace.rwsem);
2194
2195 return num_devices;
2196}
2197
2198static void btrfs_scratch_superblock(struct btrfs_fs_info *fs_info,
2199 struct block_device *bdev, int copy_num)
2200{
2201 struct btrfs_super_block *disk_super;
2202 const size_t len = sizeof(disk_super->magic);
2203 const u64 bytenr = btrfs_sb_offset(copy_num);
2204 int ret;
2205
2206 disk_super = btrfs_read_disk_super(bdev, bytenr, bytenr);
2207 if (IS_ERR(disk_super))
2208 return;
2209
2210 memset(&disk_super->magic, 0, len);
2211 folio_mark_dirty(virt_to_folio(disk_super));
2212 btrfs_release_disk_super(disk_super);
2213
2214 ret = sync_blockdev_range(bdev, bytenr, bytenr + len - 1);
2215 if (ret)
2216 btrfs_warn(fs_info, "error clearing superblock number %d (%d)",
2217 copy_num, ret);
2218}
2219
2220void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info, struct btrfs_device *device)
2221{
2222 int copy_num;
2223 struct block_device *bdev = device->bdev;
2224
2225 if (!bdev)
2226 return;
2227
2228 for (copy_num = 0; copy_num < BTRFS_SUPER_MIRROR_MAX; copy_num++) {
2229 if (bdev_is_zoned(bdev))
2230 btrfs_reset_sb_log_zones(bdev, copy_num);
2231 else
2232 btrfs_scratch_superblock(fs_info, bdev, copy_num);
2233 }
2234
2235 /* Notify udev that device has changed */
2236 btrfs_kobject_uevent(bdev, KOBJ_CHANGE);
2237
2238 /* Update ctime/mtime for device path for libblkid */
2239 update_dev_time(device->name->str);
2240}
2241
2242int btrfs_rm_device(struct btrfs_fs_info *fs_info,
2243 struct btrfs_dev_lookup_args *args,
2244 struct file **bdev_file)
2245{
2246 struct btrfs_trans_handle *trans;
2247 struct btrfs_device *device;
2248 struct btrfs_fs_devices *cur_devices;
2249 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2250 u64 num_devices;
2251 int ret = 0;
2252
2253 if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) {
2254 btrfs_err(fs_info, "device remove not supported on extent tree v2 yet");
2255 return -EINVAL;
2256 }
2257
2258 /*
2259 * The device list in fs_devices is accessed without locks (neither
2260 * uuid_mutex nor device_list_mutex) as it won't change on a mounted
2261 * filesystem and another device rm cannot run.
2262 */
2263 num_devices = btrfs_num_devices(fs_info);
2264
2265 ret = btrfs_check_raid_min_devices(fs_info, num_devices - 1);
2266 if (ret)
2267 return ret;
2268
2269 device = btrfs_find_device(fs_info->fs_devices, args);
2270 if (!device) {
2271 if (args->missing)
2272 ret = BTRFS_ERROR_DEV_MISSING_NOT_FOUND;
2273 else
2274 ret = -ENOENT;
2275 return ret;
2276 }
2277
2278 if (btrfs_pinned_by_swapfile(fs_info, device)) {
2279 btrfs_warn_in_rcu(fs_info,
2280 "cannot remove device %s (devid %llu) due to active swapfile",
2281 btrfs_dev_name(device), device->devid);
2282 return -ETXTBSY;
2283 }
2284
2285 if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
2286 return BTRFS_ERROR_DEV_TGT_REPLACE;
2287
2288 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
2289 fs_info->fs_devices->rw_devices == 1)
2290 return BTRFS_ERROR_DEV_ONLY_WRITABLE;
2291
2292 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2293 mutex_lock(&fs_info->chunk_mutex);
2294 list_del_init(&device->dev_alloc_list);
2295 device->fs_devices->rw_devices--;
2296 mutex_unlock(&fs_info->chunk_mutex);
2297 }
2298
2299 ret = btrfs_shrink_device(device, 0);
2300 if (ret)
2301 goto error_undo;
2302
2303 trans = btrfs_start_transaction(fs_info->chunk_root, 0);
2304 if (IS_ERR(trans)) {
2305 ret = PTR_ERR(trans);
2306 goto error_undo;
2307 }
2308
2309 ret = btrfs_rm_dev_item(trans, device);
2310 if (ret) {
2311 /* Any error in dev item removal is critical */
2312 btrfs_crit(fs_info,
2313 "failed to remove device item for devid %llu: %d",
2314 device->devid, ret);
2315 btrfs_abort_transaction(trans, ret);
2316 btrfs_end_transaction(trans);
2317 return ret;
2318 }
2319
2320 clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
2321 btrfs_scrub_cancel_dev(device);
2322
2323 /*
2324 * the device list mutex makes sure that we don't change
2325 * the device list while someone else is writing out all
2326 * the device supers. Whoever is writing all supers, should
2327 * lock the device list mutex before getting the number of
2328 * devices in the super block (super_copy). Conversely,
2329 * whoever updates the number of devices in the super block
2330 * (super_copy) should hold the device list mutex.
2331 */
2332
2333 /*
2334 * In normal cases the cur_devices == fs_devices. But in case
2335 * of deleting a seed device, the cur_devices should point to
2336 * its own fs_devices listed under the fs_devices->seed_list.
2337 */
2338 cur_devices = device->fs_devices;
2339 mutex_lock(&fs_devices->device_list_mutex);
2340 list_del_rcu(&device->dev_list);
2341
2342 cur_devices->num_devices--;
2343 cur_devices->total_devices--;
2344 /* Update total_devices of the parent fs_devices if it's seed */
2345 if (cur_devices != fs_devices)
2346 fs_devices->total_devices--;
2347
2348 if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
2349 cur_devices->missing_devices--;
2350
2351 btrfs_assign_next_active_device(device, NULL);
2352
2353 if (device->bdev_file) {
2354 cur_devices->open_devices--;
2355 /* remove sysfs entry */
2356 btrfs_sysfs_remove_device(device);
2357 }
2358
2359 num_devices = btrfs_super_num_devices(fs_info->super_copy) - 1;
2360 btrfs_set_super_num_devices(fs_info->super_copy, num_devices);
2361 mutex_unlock(&fs_devices->device_list_mutex);
2362
2363 /*
2364 * At this point, the device is zero sized and detached from the
2365 * devices list. All that's left is to zero out the old supers and
2366 * free the device.
2367 *
2368 * We cannot call btrfs_close_bdev() here because we're holding the sb
2369 * write lock, and fput() on the block device will pull in the
2370 * ->open_mutex on the block device and it's dependencies. Instead
2371 * just flush the device and let the caller do the final bdev_release.
2372 */
2373 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2374 btrfs_scratch_superblocks(fs_info, device);
2375 if (device->bdev) {
2376 sync_blockdev(device->bdev);
2377 invalidate_bdev(device->bdev);
2378 }
2379 }
2380
2381 *bdev_file = device->bdev_file;
2382 synchronize_rcu();
2383 btrfs_free_device(device);
2384
2385 /*
2386 * This can happen if cur_devices is the private seed devices list. We
2387 * cannot call close_fs_devices() here because it expects the uuid_mutex
2388 * to be held, but in fact we don't need that for the private
2389 * seed_devices, we can simply decrement cur_devices->opened and then
2390 * remove it from our list and free the fs_devices.
2391 */
2392 if (cur_devices->num_devices == 0) {
2393 list_del_init(&cur_devices->seed_list);
2394 ASSERT(cur_devices->opened == 1);
2395 cur_devices->opened--;
2396 free_fs_devices(cur_devices);
2397 }
2398
2399 ret = btrfs_commit_transaction(trans);
2400
2401 return ret;
2402
2403error_undo:
2404 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2405 mutex_lock(&fs_info->chunk_mutex);
2406 list_add(&device->dev_alloc_list,
2407 &fs_devices->alloc_list);
2408 device->fs_devices->rw_devices++;
2409 mutex_unlock(&fs_info->chunk_mutex);
2410 }
2411 return ret;
2412}
2413
2414void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev)
2415{
2416 struct btrfs_fs_devices *fs_devices;
2417
2418 lockdep_assert_held(&srcdev->fs_info->fs_devices->device_list_mutex);
2419
2420 /*
2421 * in case of fs with no seed, srcdev->fs_devices will point
2422 * to fs_devices of fs_info. However when the dev being replaced is
2423 * a seed dev it will point to the seed's local fs_devices. In short
2424 * srcdev will have its correct fs_devices in both the cases.
2425 */
2426 fs_devices = srcdev->fs_devices;
2427
2428 list_del_rcu(&srcdev->dev_list);
2429 list_del(&srcdev->dev_alloc_list);
2430 fs_devices->num_devices--;
2431 if (test_bit(BTRFS_DEV_STATE_MISSING, &srcdev->dev_state))
2432 fs_devices->missing_devices--;
2433
2434 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state))
2435 fs_devices->rw_devices--;
2436
2437 if (srcdev->bdev)
2438 fs_devices->open_devices--;
2439}
2440
2441void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev)
2442{
2443 struct btrfs_fs_devices *fs_devices = srcdev->fs_devices;
2444
2445 mutex_lock(&uuid_mutex);
2446
2447 btrfs_close_bdev(srcdev);
2448 synchronize_rcu();
2449 btrfs_free_device(srcdev);
2450
2451 /* if this is no devs we rather delete the fs_devices */
2452 if (!fs_devices->num_devices) {
2453 /*
2454 * On a mounted FS, num_devices can't be zero unless it's a
2455 * seed. In case of a seed device being replaced, the replace
2456 * target added to the sprout FS, so there will be no more
2457 * device left under the seed FS.
2458 */
2459 ASSERT(fs_devices->seeding);
2460
2461 list_del_init(&fs_devices->seed_list);
2462 close_fs_devices(fs_devices);
2463 free_fs_devices(fs_devices);
2464 }
2465 mutex_unlock(&uuid_mutex);
2466}
2467
2468void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev)
2469{
2470 struct btrfs_fs_devices *fs_devices = tgtdev->fs_info->fs_devices;
2471
2472 mutex_lock(&fs_devices->device_list_mutex);
2473
2474 btrfs_sysfs_remove_device(tgtdev);
2475
2476 if (tgtdev->bdev)
2477 fs_devices->open_devices--;
2478
2479 fs_devices->num_devices--;
2480
2481 btrfs_assign_next_active_device(tgtdev, NULL);
2482
2483 list_del_rcu(&tgtdev->dev_list);
2484
2485 mutex_unlock(&fs_devices->device_list_mutex);
2486
2487 btrfs_scratch_superblocks(tgtdev->fs_info, tgtdev);
2488
2489 btrfs_close_bdev(tgtdev);
2490 synchronize_rcu();
2491 btrfs_free_device(tgtdev);
2492}
2493
2494/*
2495 * Populate args from device at path.
2496 *
2497 * @fs_info: the filesystem
2498 * @args: the args to populate
2499 * @path: the path to the device
2500 *
2501 * This will read the super block of the device at @path and populate @args with
2502 * the devid, fsid, and uuid. This is meant to be used for ioctls that need to
2503 * lookup a device to operate on, but need to do it before we take any locks.
2504 * This properly handles the special case of "missing" that a user may pass in,
2505 * and does some basic sanity checks. The caller must make sure that @path is
2506 * properly NUL terminated before calling in, and must call
2507 * btrfs_put_dev_args_from_path() in order to free up the temporary fsid and
2508 * uuid buffers.
2509 *
2510 * Return: 0 for success, -errno for failure
2511 */
2512int btrfs_get_dev_args_from_path(struct btrfs_fs_info *fs_info,
2513 struct btrfs_dev_lookup_args *args,
2514 const char *path)
2515{
2516 struct btrfs_super_block *disk_super;
2517 struct file *bdev_file;
2518 int ret;
2519
2520 if (!path || !path[0])
2521 return -EINVAL;
2522 if (!strcmp(path, "missing")) {
2523 args->missing = true;
2524 return 0;
2525 }
2526
2527 args->uuid = kzalloc(BTRFS_UUID_SIZE, GFP_KERNEL);
2528 args->fsid = kzalloc(BTRFS_FSID_SIZE, GFP_KERNEL);
2529 if (!args->uuid || !args->fsid) {
2530 btrfs_put_dev_args_from_path(args);
2531 return -ENOMEM;
2532 }
2533
2534 ret = btrfs_get_bdev_and_sb(path, BLK_OPEN_READ, NULL, 0,
2535 &bdev_file, &disk_super);
2536 if (ret) {
2537 btrfs_put_dev_args_from_path(args);
2538 return ret;
2539 }
2540
2541 args->devid = btrfs_stack_device_id(&disk_super->dev_item);
2542 memcpy(args->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE);
2543 if (btrfs_fs_incompat(fs_info, METADATA_UUID))
2544 memcpy(args->fsid, disk_super->metadata_uuid, BTRFS_FSID_SIZE);
2545 else
2546 memcpy(args->fsid, disk_super->fsid, BTRFS_FSID_SIZE);
2547 btrfs_release_disk_super(disk_super);
2548 fput(bdev_file);
2549 return 0;
2550}
2551
2552/*
2553 * Only use this jointly with btrfs_get_dev_args_from_path() because we will
2554 * allocate our ->uuid and ->fsid pointers, everybody else uses local variables
2555 * that don't need to be freed.
2556 */
2557void btrfs_put_dev_args_from_path(struct btrfs_dev_lookup_args *args)
2558{
2559 kfree(args->uuid);
2560 kfree(args->fsid);
2561 args->uuid = NULL;
2562 args->fsid = NULL;
2563}
2564
2565struct btrfs_device *btrfs_find_device_by_devspec(
2566 struct btrfs_fs_info *fs_info, u64 devid,
2567 const char *device_path)
2568{
2569 BTRFS_DEV_LOOKUP_ARGS(args);
2570 struct btrfs_device *device;
2571 int ret;
2572
2573 if (devid) {
2574 args.devid = devid;
2575 device = btrfs_find_device(fs_info->fs_devices, &args);
2576 if (!device)
2577 return ERR_PTR(-ENOENT);
2578 return device;
2579 }
2580
2581 ret = btrfs_get_dev_args_from_path(fs_info, &args, device_path);
2582 if (ret)
2583 return ERR_PTR(ret);
2584 device = btrfs_find_device(fs_info->fs_devices, &args);
2585 btrfs_put_dev_args_from_path(&args);
2586 if (!device)
2587 return ERR_PTR(-ENOENT);
2588 return device;
2589}
2590
2591static struct btrfs_fs_devices *btrfs_init_sprout(struct btrfs_fs_info *fs_info)
2592{
2593 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2594 struct btrfs_fs_devices *old_devices;
2595 struct btrfs_fs_devices *seed_devices;
2596
2597 lockdep_assert_held(&uuid_mutex);
2598 if (!fs_devices->seeding)
2599 return ERR_PTR(-EINVAL);
2600
2601 /*
2602 * Private copy of the seed devices, anchored at
2603 * fs_info->fs_devices->seed_list
2604 */
2605 seed_devices = alloc_fs_devices(NULL);
2606 if (IS_ERR(seed_devices))
2607 return seed_devices;
2608
2609 /*
2610 * It's necessary to retain a copy of the original seed fs_devices in
2611 * fs_uuids so that filesystems which have been seeded can successfully
2612 * reference the seed device from open_seed_devices. This also supports
2613 * multiple fs seed.
2614 */
2615 old_devices = clone_fs_devices(fs_devices);
2616 if (IS_ERR(old_devices)) {
2617 kfree(seed_devices);
2618 return old_devices;
2619 }
2620
2621 list_add(&old_devices->fs_list, &fs_uuids);
2622
2623 memcpy(seed_devices, fs_devices, sizeof(*seed_devices));
2624 seed_devices->opened = 1;
2625 INIT_LIST_HEAD(&seed_devices->devices);
2626 INIT_LIST_HEAD(&seed_devices->alloc_list);
2627 mutex_init(&seed_devices->device_list_mutex);
2628
2629 return seed_devices;
2630}
2631
2632/*
2633 * Splice seed devices into the sprout fs_devices.
2634 * Generate a new fsid for the sprouted read-write filesystem.
2635 */
2636static void btrfs_setup_sprout(struct btrfs_fs_info *fs_info,
2637 struct btrfs_fs_devices *seed_devices)
2638{
2639 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2640 struct btrfs_super_block *disk_super = fs_info->super_copy;
2641 struct btrfs_device *device;
2642 u64 super_flags;
2643
2644 /*
2645 * We are updating the fsid, the thread leading to device_list_add()
2646 * could race, so uuid_mutex is needed.
2647 */
2648 lockdep_assert_held(&uuid_mutex);
2649
2650 /*
2651 * The threads listed below may traverse dev_list but can do that without
2652 * device_list_mutex:
2653 * - All device ops and balance - as we are in btrfs_exclop_start.
2654 * - Various dev_list readers - are using RCU.
2655 * - btrfs_ioctl_fitrim() - is using RCU.
2656 *
2657 * For-read threads as below are using device_list_mutex:
2658 * - Readonly scrub btrfs_scrub_dev()
2659 * - Readonly scrub btrfs_scrub_progress()
2660 * - btrfs_get_dev_stats()
2661 */
2662 lockdep_assert_held(&fs_devices->device_list_mutex);
2663
2664 list_splice_init_rcu(&fs_devices->devices, &seed_devices->devices,
2665 synchronize_rcu);
2666 list_for_each_entry(device, &seed_devices->devices, dev_list)
2667 device->fs_devices = seed_devices;
2668
2669 fs_devices->seeding = false;
2670 fs_devices->num_devices = 0;
2671 fs_devices->open_devices = 0;
2672 fs_devices->missing_devices = 0;
2673 fs_devices->rotating = false;
2674 list_add(&seed_devices->seed_list, &fs_devices->seed_list);
2675
2676 generate_random_uuid(fs_devices->fsid);
2677 memcpy(fs_devices->metadata_uuid, fs_devices->fsid, BTRFS_FSID_SIZE);
2678 memcpy(disk_super->fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
2679
2680 super_flags = btrfs_super_flags(disk_super) &
2681 ~BTRFS_SUPER_FLAG_SEEDING;
2682 btrfs_set_super_flags(disk_super, super_flags);
2683}
2684
2685/*
2686 * Store the expected generation for seed devices in device items.
2687 */
2688static int btrfs_finish_sprout(struct btrfs_trans_handle *trans)
2689{
2690 BTRFS_DEV_LOOKUP_ARGS(args);
2691 struct btrfs_fs_info *fs_info = trans->fs_info;
2692 struct btrfs_root *root = fs_info->chunk_root;
2693 struct btrfs_path *path;
2694 struct extent_buffer *leaf;
2695 struct btrfs_dev_item *dev_item;
2696 struct btrfs_device *device;
2697 struct btrfs_key key;
2698 u8 fs_uuid[BTRFS_FSID_SIZE];
2699 u8 dev_uuid[BTRFS_UUID_SIZE];
2700 int ret;
2701
2702 path = btrfs_alloc_path();
2703 if (!path)
2704 return -ENOMEM;
2705
2706 key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2707 key.offset = 0;
2708 key.type = BTRFS_DEV_ITEM_KEY;
2709
2710 while (1) {
2711 btrfs_reserve_chunk_metadata(trans, false);
2712 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2713 btrfs_trans_release_chunk_metadata(trans);
2714 if (ret < 0)
2715 goto error;
2716
2717 leaf = path->nodes[0];
2718next_slot:
2719 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
2720 ret = btrfs_next_leaf(root, path);
2721 if (ret > 0)
2722 break;
2723 if (ret < 0)
2724 goto error;
2725 leaf = path->nodes[0];
2726 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2727 btrfs_release_path(path);
2728 continue;
2729 }
2730
2731 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2732 if (key.objectid != BTRFS_DEV_ITEMS_OBJECTID ||
2733 key.type != BTRFS_DEV_ITEM_KEY)
2734 break;
2735
2736 dev_item = btrfs_item_ptr(leaf, path->slots[0],
2737 struct btrfs_dev_item);
2738 args.devid = btrfs_device_id(leaf, dev_item);
2739 read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
2740 BTRFS_UUID_SIZE);
2741 read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
2742 BTRFS_FSID_SIZE);
2743 args.uuid = dev_uuid;
2744 args.fsid = fs_uuid;
2745 device = btrfs_find_device(fs_info->fs_devices, &args);
2746 BUG_ON(!device); /* Logic error */
2747
2748 if (device->fs_devices->seeding) {
2749 btrfs_set_device_generation(leaf, dev_item,
2750 device->generation);
2751 btrfs_mark_buffer_dirty(trans, leaf);
2752 }
2753
2754 path->slots[0]++;
2755 goto next_slot;
2756 }
2757 ret = 0;
2758error:
2759 btrfs_free_path(path);
2760 return ret;
2761}
2762
2763int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path)
2764{
2765 struct btrfs_root *root = fs_info->dev_root;
2766 struct btrfs_trans_handle *trans;
2767 struct btrfs_device *device;
2768 struct file *bdev_file;
2769 struct super_block *sb = fs_info->sb;
2770 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2771 struct btrfs_fs_devices *seed_devices = NULL;
2772 u64 orig_super_total_bytes;
2773 u64 orig_super_num_devices;
2774 int ret = 0;
2775 bool seeding_dev = false;
2776 bool locked = false;
2777
2778 if (sb_rdonly(sb) && !fs_devices->seeding)
2779 return -EROFS;
2780
2781 bdev_file = bdev_file_open_by_path(device_path, BLK_OPEN_WRITE,
2782 fs_info->bdev_holder, NULL);
2783 if (IS_ERR(bdev_file))
2784 return PTR_ERR(bdev_file);
2785
2786 if (!btrfs_check_device_zone_type(fs_info, file_bdev(bdev_file))) {
2787 ret = -EINVAL;
2788 goto error;
2789 }
2790
2791 if (fs_devices->seeding) {
2792 seeding_dev = true;
2793 down_write(&sb->s_umount);
2794 mutex_lock(&uuid_mutex);
2795 locked = true;
2796 }
2797
2798 sync_blockdev(file_bdev(bdev_file));
2799
2800 rcu_read_lock();
2801 list_for_each_entry_rcu(device, &fs_devices->devices, dev_list) {
2802 if (device->bdev == file_bdev(bdev_file)) {
2803 ret = -EEXIST;
2804 rcu_read_unlock();
2805 goto error;
2806 }
2807 }
2808 rcu_read_unlock();
2809
2810 device = btrfs_alloc_device(fs_info, NULL, NULL, device_path);
2811 if (IS_ERR(device)) {
2812 /* we can safely leave the fs_devices entry around */
2813 ret = PTR_ERR(device);
2814 goto error;
2815 }
2816
2817 device->fs_info = fs_info;
2818 device->bdev_file = bdev_file;
2819 device->bdev = file_bdev(bdev_file);
2820 ret = lookup_bdev(device_path, &device->devt);
2821 if (ret)
2822 goto error_free_device;
2823
2824 ret = btrfs_get_dev_zone_info(device, false);
2825 if (ret)
2826 goto error_free_device;
2827
2828 trans = btrfs_start_transaction(root, 0);
2829 if (IS_ERR(trans)) {
2830 ret = PTR_ERR(trans);
2831 goto error_free_zone;
2832 }
2833
2834 set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
2835 device->generation = trans->transid;
2836 device->io_width = fs_info->sectorsize;
2837 device->io_align = fs_info->sectorsize;
2838 device->sector_size = fs_info->sectorsize;
2839 device->total_bytes =
2840 round_down(bdev_nr_bytes(device->bdev), fs_info->sectorsize);
2841 device->disk_total_bytes = device->total_bytes;
2842 device->commit_total_bytes = device->total_bytes;
2843 set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
2844 clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
2845 device->dev_stats_valid = 1;
2846 set_blocksize(device->bdev_file, BTRFS_BDEV_BLOCKSIZE);
2847
2848 if (seeding_dev) {
2849 /* GFP_KERNEL allocation must not be under device_list_mutex */
2850 seed_devices = btrfs_init_sprout(fs_info);
2851 if (IS_ERR(seed_devices)) {
2852 ret = PTR_ERR(seed_devices);
2853 btrfs_abort_transaction(trans, ret);
2854 goto error_trans;
2855 }
2856 }
2857
2858 mutex_lock(&fs_devices->device_list_mutex);
2859 if (seeding_dev) {
2860 btrfs_setup_sprout(fs_info, seed_devices);
2861 btrfs_assign_next_active_device(fs_info->fs_devices->latest_dev,
2862 device);
2863 }
2864
2865 device->fs_devices = fs_devices;
2866
2867 mutex_lock(&fs_info->chunk_mutex);
2868 list_add_rcu(&device->dev_list, &fs_devices->devices);
2869 list_add(&device->dev_alloc_list, &fs_devices->alloc_list);
2870 fs_devices->num_devices++;
2871 fs_devices->open_devices++;
2872 fs_devices->rw_devices++;
2873 fs_devices->total_devices++;
2874 fs_devices->total_rw_bytes += device->total_bytes;
2875
2876 atomic64_add(device->total_bytes, &fs_info->free_chunk_space);
2877
2878 if (!bdev_nonrot(device->bdev))
2879 fs_devices->rotating = true;
2880
2881 orig_super_total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
2882 btrfs_set_super_total_bytes(fs_info->super_copy,
2883 round_down(orig_super_total_bytes + device->total_bytes,
2884 fs_info->sectorsize));
2885
2886 orig_super_num_devices = btrfs_super_num_devices(fs_info->super_copy);
2887 btrfs_set_super_num_devices(fs_info->super_copy,
2888 orig_super_num_devices + 1);
2889
2890 /*
2891 * we've got more storage, clear any full flags on the space
2892 * infos
2893 */
2894 btrfs_clear_space_info_full(fs_info);
2895
2896 mutex_unlock(&fs_info->chunk_mutex);
2897
2898 /* Add sysfs device entry */
2899 btrfs_sysfs_add_device(device);
2900
2901 mutex_unlock(&fs_devices->device_list_mutex);
2902
2903 if (seeding_dev) {
2904 mutex_lock(&fs_info->chunk_mutex);
2905 ret = init_first_rw_device(trans);
2906 mutex_unlock(&fs_info->chunk_mutex);
2907 if (ret) {
2908 btrfs_abort_transaction(trans, ret);
2909 goto error_sysfs;
2910 }
2911 }
2912
2913 ret = btrfs_add_dev_item(trans, device);
2914 if (ret) {
2915 btrfs_abort_transaction(trans, ret);
2916 goto error_sysfs;
2917 }
2918
2919 if (seeding_dev) {
2920 ret = btrfs_finish_sprout(trans);
2921 if (ret) {
2922 btrfs_abort_transaction(trans, ret);
2923 goto error_sysfs;
2924 }
2925
2926 /*
2927 * fs_devices now represents the newly sprouted filesystem and
2928 * its fsid has been changed by btrfs_sprout_splice().
2929 */
2930 btrfs_sysfs_update_sprout_fsid(fs_devices);
2931 }
2932
2933 ret = btrfs_commit_transaction(trans);
2934
2935 if (seeding_dev) {
2936 mutex_unlock(&uuid_mutex);
2937 up_write(&sb->s_umount);
2938 locked = false;
2939
2940 if (ret) /* transaction commit */
2941 return ret;
2942
2943 ret = btrfs_relocate_sys_chunks(fs_info);
2944 if (ret < 0)
2945 btrfs_handle_fs_error(fs_info, ret,
2946 "Failed to relocate sys chunks after device initialization. This can be fixed using the \"btrfs balance\" command.");
2947 trans = btrfs_attach_transaction(root);
2948 if (IS_ERR(trans)) {
2949 if (PTR_ERR(trans) == -ENOENT)
2950 return 0;
2951 ret = PTR_ERR(trans);
2952 trans = NULL;
2953 goto error_sysfs;
2954 }
2955 ret = btrfs_commit_transaction(trans);
2956 }
2957
2958 /*
2959 * Now that we have written a new super block to this device, check all
2960 * other fs_devices list if device_path alienates any other scanned
2961 * device.
2962 * We can ignore the return value as it typically returns -EINVAL and
2963 * only succeeds if the device was an alien.
2964 */
2965 btrfs_forget_devices(device->devt);
2966
2967 /* Update ctime/mtime for blkid or udev */
2968 update_dev_time(device_path);
2969
2970 return ret;
2971
2972error_sysfs:
2973 btrfs_sysfs_remove_device(device);
2974 mutex_lock(&fs_info->fs_devices->device_list_mutex);
2975 mutex_lock(&fs_info->chunk_mutex);
2976 list_del_rcu(&device->dev_list);
2977 list_del(&device->dev_alloc_list);
2978 fs_info->fs_devices->num_devices--;
2979 fs_info->fs_devices->open_devices--;
2980 fs_info->fs_devices->rw_devices--;
2981 fs_info->fs_devices->total_devices--;
2982 fs_info->fs_devices->total_rw_bytes -= device->total_bytes;
2983 atomic64_sub(device->total_bytes, &fs_info->free_chunk_space);
2984 btrfs_set_super_total_bytes(fs_info->super_copy,
2985 orig_super_total_bytes);
2986 btrfs_set_super_num_devices(fs_info->super_copy,
2987 orig_super_num_devices);
2988 mutex_unlock(&fs_info->chunk_mutex);
2989 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2990error_trans:
2991 if (trans)
2992 btrfs_end_transaction(trans);
2993error_free_zone:
2994 btrfs_destroy_dev_zone_info(device);
2995error_free_device:
2996 btrfs_free_device(device);
2997error:
2998 fput(bdev_file);
2999 if (locked) {
3000 mutex_unlock(&uuid_mutex);
3001 up_write(&sb->s_umount);
3002 }
3003 return ret;
3004}
3005
3006static noinline int btrfs_update_device(struct btrfs_trans_handle *trans,
3007 struct btrfs_device *device)
3008{
3009 int ret;
3010 struct btrfs_path *path;
3011 struct btrfs_root *root = device->fs_info->chunk_root;
3012 struct btrfs_dev_item *dev_item;
3013 struct extent_buffer *leaf;
3014 struct btrfs_key key;
3015
3016 path = btrfs_alloc_path();
3017 if (!path)
3018 return -ENOMEM;
3019
3020 key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
3021 key.type = BTRFS_DEV_ITEM_KEY;
3022 key.offset = device->devid;
3023
3024 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
3025 if (ret < 0)
3026 goto out;
3027
3028 if (ret > 0) {
3029 ret = -ENOENT;
3030 goto out;
3031 }
3032
3033 leaf = path->nodes[0];
3034 dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
3035
3036 btrfs_set_device_id(leaf, dev_item, device->devid);
3037 btrfs_set_device_type(leaf, dev_item, device->type);
3038 btrfs_set_device_io_align(leaf, dev_item, device->io_align);
3039 btrfs_set_device_io_width(leaf, dev_item, device->io_width);
3040 btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
3041 btrfs_set_device_total_bytes(leaf, dev_item,
3042 btrfs_device_get_disk_total_bytes(device));
3043 btrfs_set_device_bytes_used(leaf, dev_item,
3044 btrfs_device_get_bytes_used(device));
3045 btrfs_mark_buffer_dirty(trans, leaf);
3046
3047out:
3048 btrfs_free_path(path);
3049 return ret;
3050}
3051
3052int btrfs_grow_device(struct btrfs_trans_handle *trans,
3053 struct btrfs_device *device, u64 new_size)
3054{
3055 struct btrfs_fs_info *fs_info = device->fs_info;
3056 struct btrfs_super_block *super_copy = fs_info->super_copy;
3057 u64 old_total;
3058 u64 diff;
3059 int ret;
3060
3061 if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
3062 return -EACCES;
3063
3064 new_size = round_down(new_size, fs_info->sectorsize);
3065
3066 mutex_lock(&fs_info->chunk_mutex);
3067 old_total = btrfs_super_total_bytes(super_copy);
3068 diff = round_down(new_size - device->total_bytes, fs_info->sectorsize);
3069
3070 if (new_size <= device->total_bytes ||
3071 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
3072 mutex_unlock(&fs_info->chunk_mutex);
3073 return -EINVAL;
3074 }
3075
3076 btrfs_set_super_total_bytes(super_copy,
3077 round_down(old_total + diff, fs_info->sectorsize));
3078 device->fs_devices->total_rw_bytes += diff;
3079 atomic64_add(diff, &fs_info->free_chunk_space);
3080
3081 btrfs_device_set_total_bytes(device, new_size);
3082 btrfs_device_set_disk_total_bytes(device, new_size);
3083 btrfs_clear_space_info_full(device->fs_info);
3084 if (list_empty(&device->post_commit_list))
3085 list_add_tail(&device->post_commit_list,
3086 &trans->transaction->dev_update_list);
3087 mutex_unlock(&fs_info->chunk_mutex);
3088
3089 btrfs_reserve_chunk_metadata(trans, false);
3090 ret = btrfs_update_device(trans, device);
3091 btrfs_trans_release_chunk_metadata(trans);
3092
3093 return ret;
3094}
3095
3096static int btrfs_free_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
3097{
3098 struct btrfs_fs_info *fs_info = trans->fs_info;
3099 struct btrfs_root *root = fs_info->chunk_root;
3100 int ret;
3101 struct btrfs_path *path;
3102 struct btrfs_key key;
3103
3104 path = btrfs_alloc_path();
3105 if (!path)
3106 return -ENOMEM;
3107
3108 key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
3109 key.offset = chunk_offset;
3110 key.type = BTRFS_CHUNK_ITEM_KEY;
3111
3112 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
3113 if (ret < 0)
3114 goto out;
3115 else if (ret > 0) { /* Logic error or corruption */
3116 btrfs_err(fs_info, "failed to lookup chunk %llu when freeing",
3117 chunk_offset);
3118 btrfs_abort_transaction(trans, -ENOENT);
3119 ret = -EUCLEAN;
3120 goto out;
3121 }
3122
3123 ret = btrfs_del_item(trans, root, path);
3124 if (ret < 0) {
3125 btrfs_err(fs_info, "failed to delete chunk %llu item", chunk_offset);
3126 btrfs_abort_transaction(trans, ret);
3127 goto out;
3128 }
3129out:
3130 btrfs_free_path(path);
3131 return ret;
3132}
3133
3134static int btrfs_del_sys_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
3135{
3136 struct btrfs_super_block *super_copy = fs_info->super_copy;
3137 struct btrfs_disk_key *disk_key;
3138 struct btrfs_chunk *chunk;
3139 u8 *ptr;
3140 int ret = 0;
3141 u32 num_stripes;
3142 u32 array_size;
3143 u32 len = 0;
3144 u32 cur;
3145 struct btrfs_key key;
3146
3147 lockdep_assert_held(&fs_info->chunk_mutex);
3148 array_size = btrfs_super_sys_array_size(super_copy);
3149
3150 ptr = super_copy->sys_chunk_array;
3151 cur = 0;
3152
3153 while (cur < array_size) {
3154 disk_key = (struct btrfs_disk_key *)ptr;
3155 btrfs_disk_key_to_cpu(&key, disk_key);
3156
3157 len = sizeof(*disk_key);
3158
3159 if (key.type == BTRFS_CHUNK_ITEM_KEY) {
3160 chunk = (struct btrfs_chunk *)(ptr + len);
3161 num_stripes = btrfs_stack_chunk_num_stripes(chunk);
3162 len += btrfs_chunk_item_size(num_stripes);
3163 } else {
3164 ret = -EIO;
3165 break;
3166 }
3167 if (key.objectid == BTRFS_FIRST_CHUNK_TREE_OBJECTID &&
3168 key.offset == chunk_offset) {
3169 memmove(ptr, ptr + len, array_size - (cur + len));
3170 array_size -= len;
3171 btrfs_set_super_sys_array_size(super_copy, array_size);
3172 } else {
3173 ptr += len;
3174 cur += len;
3175 }
3176 }
3177 return ret;
3178}
3179
3180struct btrfs_chunk_map *btrfs_find_chunk_map_nolock(struct btrfs_fs_info *fs_info,
3181 u64 logical, u64 length)
3182{
3183 struct rb_node *node = fs_info->mapping_tree.rb_root.rb_node;
3184 struct rb_node *prev = NULL;
3185 struct rb_node *orig_prev;
3186 struct btrfs_chunk_map *map;
3187 struct btrfs_chunk_map *prev_map = NULL;
3188
3189 while (node) {
3190 map = rb_entry(node, struct btrfs_chunk_map, rb_node);
3191 prev = node;
3192 prev_map = map;
3193
3194 if (logical < map->start) {
3195 node = node->rb_left;
3196 } else if (logical >= map->start + map->chunk_len) {
3197 node = node->rb_right;
3198 } else {
3199 refcount_inc(&map->refs);
3200 return map;
3201 }
3202 }
3203
3204 if (!prev)
3205 return NULL;
3206
3207 orig_prev = prev;
3208 while (prev && logical >= prev_map->start + prev_map->chunk_len) {
3209 prev = rb_next(prev);
3210 prev_map = rb_entry(prev, struct btrfs_chunk_map, rb_node);
3211 }
3212
3213 if (!prev) {
3214 prev = orig_prev;
3215 prev_map = rb_entry(prev, struct btrfs_chunk_map, rb_node);
3216 while (prev && logical < prev_map->start) {
3217 prev = rb_prev(prev);
3218 prev_map = rb_entry(prev, struct btrfs_chunk_map, rb_node);
3219 }
3220 }
3221
3222 if (prev) {
3223 u64 end = logical + length;
3224
3225 /*
3226 * Caller can pass a U64_MAX length when it wants to get any
3227 * chunk starting at an offset of 'logical' or higher, so deal
3228 * with underflow by resetting the end offset to U64_MAX.
3229 */
3230 if (end < logical)
3231 end = U64_MAX;
3232
3233 if (end > prev_map->start &&
3234 logical < prev_map->start + prev_map->chunk_len) {
3235 refcount_inc(&prev_map->refs);
3236 return prev_map;
3237 }
3238 }
3239
3240 return NULL;
3241}
3242
3243struct btrfs_chunk_map *btrfs_find_chunk_map(struct btrfs_fs_info *fs_info,
3244 u64 logical, u64 length)
3245{
3246 struct btrfs_chunk_map *map;
3247
3248 read_lock(&fs_info->mapping_tree_lock);
3249 map = btrfs_find_chunk_map_nolock(fs_info, logical, length);
3250 read_unlock(&fs_info->mapping_tree_lock);
3251
3252 return map;
3253}
3254
3255/*
3256 * Find the mapping containing the given logical extent.
3257 *
3258 * @logical: Logical block offset in bytes.
3259 * @length: Length of extent in bytes.
3260 *
3261 * Return: Chunk mapping or ERR_PTR.
3262 */
3263struct btrfs_chunk_map *btrfs_get_chunk_map(struct btrfs_fs_info *fs_info,
3264 u64 logical, u64 length)
3265{
3266 struct btrfs_chunk_map *map;
3267
3268 map = btrfs_find_chunk_map(fs_info, logical, length);
3269
3270 if (unlikely(!map)) {
3271 btrfs_crit(fs_info,
3272 "unable to find chunk map for logical %llu length %llu",
3273 logical, length);
3274 return ERR_PTR(-EINVAL);
3275 }
3276
3277 if (unlikely(map->start > logical || map->start + map->chunk_len <= logical)) {
3278 btrfs_crit(fs_info,
3279 "found a bad chunk map, wanted %llu-%llu, found %llu-%llu",
3280 logical, logical + length, map->start,
3281 map->start + map->chunk_len);
3282 btrfs_free_chunk_map(map);
3283 return ERR_PTR(-EINVAL);
3284 }
3285
3286 /* Callers are responsible for dropping the reference. */
3287 return map;
3288}
3289
3290static int remove_chunk_item(struct btrfs_trans_handle *trans,
3291 struct btrfs_chunk_map *map, u64 chunk_offset)
3292{
3293 int i;
3294
3295 /*
3296 * Removing chunk items and updating the device items in the chunks btree
3297 * requires holding the chunk_mutex.
3298 * See the comment at btrfs_chunk_alloc() for the details.
3299 */
3300 lockdep_assert_held(&trans->fs_info->chunk_mutex);
3301
3302 for (i = 0; i < map->num_stripes; i++) {
3303 int ret;
3304
3305 ret = btrfs_update_device(trans, map->stripes[i].dev);
3306 if (ret)
3307 return ret;
3308 }
3309
3310 return btrfs_free_chunk(trans, chunk_offset);
3311}
3312
3313int btrfs_remove_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
3314{
3315 struct btrfs_fs_info *fs_info = trans->fs_info;
3316 struct btrfs_chunk_map *map;
3317 u64 dev_extent_len = 0;
3318 int i, ret = 0;
3319 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
3320
3321 map = btrfs_get_chunk_map(fs_info, chunk_offset, 1);
3322 if (IS_ERR(map)) {
3323 /*
3324 * This is a logic error, but we don't want to just rely on the
3325 * user having built with ASSERT enabled, so if ASSERT doesn't
3326 * do anything we still error out.
3327 */
3328 ASSERT(0);
3329 return PTR_ERR(map);
3330 }
3331
3332 /*
3333 * First delete the device extent items from the devices btree.
3334 * We take the device_list_mutex to avoid racing with the finishing phase
3335 * of a device replace operation. See the comment below before acquiring
3336 * fs_info->chunk_mutex. Note that here we do not acquire the chunk_mutex
3337 * because that can result in a deadlock when deleting the device extent
3338 * items from the devices btree - COWing an extent buffer from the btree
3339 * may result in allocating a new metadata chunk, which would attempt to
3340 * lock again fs_info->chunk_mutex.
3341 */
3342 mutex_lock(&fs_devices->device_list_mutex);
3343 for (i = 0; i < map->num_stripes; i++) {
3344 struct btrfs_device *device = map->stripes[i].dev;
3345 ret = btrfs_free_dev_extent(trans, device,
3346 map->stripes[i].physical,
3347 &dev_extent_len);
3348 if (ret) {
3349 mutex_unlock(&fs_devices->device_list_mutex);
3350 btrfs_abort_transaction(trans, ret);
3351 goto out;
3352 }
3353
3354 if (device->bytes_used > 0) {
3355 mutex_lock(&fs_info->chunk_mutex);
3356 btrfs_device_set_bytes_used(device,
3357 device->bytes_used - dev_extent_len);
3358 atomic64_add(dev_extent_len, &fs_info->free_chunk_space);
3359 btrfs_clear_space_info_full(fs_info);
3360 mutex_unlock(&fs_info->chunk_mutex);
3361 }
3362 }
3363 mutex_unlock(&fs_devices->device_list_mutex);
3364
3365 /*
3366 * We acquire fs_info->chunk_mutex for 2 reasons:
3367 *
3368 * 1) Just like with the first phase of the chunk allocation, we must
3369 * reserve system space, do all chunk btree updates and deletions, and
3370 * update the system chunk array in the superblock while holding this
3371 * mutex. This is for similar reasons as explained on the comment at
3372 * the top of btrfs_chunk_alloc();
3373 *
3374 * 2) Prevent races with the final phase of a device replace operation
3375 * that replaces the device object associated with the map's stripes,
3376 * because the device object's id can change at any time during that
3377 * final phase of the device replace operation
3378 * (dev-replace.c:btrfs_dev_replace_finishing()), so we could grab the
3379 * replaced device and then see it with an ID of
3380 * BTRFS_DEV_REPLACE_DEVID, which would cause a failure when updating
3381 * the device item, which does not exists on the chunk btree.
3382 * The finishing phase of device replace acquires both the
3383 * device_list_mutex and the chunk_mutex, in that order, so we are
3384 * safe by just acquiring the chunk_mutex.
3385 */
3386 trans->removing_chunk = true;
3387 mutex_lock(&fs_info->chunk_mutex);
3388
3389 check_system_chunk(trans, map->type);
3390
3391 ret = remove_chunk_item(trans, map, chunk_offset);
3392 /*
3393 * Normally we should not get -ENOSPC since we reserved space before
3394 * through the call to check_system_chunk().
3395 *
3396 * Despite our system space_info having enough free space, we may not
3397 * be able to allocate extents from its block groups, because all have
3398 * an incompatible profile, which will force us to allocate a new system
3399 * block group with the right profile, or right after we called
3400 * check_system_space() above, a scrub turned the only system block group
3401 * with enough free space into RO mode.
3402 * This is explained with more detail at do_chunk_alloc().
3403 *
3404 * So if we get -ENOSPC, allocate a new system chunk and retry once.
3405 */
3406 if (ret == -ENOSPC) {
3407 const u64 sys_flags = btrfs_system_alloc_profile(fs_info);
3408 struct btrfs_block_group *sys_bg;
3409
3410 sys_bg = btrfs_create_chunk(trans, sys_flags);
3411 if (IS_ERR(sys_bg)) {
3412 ret = PTR_ERR(sys_bg);
3413 btrfs_abort_transaction(trans, ret);
3414 goto out;
3415 }
3416
3417 ret = btrfs_chunk_alloc_add_chunk_item(trans, sys_bg);
3418 if (ret) {
3419 btrfs_abort_transaction(trans, ret);
3420 goto out;
3421 }
3422
3423 ret = remove_chunk_item(trans, map, chunk_offset);
3424 if (ret) {
3425 btrfs_abort_transaction(trans, ret);
3426 goto out;
3427 }
3428 } else if (ret) {
3429 btrfs_abort_transaction(trans, ret);
3430 goto out;
3431 }
3432
3433 trace_btrfs_chunk_free(fs_info, map, chunk_offset, map->chunk_len);
3434
3435 if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
3436 ret = btrfs_del_sys_chunk(fs_info, chunk_offset);
3437 if (ret) {
3438 btrfs_abort_transaction(trans, ret);
3439 goto out;
3440 }
3441 }
3442
3443 mutex_unlock(&fs_info->chunk_mutex);
3444 trans->removing_chunk = false;
3445
3446 /*
3447 * We are done with chunk btree updates and deletions, so release the
3448 * system space we previously reserved (with check_system_chunk()).
3449 */
3450 btrfs_trans_release_chunk_metadata(trans);
3451
3452 ret = btrfs_remove_block_group(trans, map);
3453 if (ret) {
3454 btrfs_abort_transaction(trans, ret);
3455 goto out;
3456 }
3457
3458out:
3459 if (trans->removing_chunk) {
3460 mutex_unlock(&fs_info->chunk_mutex);
3461 trans->removing_chunk = false;
3462 }
3463 /* once for us */
3464 btrfs_free_chunk_map(map);
3465 return ret;
3466}
3467
3468int btrfs_relocate_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
3469{
3470 struct btrfs_root *root = fs_info->chunk_root;
3471 struct btrfs_trans_handle *trans;
3472 struct btrfs_block_group *block_group;
3473 u64 length;
3474 int ret;
3475
3476 if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) {
3477 btrfs_err(fs_info,
3478 "relocate: not supported on extent tree v2 yet");
3479 return -EINVAL;
3480 }
3481
3482 /*
3483 * Prevent races with automatic removal of unused block groups.
3484 * After we relocate and before we remove the chunk with offset
3485 * chunk_offset, automatic removal of the block group can kick in,
3486 * resulting in a failure when calling btrfs_remove_chunk() below.
3487 *
3488 * Make sure to acquire this mutex before doing a tree search (dev
3489 * or chunk trees) to find chunks. Otherwise the cleaner kthread might
3490 * call btrfs_remove_chunk() (through btrfs_delete_unused_bgs()) after
3491 * we release the path used to search the chunk/dev tree and before
3492 * the current task acquires this mutex and calls us.
3493 */
3494 lockdep_assert_held(&fs_info->reclaim_bgs_lock);
3495
3496 /* step one, relocate all the extents inside this chunk */
3497 btrfs_scrub_pause(fs_info);
3498 ret = btrfs_relocate_block_group(fs_info, chunk_offset);
3499 btrfs_scrub_continue(fs_info);
3500 if (ret) {
3501 /*
3502 * If we had a transaction abort, stop all running scrubs.
3503 * See transaction.c:cleanup_transaction() why we do it here.
3504 */
3505 if (BTRFS_FS_ERROR(fs_info))
3506 btrfs_scrub_cancel(fs_info);
3507 return ret;
3508 }
3509
3510 block_group = btrfs_lookup_block_group(fs_info, chunk_offset);
3511 if (!block_group)
3512 return -ENOENT;
3513 btrfs_discard_cancel_work(&fs_info->discard_ctl, block_group);
3514 length = block_group->length;
3515 btrfs_put_block_group(block_group);
3516
3517 /*
3518 * On a zoned file system, discard the whole block group, this will
3519 * trigger a REQ_OP_ZONE_RESET operation on the device zone. If
3520 * resetting the zone fails, don't treat it as a fatal problem from the
3521 * filesystem's point of view.
3522 */
3523 if (btrfs_is_zoned(fs_info)) {
3524 ret = btrfs_discard_extent(fs_info, chunk_offset, length, NULL);
3525 if (ret)
3526 btrfs_info(fs_info,
3527 "failed to reset zone %llu after relocation",
3528 chunk_offset);
3529 }
3530
3531 trans = btrfs_start_trans_remove_block_group(root->fs_info,
3532 chunk_offset);
3533 if (IS_ERR(trans)) {
3534 ret = PTR_ERR(trans);
3535 btrfs_handle_fs_error(root->fs_info, ret, NULL);
3536 return ret;
3537 }
3538
3539 /*
3540 * step two, delete the device extents and the
3541 * chunk tree entries
3542 */
3543 ret = btrfs_remove_chunk(trans, chunk_offset);
3544 btrfs_end_transaction(trans);
3545 return ret;
3546}
3547
3548static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info)
3549{
3550 struct btrfs_root *chunk_root = fs_info->chunk_root;
3551 struct btrfs_path *path;
3552 struct extent_buffer *leaf;
3553 struct btrfs_chunk *chunk;
3554 struct btrfs_key key;
3555 struct btrfs_key found_key;
3556 u64 chunk_type;
3557 bool retried = false;
3558 int failed = 0;
3559 int ret;
3560
3561 path = btrfs_alloc_path();
3562 if (!path)
3563 return -ENOMEM;
3564
3565again:
3566 key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
3567 key.offset = (u64)-1;
3568 key.type = BTRFS_CHUNK_ITEM_KEY;
3569
3570 while (1) {
3571 mutex_lock(&fs_info->reclaim_bgs_lock);
3572 ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
3573 if (ret < 0) {
3574 mutex_unlock(&fs_info->reclaim_bgs_lock);
3575 goto error;
3576 }
3577 if (ret == 0) {
3578 /*
3579 * On the first search we would find chunk tree with
3580 * offset -1, which is not possible. On subsequent
3581 * loops this would find an existing item on an invalid
3582 * offset (one less than the previous one, wrong
3583 * alignment and size).
3584 */
3585 ret = -EUCLEAN;
3586 mutex_unlock(&fs_info->reclaim_bgs_lock);
3587 goto error;
3588 }
3589
3590 ret = btrfs_previous_item(chunk_root, path, key.objectid,
3591 key.type);
3592 if (ret)
3593 mutex_unlock(&fs_info->reclaim_bgs_lock);
3594 if (ret < 0)
3595 goto error;
3596 if (ret > 0)
3597 break;
3598
3599 leaf = path->nodes[0];
3600 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3601
3602 chunk = btrfs_item_ptr(leaf, path->slots[0],
3603 struct btrfs_chunk);
3604 chunk_type = btrfs_chunk_type(leaf, chunk);
3605 btrfs_release_path(path);
3606
3607 if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) {
3608 ret = btrfs_relocate_chunk(fs_info, found_key.offset);
3609 if (ret == -ENOSPC)
3610 failed++;
3611 else
3612 BUG_ON(ret);
3613 }
3614 mutex_unlock(&fs_info->reclaim_bgs_lock);
3615
3616 if (found_key.offset == 0)
3617 break;
3618 key.offset = found_key.offset - 1;
3619 }
3620 ret = 0;
3621 if (failed && !retried) {
3622 failed = 0;
3623 retried = true;
3624 goto again;
3625 } else if (WARN_ON(failed && retried)) {
3626 ret = -ENOSPC;
3627 }
3628error:
3629 btrfs_free_path(path);
3630 return ret;
3631}
3632
3633/*
3634 * return 1 : allocate a data chunk successfully,
3635 * return <0: errors during allocating a data chunk,
3636 * return 0 : no need to allocate a data chunk.
3637 */
3638static int btrfs_may_alloc_data_chunk(struct btrfs_fs_info *fs_info,
3639 u64 chunk_offset)
3640{
3641 struct btrfs_block_group *cache;
3642 u64 bytes_used;
3643 u64 chunk_type;
3644
3645 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3646 ASSERT(cache);
3647 chunk_type = cache->flags;
3648 btrfs_put_block_group(cache);
3649
3650 if (!(chunk_type & BTRFS_BLOCK_GROUP_DATA))
3651 return 0;
3652
3653 spin_lock(&fs_info->data_sinfo->lock);
3654 bytes_used = fs_info->data_sinfo->bytes_used;
3655 spin_unlock(&fs_info->data_sinfo->lock);
3656
3657 if (!bytes_used) {
3658 struct btrfs_trans_handle *trans;
3659 int ret;
3660
3661 trans = btrfs_join_transaction(fs_info->tree_root);
3662 if (IS_ERR(trans))
3663 return PTR_ERR(trans);
3664
3665 ret = btrfs_force_chunk_alloc(trans, BTRFS_BLOCK_GROUP_DATA);
3666 btrfs_end_transaction(trans);
3667 if (ret < 0)
3668 return ret;
3669 return 1;
3670 }
3671
3672 return 0;
3673}
3674
3675static void btrfs_disk_balance_args_to_cpu(struct btrfs_balance_args *cpu,
3676 const struct btrfs_disk_balance_args *disk)
3677{
3678 memset(cpu, 0, sizeof(*cpu));
3679
3680 cpu->profiles = le64_to_cpu(disk->profiles);
3681 cpu->usage = le64_to_cpu(disk->usage);
3682 cpu->devid = le64_to_cpu(disk->devid);
3683 cpu->pstart = le64_to_cpu(disk->pstart);
3684 cpu->pend = le64_to_cpu(disk->pend);
3685 cpu->vstart = le64_to_cpu(disk->vstart);
3686 cpu->vend = le64_to_cpu(disk->vend);
3687 cpu->target = le64_to_cpu(disk->target);
3688 cpu->flags = le64_to_cpu(disk->flags);
3689 cpu->limit = le64_to_cpu(disk->limit);
3690 cpu->stripes_min = le32_to_cpu(disk->stripes_min);
3691 cpu->stripes_max = le32_to_cpu(disk->stripes_max);
3692}
3693
3694static void btrfs_cpu_balance_args_to_disk(struct btrfs_disk_balance_args *disk,
3695 const struct btrfs_balance_args *cpu)
3696{
3697 memset(disk, 0, sizeof(*disk));
3698
3699 disk->profiles = cpu_to_le64(cpu->profiles);
3700 disk->usage = cpu_to_le64(cpu->usage);
3701 disk->devid = cpu_to_le64(cpu->devid);
3702 disk->pstart = cpu_to_le64(cpu->pstart);
3703 disk->pend = cpu_to_le64(cpu->pend);
3704 disk->vstart = cpu_to_le64(cpu->vstart);
3705 disk->vend = cpu_to_le64(cpu->vend);
3706 disk->target = cpu_to_le64(cpu->target);
3707 disk->flags = cpu_to_le64(cpu->flags);
3708 disk->limit = cpu_to_le64(cpu->limit);
3709 disk->stripes_min = cpu_to_le32(cpu->stripes_min);
3710 disk->stripes_max = cpu_to_le32(cpu->stripes_max);
3711}
3712
3713static int insert_balance_item(struct btrfs_fs_info *fs_info,
3714 struct btrfs_balance_control *bctl)
3715{
3716 struct btrfs_root *root = fs_info->tree_root;
3717 struct btrfs_trans_handle *trans;
3718 struct btrfs_balance_item *item;
3719 struct btrfs_disk_balance_args disk_bargs;
3720 struct btrfs_path *path;
3721 struct extent_buffer *leaf;
3722 struct btrfs_key key;
3723 int ret, err;
3724
3725 path = btrfs_alloc_path();
3726 if (!path)
3727 return -ENOMEM;
3728
3729 trans = btrfs_start_transaction(root, 0);
3730 if (IS_ERR(trans)) {
3731 btrfs_free_path(path);
3732 return PTR_ERR(trans);
3733 }
3734
3735 key.objectid = BTRFS_BALANCE_OBJECTID;
3736 key.type = BTRFS_TEMPORARY_ITEM_KEY;
3737 key.offset = 0;
3738
3739 ret = btrfs_insert_empty_item(trans, root, path, &key,
3740 sizeof(*item));
3741 if (ret)
3742 goto out;
3743
3744 leaf = path->nodes[0];
3745 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);
3746
3747 memzero_extent_buffer(leaf, (unsigned long)item, sizeof(*item));
3748
3749 btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->data);
3750 btrfs_set_balance_data(leaf, item, &disk_bargs);
3751 btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->meta);
3752 btrfs_set_balance_meta(leaf, item, &disk_bargs);
3753 btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->sys);
3754 btrfs_set_balance_sys(leaf, item, &disk_bargs);
3755
3756 btrfs_set_balance_flags(leaf, item, bctl->flags);
3757
3758 btrfs_mark_buffer_dirty(trans, leaf);
3759out:
3760 btrfs_free_path(path);
3761 err = btrfs_commit_transaction(trans);
3762 if (err && !ret)
3763 ret = err;
3764 return ret;
3765}
3766
3767static int del_balance_item(struct btrfs_fs_info *fs_info)
3768{
3769 struct btrfs_root *root = fs_info->tree_root;
3770 struct btrfs_trans_handle *trans;
3771 struct btrfs_path *path;
3772 struct btrfs_key key;
3773 int ret, err;
3774
3775 path = btrfs_alloc_path();
3776 if (!path)
3777 return -ENOMEM;
3778
3779 trans = btrfs_start_transaction_fallback_global_rsv(root, 0);
3780 if (IS_ERR(trans)) {
3781 btrfs_free_path(path);
3782 return PTR_ERR(trans);
3783 }
3784
3785 key.objectid = BTRFS_BALANCE_OBJECTID;
3786 key.type = BTRFS_TEMPORARY_ITEM_KEY;
3787 key.offset = 0;
3788
3789 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
3790 if (ret < 0)
3791 goto out;
3792 if (ret > 0) {
3793 ret = -ENOENT;
3794 goto out;
3795 }
3796
3797 ret = btrfs_del_item(trans, root, path);
3798out:
3799 btrfs_free_path(path);
3800 err = btrfs_commit_transaction(trans);
3801 if (err && !ret)
3802 ret = err;
3803 return ret;
3804}
3805
3806/*
3807 * This is a heuristic used to reduce the number of chunks balanced on
3808 * resume after balance was interrupted.
3809 */
3810static void update_balance_args(struct btrfs_balance_control *bctl)
3811{
3812 /*
3813 * Turn on soft mode for chunk types that were being converted.
3814 */
3815 if (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)
3816 bctl->data.flags |= BTRFS_BALANCE_ARGS_SOFT;
3817 if (bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)
3818 bctl->sys.flags |= BTRFS_BALANCE_ARGS_SOFT;
3819 if (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)
3820 bctl->meta.flags |= BTRFS_BALANCE_ARGS_SOFT;
3821
3822 /*
3823 * Turn on usage filter if is not already used. The idea is
3824 * that chunks that we have already balanced should be
3825 * reasonably full. Don't do it for chunks that are being
3826 * converted - that will keep us from relocating unconverted
3827 * (albeit full) chunks.
3828 */
3829 if (!(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3830 !(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3831 !(bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3832 bctl->data.flags |= BTRFS_BALANCE_ARGS_USAGE;
3833 bctl->data.usage = 90;
3834 }
3835 if (!(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3836 !(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3837 !(bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3838 bctl->sys.flags |= BTRFS_BALANCE_ARGS_USAGE;
3839 bctl->sys.usage = 90;
3840 }
3841 if (!(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3842 !(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3843 !(bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3844 bctl->meta.flags |= BTRFS_BALANCE_ARGS_USAGE;
3845 bctl->meta.usage = 90;
3846 }
3847}
3848
3849/*
3850 * Clear the balance status in fs_info and delete the balance item from disk.
3851 */
3852static void reset_balance_state(struct btrfs_fs_info *fs_info)
3853{
3854 struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3855 int ret;
3856
3857 ASSERT(fs_info->balance_ctl);
3858
3859 spin_lock(&fs_info->balance_lock);
3860 fs_info->balance_ctl = NULL;
3861 spin_unlock(&fs_info->balance_lock);
3862
3863 kfree(bctl);
3864 ret = del_balance_item(fs_info);
3865 if (ret)
3866 btrfs_handle_fs_error(fs_info, ret, NULL);
3867}
3868
3869/*
3870 * Balance filters. Return 1 if chunk should be filtered out
3871 * (should not be balanced).
3872 */
3873static int chunk_profiles_filter(u64 chunk_type,
3874 struct btrfs_balance_args *bargs)
3875{
3876 chunk_type = chunk_to_extended(chunk_type) &
3877 BTRFS_EXTENDED_PROFILE_MASK;
3878
3879 if (bargs->profiles & chunk_type)
3880 return 0;
3881
3882 return 1;
3883}
3884
3885static int chunk_usage_range_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset,
3886 struct btrfs_balance_args *bargs)
3887{
3888 struct btrfs_block_group *cache;
3889 u64 chunk_used;
3890 u64 user_thresh_min;
3891 u64 user_thresh_max;
3892 int ret = 1;
3893
3894 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3895 chunk_used = cache->used;
3896
3897 if (bargs->usage_min == 0)
3898 user_thresh_min = 0;
3899 else
3900 user_thresh_min = mult_perc(cache->length, bargs->usage_min);
3901
3902 if (bargs->usage_max == 0)
3903 user_thresh_max = 1;
3904 else if (bargs->usage_max > 100)
3905 user_thresh_max = cache->length;
3906 else
3907 user_thresh_max = mult_perc(cache->length, bargs->usage_max);
3908
3909 if (user_thresh_min <= chunk_used && chunk_used < user_thresh_max)
3910 ret = 0;
3911
3912 btrfs_put_block_group(cache);
3913 return ret;
3914}
3915
3916static int chunk_usage_filter(struct btrfs_fs_info *fs_info,
3917 u64 chunk_offset, struct btrfs_balance_args *bargs)
3918{
3919 struct btrfs_block_group *cache;
3920 u64 chunk_used, user_thresh;
3921 int ret = 1;
3922
3923 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3924 chunk_used = cache->used;
3925
3926 if (bargs->usage_min == 0)
3927 user_thresh = 1;
3928 else if (bargs->usage > 100)
3929 user_thresh = cache->length;
3930 else
3931 user_thresh = mult_perc(cache->length, bargs->usage);
3932
3933 if (chunk_used < user_thresh)
3934 ret = 0;
3935
3936 btrfs_put_block_group(cache);
3937 return ret;
3938}
3939
3940static int chunk_devid_filter(struct extent_buffer *leaf,
3941 struct btrfs_chunk *chunk,
3942 struct btrfs_balance_args *bargs)
3943{
3944 struct btrfs_stripe *stripe;
3945 int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
3946 int i;
3947
3948 for (i = 0; i < num_stripes; i++) {
3949 stripe = btrfs_stripe_nr(chunk, i);
3950 if (btrfs_stripe_devid(leaf, stripe) == bargs->devid)
3951 return 0;
3952 }
3953
3954 return 1;
3955}
3956
3957static u64 calc_data_stripes(u64 type, int num_stripes)
3958{
3959 const int index = btrfs_bg_flags_to_raid_index(type);
3960 const int ncopies = btrfs_raid_array[index].ncopies;
3961 const int nparity = btrfs_raid_array[index].nparity;
3962
3963 return (num_stripes - nparity) / ncopies;
3964}
3965
3966/* [pstart, pend) */
3967static int chunk_drange_filter(struct extent_buffer *leaf,
3968 struct btrfs_chunk *chunk,
3969 struct btrfs_balance_args *bargs)
3970{
3971 struct btrfs_stripe *stripe;
3972 int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
3973 u64 stripe_offset;
3974 u64 stripe_length;
3975 u64 type;
3976 int factor;
3977 int i;
3978
3979 if (!(bargs->flags & BTRFS_BALANCE_ARGS_DEVID))
3980 return 0;
3981
3982 type = btrfs_chunk_type(leaf, chunk);
3983 factor = calc_data_stripes(type, num_stripes);
3984
3985 for (i = 0; i < num_stripes; i++) {
3986 stripe = btrfs_stripe_nr(chunk, i);
3987 if (btrfs_stripe_devid(leaf, stripe) != bargs->devid)
3988 continue;
3989
3990 stripe_offset = btrfs_stripe_offset(leaf, stripe);
3991 stripe_length = btrfs_chunk_length(leaf, chunk);
3992 stripe_length = div_u64(stripe_length, factor);
3993
3994 if (stripe_offset < bargs->pend &&
3995 stripe_offset + stripe_length > bargs->pstart)
3996 return 0;
3997 }
3998
3999 return 1;
4000}
4001
4002/* [vstart, vend) */
4003static int chunk_vrange_filter(struct extent_buffer *leaf,
4004 struct btrfs_chunk *chunk,
4005 u64 chunk_offset,
4006 struct btrfs_balance_args *bargs)
4007{
4008 if (chunk_offset < bargs->vend &&
4009 chunk_offset + btrfs_chunk_length(leaf, chunk) > bargs->vstart)
4010 /* at least part of the chunk is inside this vrange */
4011 return 0;
4012
4013 return 1;
4014}
4015
4016static int chunk_stripes_range_filter(struct extent_buffer *leaf,
4017 struct btrfs_chunk *chunk,
4018 struct btrfs_balance_args *bargs)
4019{
4020 int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
4021
4022 if (bargs->stripes_min <= num_stripes
4023 && num_stripes <= bargs->stripes_max)
4024 return 0;
4025
4026 return 1;
4027}
4028
4029static int chunk_soft_convert_filter(u64 chunk_type,
4030 struct btrfs_balance_args *bargs)
4031{
4032 if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT))
4033 return 0;
4034
4035 chunk_type = chunk_to_extended(chunk_type) &
4036 BTRFS_EXTENDED_PROFILE_MASK;
4037
4038 if (bargs->target == chunk_type)
4039 return 1;
4040
4041 return 0;
4042}
4043
4044static int should_balance_chunk(struct extent_buffer *leaf,
4045 struct btrfs_chunk *chunk, u64 chunk_offset)
4046{
4047 struct btrfs_fs_info *fs_info = leaf->fs_info;
4048 struct btrfs_balance_control *bctl = fs_info->balance_ctl;
4049 struct btrfs_balance_args *bargs = NULL;
4050 u64 chunk_type = btrfs_chunk_type(leaf, chunk);
4051
4052 /* type filter */
4053 if (!((chunk_type & BTRFS_BLOCK_GROUP_TYPE_MASK) &
4054 (bctl->flags & BTRFS_BALANCE_TYPE_MASK))) {
4055 return 0;
4056 }
4057
4058 if (chunk_type & BTRFS_BLOCK_GROUP_DATA)
4059 bargs = &bctl->data;
4060 else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM)
4061 bargs = &bctl->sys;
4062 else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA)
4063 bargs = &bctl->meta;
4064
4065 /* profiles filter */
4066 if ((bargs->flags & BTRFS_BALANCE_ARGS_PROFILES) &&
4067 chunk_profiles_filter(chunk_type, bargs)) {
4068 return 0;
4069 }
4070
4071 /* usage filter */
4072 if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE) &&
4073 chunk_usage_filter(fs_info, chunk_offset, bargs)) {
4074 return 0;
4075 } else if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
4076 chunk_usage_range_filter(fs_info, chunk_offset, bargs)) {
4077 return 0;
4078 }
4079
4080 /* devid filter */
4081 if ((bargs->flags & BTRFS_BALANCE_ARGS_DEVID) &&
4082 chunk_devid_filter(leaf, chunk, bargs)) {
4083 return 0;
4084 }
4085
4086 /* drange filter, makes sense only with devid filter */
4087 if ((bargs->flags & BTRFS_BALANCE_ARGS_DRANGE) &&
4088 chunk_drange_filter(leaf, chunk, bargs)) {
4089 return 0;
4090 }
4091
4092 /* vrange filter */
4093 if ((bargs->flags & BTRFS_BALANCE_ARGS_VRANGE) &&
4094 chunk_vrange_filter(leaf, chunk, chunk_offset, bargs)) {
4095 return 0;
4096 }
4097
4098 /* stripes filter */
4099 if ((bargs->flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE) &&
4100 chunk_stripes_range_filter(leaf, chunk, bargs)) {
4101 return 0;
4102 }
4103
4104 /* soft profile changing mode */
4105 if ((bargs->flags & BTRFS_BALANCE_ARGS_SOFT) &&
4106 chunk_soft_convert_filter(chunk_type, bargs)) {
4107 return 0;
4108 }
4109
4110 /*
4111 * limited by count, must be the last filter
4112 */
4113 if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT)) {
4114 if (bargs->limit == 0)
4115 return 0;
4116 else
4117 bargs->limit--;
4118 } else if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)) {
4119 /*
4120 * Same logic as the 'limit' filter; the minimum cannot be
4121 * determined here because we do not have the global information
4122 * about the count of all chunks that satisfy the filters.
4123 */
4124 if (bargs->limit_max == 0)
4125 return 0;
4126 else
4127 bargs->limit_max--;
4128 }
4129
4130 return 1;
4131}
4132
4133static int __btrfs_balance(struct btrfs_fs_info *fs_info)
4134{
4135 struct btrfs_balance_control *bctl = fs_info->balance_ctl;
4136 struct btrfs_root *chunk_root = fs_info->chunk_root;
4137 u64 chunk_type;
4138 struct btrfs_chunk *chunk;
4139 struct btrfs_path *path = NULL;
4140 struct btrfs_key key;
4141 struct btrfs_key found_key;
4142 struct extent_buffer *leaf;
4143 int slot;
4144 int ret;
4145 int enospc_errors = 0;
4146 bool counting = true;
4147 /* The single value limit and min/max limits use the same bytes in the */
4148 u64 limit_data = bctl->data.limit;
4149 u64 limit_meta = bctl->meta.limit;
4150 u64 limit_sys = bctl->sys.limit;
4151 u32 count_data = 0;
4152 u32 count_meta = 0;
4153 u32 count_sys = 0;
4154 int chunk_reserved = 0;
4155
4156 path = btrfs_alloc_path();
4157 if (!path) {
4158 ret = -ENOMEM;
4159 goto error;
4160 }
4161
4162 /* zero out stat counters */
4163 spin_lock(&fs_info->balance_lock);
4164 memset(&bctl->stat, 0, sizeof(bctl->stat));
4165 spin_unlock(&fs_info->balance_lock);
4166again:
4167 if (!counting) {
4168 /*
4169 * The single value limit and min/max limits use the same bytes
4170 * in the
4171 */
4172 bctl->data.limit = limit_data;
4173 bctl->meta.limit = limit_meta;
4174 bctl->sys.limit = limit_sys;
4175 }
4176 key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
4177 key.offset = (u64)-1;
4178 key.type = BTRFS_CHUNK_ITEM_KEY;
4179
4180 while (1) {
4181 if ((!counting && atomic_read(&fs_info->balance_pause_req)) ||
4182 atomic_read(&fs_info->balance_cancel_req)) {
4183 ret = -ECANCELED;
4184 goto error;
4185 }
4186
4187 mutex_lock(&fs_info->reclaim_bgs_lock);
4188 ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
4189 if (ret < 0) {
4190 mutex_unlock(&fs_info->reclaim_bgs_lock);
4191 goto error;
4192 }
4193
4194 /*
4195 * this shouldn't happen, it means the last relocate
4196 * failed
4197 */
4198 if (ret == 0)
4199 BUG(); /* FIXME break ? */
4200
4201 ret = btrfs_previous_item(chunk_root, path, 0,
4202 BTRFS_CHUNK_ITEM_KEY);
4203 if (ret) {
4204 mutex_unlock(&fs_info->reclaim_bgs_lock);
4205 ret = 0;
4206 break;
4207 }
4208
4209 leaf = path->nodes[0];
4210 slot = path->slots[0];
4211 btrfs_item_key_to_cpu(leaf, &found_key, slot);
4212
4213 if (found_key.objectid != key.objectid) {
4214 mutex_unlock(&fs_info->reclaim_bgs_lock);
4215 break;
4216 }
4217
4218 chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
4219 chunk_type = btrfs_chunk_type(leaf, chunk);
4220
4221 if (!counting) {
4222 spin_lock(&fs_info->balance_lock);
4223 bctl->stat.considered++;
4224 spin_unlock(&fs_info->balance_lock);
4225 }
4226
4227 ret = should_balance_chunk(leaf, chunk, found_key.offset);
4228
4229 btrfs_release_path(path);
4230 if (!ret) {
4231 mutex_unlock(&fs_info->reclaim_bgs_lock);
4232 goto loop;
4233 }
4234
4235 if (counting) {
4236 mutex_unlock(&fs_info->reclaim_bgs_lock);
4237 spin_lock(&fs_info->balance_lock);
4238 bctl->stat.expected++;
4239 spin_unlock(&fs_info->balance_lock);
4240
4241 if (chunk_type & BTRFS_BLOCK_GROUP_DATA)
4242 count_data++;
4243 else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM)
4244 count_sys++;
4245 else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA)
4246 count_meta++;
4247
4248 goto loop;
4249 }
4250
4251 /*
4252 * Apply limit_min filter, no need to check if the LIMITS
4253 * filter is used, limit_min is 0 by default
4254 */
4255 if (((chunk_type & BTRFS_BLOCK_GROUP_DATA) &&
4256 count_data < bctl->data.limit_min)
4257 || ((chunk_type & BTRFS_BLOCK_GROUP_METADATA) &&
4258 count_meta < bctl->meta.limit_min)
4259 || ((chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) &&
4260 count_sys < bctl->sys.limit_min)) {
4261 mutex_unlock(&fs_info->reclaim_bgs_lock);
4262 goto loop;
4263 }
4264
4265 if (!chunk_reserved) {
4266 /*
4267 * We may be relocating the only data chunk we have,
4268 * which could potentially end up with losing data's
4269 * raid profile, so lets allocate an empty one in
4270 * advance.
4271 */
4272 ret = btrfs_may_alloc_data_chunk(fs_info,
4273 found_key.offset);
4274 if (ret < 0) {
4275 mutex_unlock(&fs_info->reclaim_bgs_lock);
4276 goto error;
4277 } else if (ret == 1) {
4278 chunk_reserved = 1;
4279 }
4280 }
4281
4282 ret = btrfs_relocate_chunk(fs_info, found_key.offset);
4283 mutex_unlock(&fs_info->reclaim_bgs_lock);
4284 if (ret == -ENOSPC) {
4285 enospc_errors++;
4286 } else if (ret == -ETXTBSY) {
4287 btrfs_info(fs_info,
4288 "skipping relocation of block group %llu due to active swapfile",
4289 found_key.offset);
4290 ret = 0;
4291 } else if (ret) {
4292 goto error;
4293 } else {
4294 spin_lock(&fs_info->balance_lock);
4295 bctl->stat.completed++;
4296 spin_unlock(&fs_info->balance_lock);
4297 }
4298loop:
4299 if (found_key.offset == 0)
4300 break;
4301 key.offset = found_key.offset - 1;
4302 }
4303
4304 if (counting) {
4305 btrfs_release_path(path);
4306 counting = false;
4307 goto again;
4308 }
4309error:
4310 btrfs_free_path(path);
4311 if (enospc_errors) {
4312 btrfs_info(fs_info, "%d enospc errors during balance",
4313 enospc_errors);
4314 if (!ret)
4315 ret = -ENOSPC;
4316 }
4317
4318 return ret;
4319}
4320
4321/*
4322 * See if a given profile is valid and reduced.
4323 *
4324 * @flags: profile to validate
4325 * @extended: if true @flags is treated as an extended profile
4326 */
4327static int alloc_profile_is_valid(u64 flags, int extended)
4328{
4329 u64 mask = (extended ? BTRFS_EXTENDED_PROFILE_MASK :
4330 BTRFS_BLOCK_GROUP_PROFILE_MASK);
4331
4332 flags &= ~BTRFS_BLOCK_GROUP_TYPE_MASK;
4333
4334 /* 1) check that all other bits are zeroed */
4335 if (flags & ~mask)
4336 return 0;
4337
4338 /* 2) see if profile is reduced */
4339 if (flags == 0)
4340 return !extended; /* "0" is valid for usual profiles */
4341
4342 return has_single_bit_set(flags);
4343}
4344
4345/*
4346 * Validate target profile against allowed profiles and return true if it's OK.
4347 * Otherwise print the error message and return false.
4348 */
4349static inline int validate_convert_profile(struct btrfs_fs_info *fs_info,
4350 const struct btrfs_balance_args *bargs,
4351 u64 allowed, const char *type)
4352{
4353 if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT))
4354 return true;
4355
4356 /* Profile is valid and does not have bits outside of the allowed set */
4357 if (alloc_profile_is_valid(bargs->target, 1) &&
4358 (bargs->target & ~allowed) == 0)
4359 return true;
4360
4361 btrfs_err(fs_info, "balance: invalid convert %s profile %s",
4362 type, btrfs_bg_type_to_raid_name(bargs->target));
4363 return false;
4364}
4365
4366/*
4367 * Fill @buf with textual description of balance filter flags @bargs, up to
4368 * @size_buf including the terminating null. The output may be trimmed if it
4369 * does not fit into the provided buffer.
4370 */
4371static void describe_balance_args(struct btrfs_balance_args *bargs, char *buf,
4372 u32 size_buf)
4373{
4374 int ret;
4375 u32 size_bp = size_buf;
4376 char *bp = buf;
4377 u64 flags = bargs->flags;
4378 char tmp_buf[128] = {'\0'};
4379
4380 if (!flags)
4381 return;
4382
4383#define CHECK_APPEND_NOARG(a) \
4384 do { \
4385 ret = snprintf(bp, size_bp, (a)); \
4386 if (ret < 0 || ret >= size_bp) \
4387 goto out_overflow; \
4388 size_bp -= ret; \
4389 bp += ret; \
4390 } while (0)
4391
4392#define CHECK_APPEND_1ARG(a, v1) \
4393 do { \
4394 ret = snprintf(bp, size_bp, (a), (v1)); \
4395 if (ret < 0 || ret >= size_bp) \
4396 goto out_overflow; \
4397 size_bp -= ret; \
4398 bp += ret; \
4399 } while (0)
4400
4401#define CHECK_APPEND_2ARG(a, v1, v2) \
4402 do { \
4403 ret = snprintf(bp, size_bp, (a), (v1), (v2)); \
4404 if (ret < 0 || ret >= size_bp) \
4405 goto out_overflow; \
4406 size_bp -= ret; \
4407 bp += ret; \
4408 } while (0)
4409
4410 if (flags & BTRFS_BALANCE_ARGS_CONVERT)
4411 CHECK_APPEND_1ARG("convert=%s,",
4412 btrfs_bg_type_to_raid_name(bargs->target));
4413
4414 if (flags & BTRFS_BALANCE_ARGS_SOFT)
4415 CHECK_APPEND_NOARG("soft,");
4416
4417 if (flags & BTRFS_BALANCE_ARGS_PROFILES) {
4418 btrfs_describe_block_groups(bargs->profiles, tmp_buf,
4419 sizeof(tmp_buf));
4420 CHECK_APPEND_1ARG("profiles=%s,", tmp_buf);
4421 }
4422
4423 if (flags & BTRFS_BALANCE_ARGS_USAGE)
4424 CHECK_APPEND_1ARG("usage=%llu,", bargs->usage);
4425
4426 if (flags & BTRFS_BALANCE_ARGS_USAGE_RANGE)
4427 CHECK_APPEND_2ARG("usage=%u..%u,",
4428 bargs->usage_min, bargs->usage_max);
4429
4430 if (flags & BTRFS_BALANCE_ARGS_DEVID)
4431 CHECK_APPEND_1ARG("devid=%llu,", bargs->devid);
4432
4433 if (flags & BTRFS_BALANCE_ARGS_DRANGE)
4434 CHECK_APPEND_2ARG("drange=%llu..%llu,",
4435 bargs->pstart, bargs->pend);
4436
4437 if (flags & BTRFS_BALANCE_ARGS_VRANGE)
4438 CHECK_APPEND_2ARG("vrange=%llu..%llu,",
4439 bargs->vstart, bargs->vend);
4440
4441 if (flags & BTRFS_BALANCE_ARGS_LIMIT)
4442 CHECK_APPEND_1ARG("limit=%llu,", bargs->limit);
4443
4444 if (flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)
4445 CHECK_APPEND_2ARG("limit=%u..%u,",
4446 bargs->limit_min, bargs->limit_max);
4447
4448 if (flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE)
4449 CHECK_APPEND_2ARG("stripes=%u..%u,",
4450 bargs->stripes_min, bargs->stripes_max);
4451
4452#undef CHECK_APPEND_2ARG
4453#undef CHECK_APPEND_1ARG
4454#undef CHECK_APPEND_NOARG
4455
4456out_overflow:
4457
4458 if (size_bp < size_buf)
4459 buf[size_buf - size_bp - 1] = '\0'; /* remove last , */
4460 else
4461 buf[0] = '\0';
4462}
4463
4464static void describe_balance_start_or_resume(struct btrfs_fs_info *fs_info)
4465{
4466 u32 size_buf = 1024;
4467 char tmp_buf[192] = {'\0'};
4468 char *buf;
4469 char *bp;
4470 u32 size_bp = size_buf;
4471 int ret;
4472 struct btrfs_balance_control *bctl = fs_info->balance_ctl;
4473
4474 buf = kzalloc(size_buf, GFP_KERNEL);
4475 if (!buf)
4476 return;
4477
4478 bp = buf;
4479
4480#define CHECK_APPEND_1ARG(a, v1) \
4481 do { \
4482 ret = snprintf(bp, size_bp, (a), (v1)); \
4483 if (ret < 0 || ret >= size_bp) \
4484 goto out_overflow; \
4485 size_bp -= ret; \
4486 bp += ret; \
4487 } while (0)
4488
4489 if (bctl->flags & BTRFS_BALANCE_FORCE)
4490 CHECK_APPEND_1ARG("%s", "-f ");
4491
4492 if (bctl->flags & BTRFS_BALANCE_DATA) {
4493 describe_balance_args(&bctl->data, tmp_buf, sizeof(tmp_buf));
4494 CHECK_APPEND_1ARG("-d%s ", tmp_buf);
4495 }
4496
4497 if (bctl->flags & BTRFS_BALANCE_METADATA) {
4498 describe_balance_args(&bctl->meta, tmp_buf, sizeof(tmp_buf));
4499 CHECK_APPEND_1ARG("-m%s ", tmp_buf);
4500 }
4501
4502 if (bctl->flags & BTRFS_BALANCE_SYSTEM) {
4503 describe_balance_args(&bctl->sys, tmp_buf, sizeof(tmp_buf));
4504 CHECK_APPEND_1ARG("-s%s ", tmp_buf);
4505 }
4506
4507#undef CHECK_APPEND_1ARG
4508
4509out_overflow:
4510
4511 if (size_bp < size_buf)
4512 buf[size_buf - size_bp - 1] = '\0'; /* remove last " " */
4513 btrfs_info(fs_info, "balance: %s %s",
4514 (bctl->flags & BTRFS_BALANCE_RESUME) ?
4515 "resume" : "start", buf);
4516
4517 kfree(buf);
4518}
4519
4520/*
4521 * Should be called with balance mutexe held
4522 */
4523int btrfs_balance(struct btrfs_fs_info *fs_info,
4524 struct btrfs_balance_control *bctl,
4525 struct btrfs_ioctl_balance_args *bargs)
4526{
4527 u64 meta_target, data_target;
4528 u64 allowed;
4529 int mixed = 0;
4530 int ret;
4531 u64 num_devices;
4532 unsigned seq;
4533 bool reducing_redundancy;
4534 bool paused = false;
4535 int i;
4536
4537 if (btrfs_fs_closing(fs_info) ||
4538 atomic_read(&fs_info->balance_pause_req) ||
4539 btrfs_should_cancel_balance(fs_info)) {
4540 ret = -EINVAL;
4541 goto out;
4542 }
4543
4544 allowed = btrfs_super_incompat_flags(fs_info->super_copy);
4545 if (allowed & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
4546 mixed = 1;
4547
4548 /*
4549 * In case of mixed groups both data and meta should be picked,
4550 * and identical options should be given for both of them.
4551 */
4552 allowed = BTRFS_BALANCE_DATA | BTRFS_BALANCE_METADATA;
4553 if (mixed && (bctl->flags & allowed)) {
4554 if (!(bctl->flags & BTRFS_BALANCE_DATA) ||
4555 !(bctl->flags & BTRFS_BALANCE_METADATA) ||
4556 memcmp(&bctl->data, &bctl->meta, sizeof(bctl->data))) {
4557 btrfs_err(fs_info,
4558 "balance: mixed groups data and metadata options must be the same");
4559 ret = -EINVAL;
4560 goto out;
4561 }
4562 }
4563
4564 /*
4565 * rw_devices will not change at the moment, device add/delete/replace
4566 * are exclusive
4567 */
4568 num_devices = fs_info->fs_devices->rw_devices;
4569
4570 /*
4571 * SINGLE profile on-disk has no profile bit, but in-memory we have a
4572 * special bit for it, to make it easier to distinguish. Thus we need
4573 * to set it manually, or balance would refuse the profile.
4574 */
4575 allowed = BTRFS_AVAIL_ALLOC_BIT_SINGLE;
4576 for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++)
4577 if (num_devices >= btrfs_raid_array[i].devs_min)
4578 allowed |= btrfs_raid_array[i].bg_flag;
4579
4580 if (!validate_convert_profile(fs_info, &bctl->data, allowed, "data") ||
4581 !validate_convert_profile(fs_info, &bctl->meta, allowed, "metadata") ||
4582 !validate_convert_profile(fs_info, &bctl->sys, allowed, "system")) {
4583 ret = -EINVAL;
4584 goto out;
4585 }
4586
4587 /*
4588 * Allow to reduce metadata or system integrity only if force set for
4589 * profiles with redundancy (copies, parity)
4590 */
4591 allowed = 0;
4592 for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++) {
4593 if (btrfs_raid_array[i].ncopies >= 2 ||
4594 btrfs_raid_array[i].tolerated_failures >= 1)
4595 allowed |= btrfs_raid_array[i].bg_flag;
4596 }
4597 do {
4598 seq = read_seqbegin(&fs_info->profiles_lock);
4599
4600 if (((bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
4601 (fs_info->avail_system_alloc_bits & allowed) &&
4602 !(bctl->sys.target & allowed)) ||
4603 ((bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
4604 (fs_info->avail_metadata_alloc_bits & allowed) &&
4605 !(bctl->meta.target & allowed)))
4606 reducing_redundancy = true;
4607 else
4608 reducing_redundancy = false;
4609
4610 /* if we're not converting, the target field is uninitialized */
4611 meta_target = (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) ?
4612 bctl->meta.target : fs_info->avail_metadata_alloc_bits;
4613 data_target = (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) ?
4614 bctl->data.target : fs_info->avail_data_alloc_bits;
4615 } while (read_seqretry(&fs_info->profiles_lock, seq));
4616
4617 if (reducing_redundancy) {
4618 if (bctl->flags & BTRFS_BALANCE_FORCE) {
4619 btrfs_info(fs_info,
4620 "balance: force reducing metadata redundancy");
4621 } else {
4622 btrfs_err(fs_info,
4623 "balance: reduces metadata redundancy, use --force if you want this");
4624 ret = -EINVAL;
4625 goto out;
4626 }
4627 }
4628
4629 if (btrfs_get_num_tolerated_disk_barrier_failures(meta_target) <
4630 btrfs_get_num_tolerated_disk_barrier_failures(data_target)) {
4631 btrfs_warn(fs_info,
4632 "balance: metadata profile %s has lower redundancy than data profile %s",
4633 btrfs_bg_type_to_raid_name(meta_target),
4634 btrfs_bg_type_to_raid_name(data_target));
4635 }
4636
4637 ret = insert_balance_item(fs_info, bctl);
4638 if (ret && ret != -EEXIST)
4639 goto out;
4640
4641 if (!(bctl->flags & BTRFS_BALANCE_RESUME)) {
4642 BUG_ON(ret == -EEXIST);
4643 BUG_ON(fs_info->balance_ctl);
4644 spin_lock(&fs_info->balance_lock);
4645 fs_info->balance_ctl = bctl;
4646 spin_unlock(&fs_info->balance_lock);
4647 } else {
4648 BUG_ON(ret != -EEXIST);
4649 spin_lock(&fs_info->balance_lock);
4650 update_balance_args(bctl);
4651 spin_unlock(&fs_info->balance_lock);
4652 }
4653
4654 ASSERT(!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4655 set_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags);
4656 describe_balance_start_or_resume(fs_info);
4657 mutex_unlock(&fs_info->balance_mutex);
4658
4659 ret = __btrfs_balance(fs_info);
4660
4661 mutex_lock(&fs_info->balance_mutex);
4662 if (ret == -ECANCELED && atomic_read(&fs_info->balance_pause_req)) {
4663 btrfs_info(fs_info, "balance: paused");
4664 btrfs_exclop_balance(fs_info, BTRFS_EXCLOP_BALANCE_PAUSED);
4665 paused = true;
4666 }
4667 /*
4668 * Balance can be canceled by:
4669 *
4670 * - Regular cancel request
4671 * Then ret == -ECANCELED and balance_cancel_req > 0
4672 *
4673 * - Fatal signal to "btrfs" process
4674 * Either the signal caught by wait_reserve_ticket() and callers
4675 * got -EINTR, or caught by btrfs_should_cancel_balance() and
4676 * got -ECANCELED.
4677 * Either way, in this case balance_cancel_req = 0, and
4678 * ret == -EINTR or ret == -ECANCELED.
4679 *
4680 * So here we only check the return value to catch canceled balance.
4681 */
4682 else if (ret == -ECANCELED || ret == -EINTR)
4683 btrfs_info(fs_info, "balance: canceled");
4684 else
4685 btrfs_info(fs_info, "balance: ended with status: %d", ret);
4686
4687 clear_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags);
4688
4689 if (bargs) {
4690 memset(bargs, 0, sizeof(*bargs));
4691 btrfs_update_ioctl_balance_args(fs_info, bargs);
4692 }
4693
4694 /* We didn't pause, we can clean everything up. */
4695 if (!paused) {
4696 reset_balance_state(fs_info);
4697 btrfs_exclop_finish(fs_info);
4698 }
4699
4700 wake_up(&fs_info->balance_wait_q);
4701
4702 return ret;
4703out:
4704 if (bctl->flags & BTRFS_BALANCE_RESUME)
4705 reset_balance_state(fs_info);
4706 else
4707 kfree(bctl);
4708 btrfs_exclop_finish(fs_info);
4709
4710 return ret;
4711}
4712
4713static int balance_kthread(void *data)
4714{
4715 struct btrfs_fs_info *fs_info = data;
4716 int ret = 0;
4717
4718 sb_start_write(fs_info->sb);
4719 mutex_lock(&fs_info->balance_mutex);
4720 if (fs_info->balance_ctl)
4721 ret = btrfs_balance(fs_info, fs_info->balance_ctl, NULL);
4722 mutex_unlock(&fs_info->balance_mutex);
4723 sb_end_write(fs_info->sb);
4724
4725 return ret;
4726}
4727
4728int btrfs_resume_balance_async(struct btrfs_fs_info *fs_info)
4729{
4730 struct task_struct *tsk;
4731
4732 mutex_lock(&fs_info->balance_mutex);
4733 if (!fs_info->balance_ctl) {
4734 mutex_unlock(&fs_info->balance_mutex);
4735 return 0;
4736 }
4737 mutex_unlock(&fs_info->balance_mutex);
4738
4739 if (btrfs_test_opt(fs_info, SKIP_BALANCE)) {
4740 btrfs_info(fs_info, "balance: resume skipped");
4741 return 0;
4742 }
4743
4744 spin_lock(&fs_info->super_lock);
4745 ASSERT(fs_info->exclusive_operation == BTRFS_EXCLOP_BALANCE_PAUSED);
4746 fs_info->exclusive_operation = BTRFS_EXCLOP_BALANCE;
4747 spin_unlock(&fs_info->super_lock);
4748 /*
4749 * A ro->rw remount sequence should continue with the paused balance
4750 * regardless of who pauses it, system or the user as of now, so set
4751 * the resume flag.
4752 */
4753 spin_lock(&fs_info->balance_lock);
4754 fs_info->balance_ctl->flags |= BTRFS_BALANCE_RESUME;
4755 spin_unlock(&fs_info->balance_lock);
4756
4757 tsk = kthread_run(balance_kthread, fs_info, "btrfs-balance");
4758 return PTR_ERR_OR_ZERO(tsk);
4759}
4760
4761int btrfs_recover_balance(struct btrfs_fs_info *fs_info)
4762{
4763 struct btrfs_balance_control *bctl;
4764 struct btrfs_balance_item *item;
4765 struct btrfs_disk_balance_args disk_bargs;
4766 struct btrfs_path *path;
4767 struct extent_buffer *leaf;
4768 struct btrfs_key key;
4769 int ret;
4770
4771 path = btrfs_alloc_path();
4772 if (!path)
4773 return -ENOMEM;
4774
4775 key.objectid = BTRFS_BALANCE_OBJECTID;
4776 key.type = BTRFS_TEMPORARY_ITEM_KEY;
4777 key.offset = 0;
4778
4779 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
4780 if (ret < 0)
4781 goto out;
4782 if (ret > 0) { /* ret = -ENOENT; */
4783 ret = 0;
4784 goto out;
4785 }
4786
4787 bctl = kzalloc(sizeof(*bctl), GFP_NOFS);
4788 if (!bctl) {
4789 ret = -ENOMEM;
4790 goto out;
4791 }
4792
4793 leaf = path->nodes[0];
4794 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);
4795
4796 bctl->flags = btrfs_balance_flags(leaf, item);
4797 bctl->flags |= BTRFS_BALANCE_RESUME;
4798
4799 btrfs_balance_data(leaf, item, &disk_bargs);
4800 btrfs_disk_balance_args_to_cpu(&bctl->data, &disk_bargs);
4801 btrfs_balance_meta(leaf, item, &disk_bargs);
4802 btrfs_disk_balance_args_to_cpu(&bctl->meta, &disk_bargs);
4803 btrfs_balance_sys(leaf, item, &disk_bargs);
4804 btrfs_disk_balance_args_to_cpu(&bctl->sys, &disk_bargs);
4805
4806 /*
4807 * This should never happen, as the paused balance state is recovered
4808 * during mount without any chance of other exclusive ops to collide.
4809 *
4810 * This gives the exclusive op status to balance and keeps in paused
4811 * state until user intervention (cancel or umount). If the ownership
4812 * cannot be assigned, show a message but do not fail. The balance
4813 * is in a paused state and must have fs_info::balance_ctl properly
4814 * set up.
4815 */
4816 if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE_PAUSED))
4817 btrfs_warn(fs_info,
4818 "balance: cannot set exclusive op status, resume manually");
4819
4820 btrfs_release_path(path);
4821
4822 mutex_lock(&fs_info->balance_mutex);
4823 BUG_ON(fs_info->balance_ctl);
4824 spin_lock(&fs_info->balance_lock);
4825 fs_info->balance_ctl = bctl;
4826 spin_unlock(&fs_info->balance_lock);
4827 mutex_unlock(&fs_info->balance_mutex);
4828out:
4829 btrfs_free_path(path);
4830 return ret;
4831}
4832
4833int btrfs_pause_balance(struct btrfs_fs_info *fs_info)
4834{
4835 int ret = 0;
4836
4837 mutex_lock(&fs_info->balance_mutex);
4838 if (!fs_info->balance_ctl) {
4839 mutex_unlock(&fs_info->balance_mutex);
4840 return -ENOTCONN;
4841 }
4842
4843 if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
4844 atomic_inc(&fs_info->balance_pause_req);
4845 mutex_unlock(&fs_info->balance_mutex);
4846
4847 wait_event(fs_info->balance_wait_q,
4848 !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4849
4850 mutex_lock(&fs_info->balance_mutex);
4851 /* we are good with balance_ctl ripped off from under us */
4852 BUG_ON(test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4853 atomic_dec(&fs_info->balance_pause_req);
4854 } else {
4855 ret = -ENOTCONN;
4856 }
4857
4858 mutex_unlock(&fs_info->balance_mutex);
4859 return ret;
4860}
4861
4862int btrfs_cancel_balance(struct btrfs_fs_info *fs_info)
4863{
4864 mutex_lock(&fs_info->balance_mutex);
4865 if (!fs_info->balance_ctl) {
4866 mutex_unlock(&fs_info->balance_mutex);
4867 return -ENOTCONN;
4868 }
4869
4870 /*
4871 * A paused balance with the item stored on disk can be resumed at
4872 * mount time if the mount is read-write. Otherwise it's still paused
4873 * and we must not allow cancelling as it deletes the item.
4874 */
4875 if (sb_rdonly(fs_info->sb)) {
4876 mutex_unlock(&fs_info->balance_mutex);
4877 return -EROFS;
4878 }
4879
4880 atomic_inc(&fs_info->balance_cancel_req);
4881 /*
4882 * if we are running just wait and return, balance item is
4883 * deleted in btrfs_balance in this case
4884 */
4885 if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
4886 mutex_unlock(&fs_info->balance_mutex);
4887 wait_event(fs_info->balance_wait_q,
4888 !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4889 mutex_lock(&fs_info->balance_mutex);
4890 } else {
4891 mutex_unlock(&fs_info->balance_mutex);
4892 /*
4893 * Lock released to allow other waiters to continue, we'll
4894 * reexamine the status again.
4895 */
4896 mutex_lock(&fs_info->balance_mutex);
4897
4898 if (fs_info->balance_ctl) {
4899 reset_balance_state(fs_info);
4900 btrfs_exclop_finish(fs_info);
4901 btrfs_info(fs_info, "balance: canceled");
4902 }
4903 }
4904
4905 ASSERT(!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4906 atomic_dec(&fs_info->balance_cancel_req);
4907 mutex_unlock(&fs_info->balance_mutex);
4908 return 0;
4909}
4910
4911/*
4912 * shrinking a device means finding all of the device extents past
4913 * the new size, and then following the back refs to the chunks.
4914 * The chunk relocation code actually frees the device extent
4915 */
4916int btrfs_shrink_device(struct btrfs_device *device, u64 new_size)
4917{
4918 struct btrfs_fs_info *fs_info = device->fs_info;
4919 struct btrfs_root *root = fs_info->dev_root;
4920 struct btrfs_trans_handle *trans;
4921 struct btrfs_dev_extent *dev_extent = NULL;
4922 struct btrfs_path *path;
4923 u64 length;
4924 u64 chunk_offset;
4925 int ret;
4926 int slot;
4927 int failed = 0;
4928 bool retried = false;
4929 struct extent_buffer *l;
4930 struct btrfs_key key;
4931 struct btrfs_super_block *super_copy = fs_info->super_copy;
4932 u64 old_total = btrfs_super_total_bytes(super_copy);
4933 u64 old_size = btrfs_device_get_total_bytes(device);
4934 u64 diff;
4935 u64 start;
4936 u64 free_diff = 0;
4937
4938 new_size = round_down(new_size, fs_info->sectorsize);
4939 start = new_size;
4940 diff = round_down(old_size - new_size, fs_info->sectorsize);
4941
4942 if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
4943 return -EINVAL;
4944
4945 path = btrfs_alloc_path();
4946 if (!path)
4947 return -ENOMEM;
4948
4949 path->reada = READA_BACK;
4950
4951 trans = btrfs_start_transaction(root, 0);
4952 if (IS_ERR(trans)) {
4953 btrfs_free_path(path);
4954 return PTR_ERR(trans);
4955 }
4956
4957 mutex_lock(&fs_info->chunk_mutex);
4958
4959 btrfs_device_set_total_bytes(device, new_size);
4960 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
4961 device->fs_devices->total_rw_bytes -= diff;
4962
4963 /*
4964 * The new free_chunk_space is new_size - used, so we have to
4965 * subtract the delta of the old free_chunk_space which included
4966 * old_size - used. If used > new_size then just subtract this
4967 * entire device's free space.
4968 */
4969 if (device->bytes_used < new_size)
4970 free_diff = (old_size - device->bytes_used) -
4971 (new_size - device->bytes_used);
4972 else
4973 free_diff = old_size - device->bytes_used;
4974 atomic64_sub(free_diff, &fs_info->free_chunk_space);
4975 }
4976
4977 /*
4978 * Once the device's size has been set to the new size, ensure all
4979 * in-memory chunks are synced to disk so that the loop below sees them
4980 * and relocates them accordingly.
4981 */
4982 if (contains_pending_extent(device, &start, diff)) {
4983 mutex_unlock(&fs_info->chunk_mutex);
4984 ret = btrfs_commit_transaction(trans);
4985 if (ret)
4986 goto done;
4987 } else {
4988 mutex_unlock(&fs_info->chunk_mutex);
4989 btrfs_end_transaction(trans);
4990 }
4991
4992again:
4993 key.objectid = device->devid;
4994 key.offset = (u64)-1;
4995 key.type = BTRFS_DEV_EXTENT_KEY;
4996
4997 do {
4998 mutex_lock(&fs_info->reclaim_bgs_lock);
4999 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
5000 if (ret < 0) {
5001 mutex_unlock(&fs_info->reclaim_bgs_lock);
5002 goto done;
5003 }
5004
5005 ret = btrfs_previous_item(root, path, 0, key.type);
5006 if (ret) {
5007 mutex_unlock(&fs_info->reclaim_bgs_lock);
5008 if (ret < 0)
5009 goto done;
5010 ret = 0;
5011 btrfs_release_path(path);
5012 break;
5013 }
5014
5015 l = path->nodes[0];
5016 slot = path->slots[0];
5017 btrfs_item_key_to_cpu(l, &key, path->slots[0]);
5018
5019 if (key.objectid != device->devid) {
5020 mutex_unlock(&fs_info->reclaim_bgs_lock);
5021 btrfs_release_path(path);
5022 break;
5023 }
5024
5025 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
5026 length = btrfs_dev_extent_length(l, dev_extent);
5027
5028 if (key.offset + length <= new_size) {
5029 mutex_unlock(&fs_info->reclaim_bgs_lock);
5030 btrfs_release_path(path);
5031 break;
5032 }
5033
5034 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
5035 btrfs_release_path(path);
5036
5037 /*
5038 * We may be relocating the only data chunk we have,
5039 * which could potentially end up with losing data's
5040 * raid profile, so lets allocate an empty one in
5041 * advance.
5042 */
5043 ret = btrfs_may_alloc_data_chunk(fs_info, chunk_offset);
5044 if (ret < 0) {
5045 mutex_unlock(&fs_info->reclaim_bgs_lock);
5046 goto done;
5047 }
5048
5049 ret = btrfs_relocate_chunk(fs_info, chunk_offset);
5050 mutex_unlock(&fs_info->reclaim_bgs_lock);
5051 if (ret == -ENOSPC) {
5052 failed++;
5053 } else if (ret) {
5054 if (ret == -ETXTBSY) {
5055 btrfs_warn(fs_info,
5056 "could not shrink block group %llu due to active swapfile",
5057 chunk_offset);
5058 }
5059 goto done;
5060 }
5061 } while (key.offset-- > 0);
5062
5063 if (failed && !retried) {
5064 failed = 0;
5065 retried = true;
5066 goto again;
5067 } else if (failed && retried) {
5068 ret = -ENOSPC;
5069 goto done;
5070 }
5071
5072 /* Shrinking succeeded, else we would be at "done". */
5073 trans = btrfs_start_transaction(root, 0);
5074 if (IS_ERR(trans)) {
5075 ret = PTR_ERR(trans);
5076 goto done;
5077 }
5078
5079 mutex_lock(&fs_info->chunk_mutex);
5080 /* Clear all state bits beyond the shrunk device size */
5081 clear_extent_bits(&device->alloc_state, new_size, (u64)-1,
5082 CHUNK_STATE_MASK);
5083
5084 btrfs_device_set_disk_total_bytes(device, new_size);
5085 if (list_empty(&device->post_commit_list))
5086 list_add_tail(&device->post_commit_list,
5087 &trans->transaction->dev_update_list);
5088
5089 WARN_ON(diff > old_total);
5090 btrfs_set_super_total_bytes(super_copy,
5091 round_down(old_total - diff, fs_info->sectorsize));
5092 mutex_unlock(&fs_info->chunk_mutex);
5093
5094 btrfs_reserve_chunk_metadata(trans, false);
5095 /* Now btrfs_update_device() will change the on-disk size. */
5096 ret = btrfs_update_device(trans, device);
5097 btrfs_trans_release_chunk_metadata(trans);
5098 if (ret < 0) {
5099 btrfs_abort_transaction(trans, ret);
5100 btrfs_end_transaction(trans);
5101 } else {
5102 ret = btrfs_commit_transaction(trans);
5103 }
5104done:
5105 btrfs_free_path(path);
5106 if (ret) {
5107 mutex_lock(&fs_info->chunk_mutex);
5108 btrfs_device_set_total_bytes(device, old_size);
5109 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
5110 device->fs_devices->total_rw_bytes += diff;
5111 atomic64_add(free_diff, &fs_info->free_chunk_space);
5112 }
5113 mutex_unlock(&fs_info->chunk_mutex);
5114 }
5115 return ret;
5116}
5117
5118static int btrfs_add_system_chunk(struct btrfs_fs_info *fs_info,
5119 struct btrfs_key *key,
5120 struct btrfs_chunk *chunk, int item_size)
5121{
5122 struct btrfs_super_block *super_copy = fs_info->super_copy;
5123 struct btrfs_disk_key disk_key;
5124 u32 array_size;
5125 u8 *ptr;
5126
5127 lockdep_assert_held(&fs_info->chunk_mutex);
5128
5129 array_size = btrfs_super_sys_array_size(super_copy);
5130 if (array_size + item_size + sizeof(disk_key)
5131 > BTRFS_SYSTEM_CHUNK_ARRAY_SIZE)
5132 return -EFBIG;
5133
5134 ptr = super_copy->sys_chunk_array + array_size;
5135 btrfs_cpu_key_to_disk(&disk_key, key);
5136 memcpy(ptr, &disk_key, sizeof(disk_key));
5137 ptr += sizeof(disk_key);
5138 memcpy(ptr, chunk, item_size);
5139 item_size += sizeof(disk_key);
5140 btrfs_set_super_sys_array_size(super_copy, array_size + item_size);
5141
5142 return 0;
5143}
5144
5145/*
5146 * sort the devices in descending order by max_avail, total_avail
5147 */
5148static int btrfs_cmp_device_info(const void *a, const void *b)
5149{
5150 const struct btrfs_device_info *di_a = a;
5151 const struct btrfs_device_info *di_b = b;
5152
5153 if (di_a->max_avail > di_b->max_avail)
5154 return -1;
5155 if (di_a->max_avail < di_b->max_avail)
5156 return 1;
5157 if (di_a->total_avail > di_b->total_avail)
5158 return -1;
5159 if (di_a->total_avail < di_b->total_avail)
5160 return 1;
5161 return 0;
5162}
5163
5164static void check_raid56_incompat_flag(struct btrfs_fs_info *info, u64 type)
5165{
5166 if (!(type & BTRFS_BLOCK_GROUP_RAID56_MASK))
5167 return;
5168
5169 btrfs_set_fs_incompat(info, RAID56);
5170}
5171
5172static void check_raid1c34_incompat_flag(struct btrfs_fs_info *info, u64 type)
5173{
5174 if (!(type & (BTRFS_BLOCK_GROUP_RAID1C3 | BTRFS_BLOCK_GROUP_RAID1C4)))
5175 return;
5176
5177 btrfs_set_fs_incompat(info, RAID1C34);
5178}
5179
5180/*
5181 * Structure used internally for btrfs_create_chunk() function.
5182 * Wraps needed parameters.
5183 */
5184struct alloc_chunk_ctl {
5185 u64 start;
5186 u64 type;
5187 /* Total number of stripes to allocate */
5188 int num_stripes;
5189 /* sub_stripes info for map */
5190 int sub_stripes;
5191 /* Stripes per device */
5192 int dev_stripes;
5193 /* Maximum number of devices to use */
5194 int devs_max;
5195 /* Minimum number of devices to use */
5196 int devs_min;
5197 /* ndevs has to be a multiple of this */
5198 int devs_increment;
5199 /* Number of copies */
5200 int ncopies;
5201 /* Number of stripes worth of bytes to store parity information */
5202 int nparity;
5203 u64 max_stripe_size;
5204 u64 max_chunk_size;
5205 u64 dev_extent_min;
5206 u64 stripe_size;
5207 u64 chunk_size;
5208 int ndevs;
5209};
5210
5211static void init_alloc_chunk_ctl_policy_regular(
5212 struct btrfs_fs_devices *fs_devices,
5213 struct alloc_chunk_ctl *ctl)
5214{
5215 struct btrfs_space_info *space_info;
5216
5217 space_info = btrfs_find_space_info(fs_devices->fs_info, ctl->type);
5218 ASSERT(space_info);
5219
5220 ctl->max_chunk_size = READ_ONCE(space_info->chunk_size);
5221 ctl->max_stripe_size = min_t(u64, ctl->max_chunk_size, SZ_1G);
5222
5223 if (ctl->type & BTRFS_BLOCK_GROUP_SYSTEM)
5224 ctl->devs_max = min_t(int, ctl->devs_max, BTRFS_MAX_DEVS_SYS_CHUNK);
5225
5226 /* We don't want a chunk larger than 10% of writable space */
5227 ctl->max_chunk_size = min(mult_perc(fs_devices->total_rw_bytes, 10),
5228 ctl->max_chunk_size);
5229 ctl->dev_extent_min = btrfs_stripe_nr_to_offset(ctl->dev_stripes);
5230}
5231
5232static void init_alloc_chunk_ctl_policy_zoned(
5233 struct btrfs_fs_devices *fs_devices,
5234 struct alloc_chunk_ctl *ctl)
5235{
5236 u64 zone_size = fs_devices->fs_info->zone_size;
5237 u64 limit;
5238 int min_num_stripes = ctl->devs_min * ctl->dev_stripes;
5239 int min_data_stripes = (min_num_stripes - ctl->nparity) / ctl->ncopies;
5240 u64 min_chunk_size = min_data_stripes * zone_size;
5241 u64 type = ctl->type;
5242
5243 ctl->max_stripe_size = zone_size;
5244 if (type & BTRFS_BLOCK_GROUP_DATA) {
5245 ctl->max_chunk_size = round_down(BTRFS_MAX_DATA_CHUNK_SIZE,
5246 zone_size);
5247 } else if (type & BTRFS_BLOCK_GROUP_METADATA) {
5248 ctl->max_chunk_size = ctl->max_stripe_size;
5249 } else if (type & BTRFS_BLOCK_GROUP_SYSTEM) {
5250 ctl->max_chunk_size = 2 * ctl->max_stripe_size;
5251 ctl->devs_max = min_t(int, ctl->devs_max,
5252 BTRFS_MAX_DEVS_SYS_CHUNK);
5253 } else {
5254 BUG();
5255 }
5256
5257 /* We don't want a chunk larger than 10% of writable space */
5258 limit = max(round_down(mult_perc(fs_devices->total_rw_bytes, 10),
5259 zone_size),
5260 min_chunk_size);
5261 ctl->max_chunk_size = min(limit, ctl->max_chunk_size);
5262 ctl->dev_extent_min = zone_size * ctl->dev_stripes;
5263}
5264
5265static void init_alloc_chunk_ctl(struct btrfs_fs_devices *fs_devices,
5266 struct alloc_chunk_ctl *ctl)
5267{
5268 int index = btrfs_bg_flags_to_raid_index(ctl->type);
5269
5270 ctl->sub_stripes = btrfs_raid_array[index].sub_stripes;
5271 ctl->dev_stripes = btrfs_raid_array[index].dev_stripes;
5272 ctl->devs_max = btrfs_raid_array[index].devs_max;
5273 if (!ctl->devs_max)
5274 ctl->devs_max = BTRFS_MAX_DEVS(fs_devices->fs_info);
5275 ctl->devs_min = btrfs_raid_array[index].devs_min;
5276 ctl->devs_increment = btrfs_raid_array[index].devs_increment;
5277 ctl->ncopies = btrfs_raid_array[index].ncopies;
5278 ctl->nparity = btrfs_raid_array[index].nparity;
5279 ctl->ndevs = 0;
5280
5281 switch (fs_devices->chunk_alloc_policy) {
5282 case BTRFS_CHUNK_ALLOC_REGULAR:
5283 init_alloc_chunk_ctl_policy_regular(fs_devices, ctl);
5284 break;
5285 case BTRFS_CHUNK_ALLOC_ZONED:
5286 init_alloc_chunk_ctl_policy_zoned(fs_devices, ctl);
5287 break;
5288 default:
5289 BUG();
5290 }
5291}
5292
5293static int gather_device_info(struct btrfs_fs_devices *fs_devices,
5294 struct alloc_chunk_ctl *ctl,
5295 struct btrfs_device_info *devices_info)
5296{
5297 struct btrfs_fs_info *info = fs_devices->fs_info;
5298 struct btrfs_device *device;
5299 u64 total_avail;
5300 u64 dev_extent_want = ctl->max_stripe_size * ctl->dev_stripes;
5301 int ret;
5302 int ndevs = 0;
5303 u64 max_avail;
5304 u64 dev_offset;
5305
5306 /*
5307 * in the first pass through the devices list, we gather information
5308 * about the available holes on each device.
5309 */
5310 list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
5311 if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
5312 WARN(1, KERN_ERR
5313 "BTRFS: read-only device in alloc_list\n");
5314 continue;
5315 }
5316
5317 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
5318 &device->dev_state) ||
5319 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
5320 continue;
5321
5322 if (device->total_bytes > device->bytes_used)
5323 total_avail = device->total_bytes - device->bytes_used;
5324 else
5325 total_avail = 0;
5326
5327 /* If there is no space on this device, skip it. */
5328 if (total_avail < ctl->dev_extent_min)
5329 continue;
5330
5331 ret = find_free_dev_extent(device, dev_extent_want, &dev_offset,
5332 &max_avail);
5333 if (ret && ret != -ENOSPC)
5334 return ret;
5335
5336 if (ret == 0)
5337 max_avail = dev_extent_want;
5338
5339 if (max_avail < ctl->dev_extent_min) {
5340 if (btrfs_test_opt(info, ENOSPC_DEBUG))
5341 btrfs_debug(info,
5342 "%s: devid %llu has no free space, have=%llu want=%llu",
5343 __func__, device->devid, max_avail,
5344 ctl->dev_extent_min);
5345 continue;
5346 }
5347
5348 if (ndevs == fs_devices->rw_devices) {
5349 WARN(1, "%s: found more than %llu devices\n",
5350 __func__, fs_devices->rw_devices);
5351 break;
5352 }
5353 devices_info[ndevs].dev_offset = dev_offset;
5354 devices_info[ndevs].max_avail = max_avail;
5355 devices_info[ndevs].total_avail = total_avail;
5356 devices_info[ndevs].dev = device;
5357 ++ndevs;
5358 }
5359 ctl->ndevs = ndevs;
5360
5361 /*
5362 * now sort the devices by hole size / available space
5363 */
5364 sort(devices_info, ndevs, sizeof(struct btrfs_device_info),
5365 btrfs_cmp_device_info, NULL);
5366
5367 return 0;
5368}
5369
5370static int decide_stripe_size_regular(struct alloc_chunk_ctl *ctl,
5371 struct btrfs_device_info *devices_info)
5372{
5373 /* Number of stripes that count for block group size */
5374 int data_stripes;
5375
5376 /*
5377 * The primary goal is to maximize the number of stripes, so use as
5378 * many devices as possible, even if the stripes are not maximum sized.
5379 *
5380 * The DUP profile stores more than one stripe per device, the
5381 * max_avail is the total size so we have to adjust.
5382 */
5383 ctl->stripe_size = div_u64(devices_info[ctl->ndevs - 1].max_avail,
5384 ctl->dev_stripes);
5385 ctl->num_stripes = ctl->ndevs * ctl->dev_stripes;
5386
5387 /* This will have to be fixed for RAID1 and RAID10 over more drives */
5388 data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies;
5389
5390 /*
5391 * Use the number of data stripes to figure out how big this chunk is
5392 * really going to be in terms of logical address space, and compare
5393 * that answer with the max chunk size. If it's higher, we try to
5394 * reduce stripe_size.
5395 */
5396 if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) {
5397 /*
5398 * Reduce stripe_size, round it up to a 16MB boundary again and
5399 * then use it, unless it ends up being even bigger than the
5400 * previous value we had already.
5401 */
5402 ctl->stripe_size = min(round_up(div_u64(ctl->max_chunk_size,
5403 data_stripes), SZ_16M),
5404 ctl->stripe_size);
5405 }
5406
5407 /* Stripe size should not go beyond 1G. */
5408 ctl->stripe_size = min_t(u64, ctl->stripe_size, SZ_1G);
5409
5410 /* Align to BTRFS_STRIPE_LEN */
5411 ctl->stripe_size = round_down(ctl->stripe_size, BTRFS_STRIPE_LEN);
5412 ctl->chunk_size = ctl->stripe_size * data_stripes;
5413
5414 return 0;
5415}
5416
5417static int decide_stripe_size_zoned(struct alloc_chunk_ctl *ctl,
5418 struct btrfs_device_info *devices_info)
5419{
5420 u64 zone_size = devices_info[0].dev->zone_info->zone_size;
5421 /* Number of stripes that count for block group size */
5422 int data_stripes;
5423
5424 /*
5425 * It should hold because:
5426 * dev_extent_min == dev_extent_want == zone_size * dev_stripes
5427 */
5428 ASSERT(devices_info[ctl->ndevs - 1].max_avail == ctl->dev_extent_min);
5429
5430 ctl->stripe_size = zone_size;
5431 ctl->num_stripes = ctl->ndevs * ctl->dev_stripes;
5432 data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies;
5433
5434 /* stripe_size is fixed in zoned filesystem. Reduce ndevs instead. */
5435 if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) {
5436 ctl->ndevs = div_u64(div_u64(ctl->max_chunk_size * ctl->ncopies,
5437 ctl->stripe_size) + ctl->nparity,
5438 ctl->dev_stripes);
5439 ctl->num_stripes = ctl->ndevs * ctl->dev_stripes;
5440 data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies;
5441 ASSERT(ctl->stripe_size * data_stripes <= ctl->max_chunk_size);
5442 }
5443
5444 ctl->chunk_size = ctl->stripe_size * data_stripes;
5445
5446 return 0;
5447}
5448
5449static int decide_stripe_size(struct btrfs_fs_devices *fs_devices,
5450 struct alloc_chunk_ctl *ctl,
5451 struct btrfs_device_info *devices_info)
5452{
5453 struct btrfs_fs_info *info = fs_devices->fs_info;
5454
5455 /*
5456 * Round down to number of usable stripes, devs_increment can be any
5457 * number so we can't use round_down() that requires power of 2, while
5458 * rounddown is safe.
5459 */
5460 ctl->ndevs = rounddown(ctl->ndevs, ctl->devs_increment);
5461
5462 if (ctl->ndevs < ctl->devs_min) {
5463 if (btrfs_test_opt(info, ENOSPC_DEBUG)) {
5464 btrfs_debug(info,
5465 "%s: not enough devices with free space: have=%d minimum required=%d",
5466 __func__, ctl->ndevs, ctl->devs_min);
5467 }
5468 return -ENOSPC;
5469 }
5470
5471 ctl->ndevs = min(ctl->ndevs, ctl->devs_max);
5472
5473 switch (fs_devices->chunk_alloc_policy) {
5474 case BTRFS_CHUNK_ALLOC_REGULAR:
5475 return decide_stripe_size_regular(ctl, devices_info);
5476 case BTRFS_CHUNK_ALLOC_ZONED:
5477 return decide_stripe_size_zoned(ctl, devices_info);
5478 default:
5479 BUG();
5480 }
5481}
5482
5483static void chunk_map_device_set_bits(struct btrfs_chunk_map *map, unsigned int bits)
5484{
5485 for (int i = 0; i < map->num_stripes; i++) {
5486 struct btrfs_io_stripe *stripe = &map->stripes[i];
5487 struct btrfs_device *device = stripe->dev;
5488
5489 set_extent_bit(&device->alloc_state, stripe->physical,
5490 stripe->physical + map->stripe_size - 1,
5491 bits | EXTENT_NOWAIT, NULL);
5492 }
5493}
5494
5495static void chunk_map_device_clear_bits(struct btrfs_chunk_map *map, unsigned int bits)
5496{
5497 for (int i = 0; i < map->num_stripes; i++) {
5498 struct btrfs_io_stripe *stripe = &map->stripes[i];
5499 struct btrfs_device *device = stripe->dev;
5500
5501 __clear_extent_bit(&device->alloc_state, stripe->physical,
5502 stripe->physical + map->stripe_size - 1,
5503 bits | EXTENT_NOWAIT,
5504 NULL, NULL);
5505 }
5506}
5507
5508void btrfs_remove_chunk_map(struct btrfs_fs_info *fs_info, struct btrfs_chunk_map *map)
5509{
5510 write_lock(&fs_info->mapping_tree_lock);
5511 rb_erase_cached(&map->rb_node, &fs_info->mapping_tree);
5512 RB_CLEAR_NODE(&map->rb_node);
5513 chunk_map_device_clear_bits(map, CHUNK_ALLOCATED);
5514 write_unlock(&fs_info->mapping_tree_lock);
5515
5516 /* Once for the tree reference. */
5517 btrfs_free_chunk_map(map);
5518}
5519
5520EXPORT_FOR_TESTS
5521int btrfs_add_chunk_map(struct btrfs_fs_info *fs_info, struct btrfs_chunk_map *map)
5522{
5523 struct rb_node **p;
5524 struct rb_node *parent = NULL;
5525 bool leftmost = true;
5526
5527 write_lock(&fs_info->mapping_tree_lock);
5528 p = &fs_info->mapping_tree.rb_root.rb_node;
5529 while (*p) {
5530 struct btrfs_chunk_map *entry;
5531
5532 parent = *p;
5533 entry = rb_entry(parent, struct btrfs_chunk_map, rb_node);
5534
5535 if (map->start < entry->start) {
5536 p = &(*p)->rb_left;
5537 } else if (map->start > entry->start) {
5538 p = &(*p)->rb_right;
5539 leftmost = false;
5540 } else {
5541 write_unlock(&fs_info->mapping_tree_lock);
5542 return -EEXIST;
5543 }
5544 }
5545 rb_link_node(&map->rb_node, parent, p);
5546 rb_insert_color_cached(&map->rb_node, &fs_info->mapping_tree, leftmost);
5547 chunk_map_device_set_bits(map, CHUNK_ALLOCATED);
5548 chunk_map_device_clear_bits(map, CHUNK_TRIMMED);
5549 write_unlock(&fs_info->mapping_tree_lock);
5550
5551 return 0;
5552}
5553
5554EXPORT_FOR_TESTS
5555struct btrfs_chunk_map *btrfs_alloc_chunk_map(int num_stripes, gfp_t gfp)
5556{
5557 struct btrfs_chunk_map *map;
5558
5559 map = kmalloc(btrfs_chunk_map_size(num_stripes), gfp);
5560 if (!map)
5561 return NULL;
5562
5563 refcount_set(&map->refs, 1);
5564 RB_CLEAR_NODE(&map->rb_node);
5565
5566 return map;
5567}
5568
5569static struct btrfs_block_group *create_chunk(struct btrfs_trans_handle *trans,
5570 struct alloc_chunk_ctl *ctl,
5571 struct btrfs_device_info *devices_info)
5572{
5573 struct btrfs_fs_info *info = trans->fs_info;
5574 struct btrfs_chunk_map *map;
5575 struct btrfs_block_group *block_group;
5576 u64 start = ctl->start;
5577 u64 type = ctl->type;
5578 int ret;
5579
5580 map = btrfs_alloc_chunk_map(ctl->num_stripes, GFP_NOFS);
5581 if (!map)
5582 return ERR_PTR(-ENOMEM);
5583
5584 map->start = start;
5585 map->chunk_len = ctl->chunk_size;
5586 map->stripe_size = ctl->stripe_size;
5587 map->type = type;
5588 map->io_align = BTRFS_STRIPE_LEN;
5589 map->io_width = BTRFS_STRIPE_LEN;
5590 map->sub_stripes = ctl->sub_stripes;
5591 map->num_stripes = ctl->num_stripes;
5592
5593 for (int i = 0; i < ctl->ndevs; i++) {
5594 for (int j = 0; j < ctl->dev_stripes; j++) {
5595 int s = i * ctl->dev_stripes + j;
5596 map->stripes[s].dev = devices_info[i].dev;
5597 map->stripes[s].physical = devices_info[i].dev_offset +
5598 j * ctl->stripe_size;
5599 }
5600 }
5601
5602 trace_btrfs_chunk_alloc(info, map, start, ctl->chunk_size);
5603
5604 ret = btrfs_add_chunk_map(info, map);
5605 if (ret) {
5606 btrfs_free_chunk_map(map);
5607 return ERR_PTR(ret);
5608 }
5609
5610 block_group = btrfs_make_block_group(trans, type, start, ctl->chunk_size);
5611 if (IS_ERR(block_group)) {
5612 btrfs_remove_chunk_map(info, map);
5613 return block_group;
5614 }
5615
5616 for (int i = 0; i < map->num_stripes; i++) {
5617 struct btrfs_device *dev = map->stripes[i].dev;
5618
5619 btrfs_device_set_bytes_used(dev,
5620 dev->bytes_used + ctl->stripe_size);
5621 if (list_empty(&dev->post_commit_list))
5622 list_add_tail(&dev->post_commit_list,
5623 &trans->transaction->dev_update_list);
5624 }
5625
5626 atomic64_sub(ctl->stripe_size * map->num_stripes,
5627 &info->free_chunk_space);
5628
5629 check_raid56_incompat_flag(info, type);
5630 check_raid1c34_incompat_flag(info, type);
5631
5632 return block_group;
5633}
5634
5635struct btrfs_block_group *btrfs_create_chunk(struct btrfs_trans_handle *trans,
5636 u64 type)
5637{
5638 struct btrfs_fs_info *info = trans->fs_info;
5639 struct btrfs_fs_devices *fs_devices = info->fs_devices;
5640 struct btrfs_device_info *devices_info = NULL;
5641 struct alloc_chunk_ctl ctl;
5642 struct btrfs_block_group *block_group;
5643 int ret;
5644
5645 lockdep_assert_held(&info->chunk_mutex);
5646
5647 if (!alloc_profile_is_valid(type, 0)) {
5648 ASSERT(0);
5649 return ERR_PTR(-EINVAL);
5650 }
5651
5652 if (list_empty(&fs_devices->alloc_list)) {
5653 if (btrfs_test_opt(info, ENOSPC_DEBUG))
5654 btrfs_debug(info, "%s: no writable device", __func__);
5655 return ERR_PTR(-ENOSPC);
5656 }
5657
5658 if (!(type & BTRFS_BLOCK_GROUP_TYPE_MASK)) {
5659 btrfs_err(info, "invalid chunk type 0x%llx requested", type);
5660 ASSERT(0);
5661 return ERR_PTR(-EINVAL);
5662 }
5663
5664 ctl.start = find_next_chunk(info);
5665 ctl.type = type;
5666 init_alloc_chunk_ctl(fs_devices, &ctl);
5667
5668 devices_info = kcalloc(fs_devices->rw_devices, sizeof(*devices_info),
5669 GFP_NOFS);
5670 if (!devices_info)
5671 return ERR_PTR(-ENOMEM);
5672
5673 ret = gather_device_info(fs_devices, &ctl, devices_info);
5674 if (ret < 0) {
5675 block_group = ERR_PTR(ret);
5676 goto out;
5677 }
5678
5679 ret = decide_stripe_size(fs_devices, &ctl, devices_info);
5680 if (ret < 0) {
5681 block_group = ERR_PTR(ret);
5682 goto out;
5683 }
5684
5685 block_group = create_chunk(trans, &ctl, devices_info);
5686
5687out:
5688 kfree(devices_info);
5689 return block_group;
5690}
5691
5692/*
5693 * This function, btrfs_chunk_alloc_add_chunk_item(), typically belongs to the
5694 * phase 1 of chunk allocation. It belongs to phase 2 only when allocating system
5695 * chunks.
5696 *
5697 * See the comment at btrfs_chunk_alloc() for details about the chunk allocation
5698 * phases.
5699 */
5700int btrfs_chunk_alloc_add_chunk_item(struct btrfs_trans_handle *trans,
5701 struct btrfs_block_group *bg)
5702{
5703 struct btrfs_fs_info *fs_info = trans->fs_info;
5704 struct btrfs_root *chunk_root = fs_info->chunk_root;
5705 struct btrfs_key key;
5706 struct btrfs_chunk *chunk;
5707 struct btrfs_stripe *stripe;
5708 struct btrfs_chunk_map *map;
5709 size_t item_size;
5710 int i;
5711 int ret;
5712
5713 /*
5714 * We take the chunk_mutex for 2 reasons:
5715 *
5716 * 1) Updates and insertions in the chunk btree must be done while holding
5717 * the chunk_mutex, as well as updating the system chunk array in the
5718 * superblock. See the comment on top of btrfs_chunk_alloc() for the
5719 * details;
5720 *
5721 * 2) To prevent races with the final phase of a device replace operation
5722 * that replaces the device object associated with the map's stripes,
5723 * because the device object's id can change at any time during that
5724 * final phase of the device replace operation
5725 * (dev-replace.c:btrfs_dev_replace_finishing()), so we could grab the
5726 * replaced device and then see it with an ID of BTRFS_DEV_REPLACE_DEVID,
5727 * which would cause a failure when updating the device item, which does
5728 * not exists, or persisting a stripe of the chunk item with such ID.
5729 * Here we can't use the device_list_mutex because our caller already
5730 * has locked the chunk_mutex, and the final phase of device replace
5731 * acquires both mutexes - first the device_list_mutex and then the
5732 * chunk_mutex. Using any of those two mutexes protects us from a
5733 * concurrent device replace.
5734 */
5735 lockdep_assert_held(&fs_info->chunk_mutex);
5736
5737 map = btrfs_get_chunk_map(fs_info, bg->start, bg->length);
5738 if (IS_ERR(map)) {
5739 ret = PTR_ERR(map);
5740 btrfs_abort_transaction(trans, ret);
5741 return ret;
5742 }
5743
5744 item_size = btrfs_chunk_item_size(map->num_stripes);
5745
5746 chunk = kzalloc(item_size, GFP_NOFS);
5747 if (!chunk) {
5748 ret = -ENOMEM;
5749 btrfs_abort_transaction(trans, ret);
5750 goto out;
5751 }
5752
5753 for (i = 0; i < map->num_stripes; i++) {
5754 struct btrfs_device *device = map->stripes[i].dev;
5755
5756 ret = btrfs_update_device(trans, device);
5757 if (ret)
5758 goto out;
5759 }
5760
5761 stripe = &chunk->stripe;
5762 for (i = 0; i < map->num_stripes; i++) {
5763 struct btrfs_device *device = map->stripes[i].dev;
5764 const u64 dev_offset = map->stripes[i].physical;
5765
5766 btrfs_set_stack_stripe_devid(stripe, device->devid);
5767 btrfs_set_stack_stripe_offset(stripe, dev_offset);
5768 memcpy(stripe->dev_uuid, device->uuid, BTRFS_UUID_SIZE);
5769 stripe++;
5770 }
5771
5772 btrfs_set_stack_chunk_length(chunk, bg->length);
5773 btrfs_set_stack_chunk_owner(chunk, BTRFS_EXTENT_TREE_OBJECTID);
5774 btrfs_set_stack_chunk_stripe_len(chunk, BTRFS_STRIPE_LEN);
5775 btrfs_set_stack_chunk_type(chunk, map->type);
5776 btrfs_set_stack_chunk_num_stripes(chunk, map->num_stripes);
5777 btrfs_set_stack_chunk_io_align(chunk, BTRFS_STRIPE_LEN);
5778 btrfs_set_stack_chunk_io_width(chunk, BTRFS_STRIPE_LEN);
5779 btrfs_set_stack_chunk_sector_size(chunk, fs_info->sectorsize);
5780 btrfs_set_stack_chunk_sub_stripes(chunk, map->sub_stripes);
5781
5782 key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
5783 key.type = BTRFS_CHUNK_ITEM_KEY;
5784 key.offset = bg->start;
5785
5786 ret = btrfs_insert_item(trans, chunk_root, &key, chunk, item_size);
5787 if (ret)
5788 goto out;
5789
5790 set_bit(BLOCK_GROUP_FLAG_CHUNK_ITEM_INSERTED, &bg->runtime_flags);
5791
5792 if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
5793 ret = btrfs_add_system_chunk(fs_info, &key, chunk, item_size);
5794 if (ret)
5795 goto out;
5796 }
5797
5798out:
5799 kfree(chunk);
5800 btrfs_free_chunk_map(map);
5801 return ret;
5802}
5803
5804static noinline int init_first_rw_device(struct btrfs_trans_handle *trans)
5805{
5806 struct btrfs_fs_info *fs_info = trans->fs_info;
5807 u64 alloc_profile;
5808 struct btrfs_block_group *meta_bg;
5809 struct btrfs_block_group *sys_bg;
5810
5811 /*
5812 * When adding a new device for sprouting, the seed device is read-only
5813 * so we must first allocate a metadata and a system chunk. But before
5814 * adding the block group items to the extent, device and chunk btrees,
5815 * we must first:
5816 *
5817 * 1) Create both chunks without doing any changes to the btrees, as
5818 * otherwise we would get -ENOSPC since the block groups from the
5819 * seed device are read-only;
5820 *
5821 * 2) Add the device item for the new sprout device - finishing the setup
5822 * of a new block group requires updating the device item in the chunk
5823 * btree, so it must exist when we attempt to do it. The previous step
5824 * ensures this does not fail with -ENOSPC.
5825 *
5826 * After that we can add the block group items to their btrees:
5827 * update existing device item in the chunk btree, add a new block group
5828 * item to the extent btree, add a new chunk item to the chunk btree and
5829 * finally add the new device extent items to the devices btree.
5830 */
5831
5832 alloc_profile = btrfs_metadata_alloc_profile(fs_info);
5833 meta_bg = btrfs_create_chunk(trans, alloc_profile);
5834 if (IS_ERR(meta_bg))
5835 return PTR_ERR(meta_bg);
5836
5837 alloc_profile = btrfs_system_alloc_profile(fs_info);
5838 sys_bg = btrfs_create_chunk(trans, alloc_profile);
5839 if (IS_ERR(sys_bg))
5840 return PTR_ERR(sys_bg);
5841
5842 return 0;
5843}
5844
5845static inline int btrfs_chunk_max_errors(struct btrfs_chunk_map *map)
5846{
5847 const int index = btrfs_bg_flags_to_raid_index(map->type);
5848
5849 return btrfs_raid_array[index].tolerated_failures;
5850}
5851
5852bool btrfs_chunk_writeable(struct btrfs_fs_info *fs_info, u64 chunk_offset)
5853{
5854 struct btrfs_chunk_map *map;
5855 int miss_ndevs = 0;
5856 int i;
5857 bool ret = true;
5858
5859 map = btrfs_get_chunk_map(fs_info, chunk_offset, 1);
5860 if (IS_ERR(map))
5861 return false;
5862
5863 for (i = 0; i < map->num_stripes; i++) {
5864 if (test_bit(BTRFS_DEV_STATE_MISSING,
5865 &map->stripes[i].dev->dev_state)) {
5866 miss_ndevs++;
5867 continue;
5868 }
5869 if (!test_bit(BTRFS_DEV_STATE_WRITEABLE,
5870 &map->stripes[i].dev->dev_state)) {
5871 ret = false;
5872 goto end;
5873 }
5874 }
5875
5876 /*
5877 * If the number of missing devices is larger than max errors, we can
5878 * not write the data into that chunk successfully.
5879 */
5880 if (miss_ndevs > btrfs_chunk_max_errors(map))
5881 ret = false;
5882end:
5883 btrfs_free_chunk_map(map);
5884 return ret;
5885}
5886
5887void btrfs_mapping_tree_free(struct btrfs_fs_info *fs_info)
5888{
5889 write_lock(&fs_info->mapping_tree_lock);
5890 while (!RB_EMPTY_ROOT(&fs_info->mapping_tree.rb_root)) {
5891 struct btrfs_chunk_map *map;
5892 struct rb_node *node;
5893
5894 node = rb_first_cached(&fs_info->mapping_tree);
5895 map = rb_entry(node, struct btrfs_chunk_map, rb_node);
5896 rb_erase_cached(&map->rb_node, &fs_info->mapping_tree);
5897 RB_CLEAR_NODE(&map->rb_node);
5898 chunk_map_device_clear_bits(map, CHUNK_ALLOCATED);
5899 /* Once for the tree ref. */
5900 btrfs_free_chunk_map(map);
5901 cond_resched_rwlock_write(&fs_info->mapping_tree_lock);
5902 }
5903 write_unlock(&fs_info->mapping_tree_lock);
5904}
5905
5906static int btrfs_chunk_map_num_copies(const struct btrfs_chunk_map *map)
5907{
5908 enum btrfs_raid_types index = btrfs_bg_flags_to_raid_index(map->type);
5909
5910 if (map->type & BTRFS_BLOCK_GROUP_RAID5)
5911 return 2;
5912
5913 /*
5914 * There could be two corrupted data stripes, we need to loop retry in
5915 * order to rebuild the correct data.
5916 *
5917 * Fail a stripe at a time on every retry except the stripe under
5918 * reconstruction.
5919 */
5920 if (map->type & BTRFS_BLOCK_GROUP_RAID6)
5921 return map->num_stripes;
5922
5923 /* Non-RAID56, use their ncopies from btrfs_raid_array. */
5924 return btrfs_raid_array[index].ncopies;
5925}
5926
5927int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len)
5928{
5929 struct btrfs_chunk_map *map;
5930 int ret;
5931
5932 map = btrfs_get_chunk_map(fs_info, logical, len);
5933 if (IS_ERR(map))
5934 /*
5935 * We could return errors for these cases, but that could get
5936 * ugly and we'd probably do the same thing which is just not do
5937 * anything else and exit, so return 1 so the callers don't try
5938 * to use other copies.
5939 */
5940 return 1;
5941
5942 ret = btrfs_chunk_map_num_copies(map);
5943 btrfs_free_chunk_map(map);
5944 return ret;
5945}
5946
5947unsigned long btrfs_full_stripe_len(struct btrfs_fs_info *fs_info,
5948 u64 logical)
5949{
5950 struct btrfs_chunk_map *map;
5951 unsigned long len = fs_info->sectorsize;
5952
5953 if (!btrfs_fs_incompat(fs_info, RAID56))
5954 return len;
5955
5956 map = btrfs_get_chunk_map(fs_info, logical, len);
5957
5958 if (!WARN_ON(IS_ERR(map))) {
5959 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
5960 len = btrfs_stripe_nr_to_offset(nr_data_stripes(map));
5961 btrfs_free_chunk_map(map);
5962 }
5963 return len;
5964}
5965
5966static int find_live_mirror(struct btrfs_fs_info *fs_info,
5967 struct btrfs_chunk_map *map, int first,
5968 int dev_replace_is_ongoing)
5969{
5970 const enum btrfs_read_policy policy = READ_ONCE(fs_info->fs_devices->read_policy);
5971 int i;
5972 int num_stripes;
5973 int preferred_mirror;
5974 int tolerance;
5975 struct btrfs_device *srcdev;
5976
5977 ASSERT((map->type &
5978 (BTRFS_BLOCK_GROUP_RAID1_MASK | BTRFS_BLOCK_GROUP_RAID10)));
5979
5980 if (map->type & BTRFS_BLOCK_GROUP_RAID10)
5981 num_stripes = map->sub_stripes;
5982 else
5983 num_stripes = map->num_stripes;
5984
5985 switch (policy) {
5986 default:
5987 /* Shouldn't happen, just warn and use pid instead of failing */
5988 btrfs_warn_rl(fs_info, "unknown read_policy type %u, reset to pid",
5989 policy);
5990 WRITE_ONCE(fs_info->fs_devices->read_policy, BTRFS_READ_POLICY_PID);
5991 fallthrough;
5992 case BTRFS_READ_POLICY_PID:
5993 preferred_mirror = first + (current->pid % num_stripes);
5994 break;
5995 }
5996
5997 if (dev_replace_is_ongoing &&
5998 fs_info->dev_replace.cont_reading_from_srcdev_mode ==
5999 BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_AVOID)
6000 srcdev = fs_info->dev_replace.srcdev;
6001 else
6002 srcdev = NULL;
6003
6004 /*
6005 * try to avoid the drive that is the source drive for a
6006 * dev-replace procedure, only choose it if no other non-missing
6007 * mirror is available
6008 */
6009 for (tolerance = 0; tolerance < 2; tolerance++) {
6010 if (map->stripes[preferred_mirror].dev->bdev &&
6011 (tolerance || map->stripes[preferred_mirror].dev != srcdev))
6012 return preferred_mirror;
6013 for (i = first; i < first + num_stripes; i++) {
6014 if (map->stripes[i].dev->bdev &&
6015 (tolerance || map->stripes[i].dev != srcdev))
6016 return i;
6017 }
6018 }
6019
6020 /* we couldn't find one that doesn't fail. Just return something
6021 * and the io error handling code will clean up eventually
6022 */
6023 return preferred_mirror;
6024}
6025
6026EXPORT_FOR_TESTS
6027struct btrfs_io_context *alloc_btrfs_io_context(struct btrfs_fs_info *fs_info,
6028 u64 logical, u16 total_stripes)
6029{
6030 struct btrfs_io_context *bioc;
6031
6032 bioc = kzalloc(
6033 /* The size of btrfs_io_context */
6034 sizeof(struct btrfs_io_context) +
6035 /* Plus the variable array for the stripes */
6036 sizeof(struct btrfs_io_stripe) * (total_stripes),
6037 GFP_NOFS);
6038
6039 if (!bioc)
6040 return NULL;
6041
6042 refcount_set(&bioc->refs, 1);
6043
6044 bioc->fs_info = fs_info;
6045 bioc->replace_stripe_src = -1;
6046 bioc->full_stripe_logical = (u64)-1;
6047 bioc->logical = logical;
6048
6049 return bioc;
6050}
6051
6052void btrfs_get_bioc(struct btrfs_io_context *bioc)
6053{
6054 WARN_ON(!refcount_read(&bioc->refs));
6055 refcount_inc(&bioc->refs);
6056}
6057
6058void btrfs_put_bioc(struct btrfs_io_context *bioc)
6059{
6060 if (!bioc)
6061 return;
6062 if (refcount_dec_and_test(&bioc->refs))
6063 kfree(bioc);
6064}
6065
6066/*
6067 * Please note that, discard won't be sent to target device of device
6068 * replace.
6069 */
6070struct btrfs_discard_stripe *btrfs_map_discard(struct btrfs_fs_info *fs_info,
6071 u64 logical, u64 *length_ret,
6072 u32 *num_stripes)
6073{
6074 struct btrfs_chunk_map *map;
6075 struct btrfs_discard_stripe *stripes;
6076 u64 length = *length_ret;
6077 u64 offset;
6078 u32 stripe_nr;
6079 u32 stripe_nr_end;
6080 u32 stripe_cnt;
6081 u64 stripe_end_offset;
6082 u64 stripe_offset;
6083 u32 stripe_index;
6084 u32 factor = 0;
6085 u32 sub_stripes = 0;
6086 u32 stripes_per_dev = 0;
6087 u32 remaining_stripes = 0;
6088 u32 last_stripe = 0;
6089 int ret;
6090 int i;
6091
6092 map = btrfs_get_chunk_map(fs_info, logical, length);
6093 if (IS_ERR(map))
6094 return ERR_CAST(map);
6095
6096 /* we don't discard raid56 yet */
6097 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
6098 ret = -EOPNOTSUPP;
6099 goto out_free_map;
6100 }
6101
6102 offset = logical - map->start;
6103 length = min_t(u64, map->start + map->chunk_len - logical, length);
6104 *length_ret = length;
6105
6106 /*
6107 * stripe_nr counts the total number of stripes we have to stride
6108 * to get to this block
6109 */
6110 stripe_nr = offset >> BTRFS_STRIPE_LEN_SHIFT;
6111
6112 /* stripe_offset is the offset of this block in its stripe */
6113 stripe_offset = offset - btrfs_stripe_nr_to_offset(stripe_nr);
6114
6115 stripe_nr_end = round_up(offset + length, BTRFS_STRIPE_LEN) >>
6116 BTRFS_STRIPE_LEN_SHIFT;
6117 stripe_cnt = stripe_nr_end - stripe_nr;
6118 stripe_end_offset = btrfs_stripe_nr_to_offset(stripe_nr_end) -
6119 (offset + length);
6120 /*
6121 * after this, stripe_nr is the number of stripes on this
6122 * device we have to walk to find the data, and stripe_index is
6123 * the number of our device in the stripe array
6124 */
6125 *num_stripes = 1;
6126 stripe_index = 0;
6127 if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
6128 BTRFS_BLOCK_GROUP_RAID10)) {
6129 if (map->type & BTRFS_BLOCK_GROUP_RAID0)
6130 sub_stripes = 1;
6131 else
6132 sub_stripes = map->sub_stripes;
6133
6134 factor = map->num_stripes / sub_stripes;
6135 *num_stripes = min_t(u64, map->num_stripes,
6136 sub_stripes * stripe_cnt);
6137 stripe_index = stripe_nr % factor;
6138 stripe_nr /= factor;
6139 stripe_index *= sub_stripes;
6140
6141 remaining_stripes = stripe_cnt % factor;
6142 stripes_per_dev = stripe_cnt / factor;
6143 last_stripe = ((stripe_nr_end - 1) % factor) * sub_stripes;
6144 } else if (map->type & (BTRFS_BLOCK_GROUP_RAID1_MASK |
6145 BTRFS_BLOCK_GROUP_DUP)) {
6146 *num_stripes = map->num_stripes;
6147 } else {
6148 stripe_index = stripe_nr % map->num_stripes;
6149 stripe_nr /= map->num_stripes;
6150 }
6151
6152 stripes = kcalloc(*num_stripes, sizeof(*stripes), GFP_NOFS);
6153 if (!stripes) {
6154 ret = -ENOMEM;
6155 goto out_free_map;
6156 }
6157
6158 for (i = 0; i < *num_stripes; i++) {
6159 stripes[i].physical =
6160 map->stripes[stripe_index].physical +
6161 stripe_offset + btrfs_stripe_nr_to_offset(stripe_nr);
6162 stripes[i].dev = map->stripes[stripe_index].dev;
6163
6164 if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
6165 BTRFS_BLOCK_GROUP_RAID10)) {
6166 stripes[i].length = btrfs_stripe_nr_to_offset(stripes_per_dev);
6167
6168 if (i / sub_stripes < remaining_stripes)
6169 stripes[i].length += BTRFS_STRIPE_LEN;
6170
6171 /*
6172 * Special for the first stripe and
6173 * the last stripe:
6174 *
6175 * |-------|...|-------|
6176 * |----------|
6177 * off end_off
6178 */
6179 if (i < sub_stripes)
6180 stripes[i].length -= stripe_offset;
6181
6182 if (stripe_index >= last_stripe &&
6183 stripe_index <= (last_stripe +
6184 sub_stripes - 1))
6185 stripes[i].length -= stripe_end_offset;
6186
6187 if (i == sub_stripes - 1)
6188 stripe_offset = 0;
6189 } else {
6190 stripes[i].length = length;
6191 }
6192
6193 stripe_index++;
6194 if (stripe_index == map->num_stripes) {
6195 stripe_index = 0;
6196 stripe_nr++;
6197 }
6198 }
6199
6200 btrfs_free_chunk_map(map);
6201 return stripes;
6202out_free_map:
6203 btrfs_free_chunk_map(map);
6204 return ERR_PTR(ret);
6205}
6206
6207static bool is_block_group_to_copy(struct btrfs_fs_info *fs_info, u64 logical)
6208{
6209 struct btrfs_block_group *cache;
6210 bool ret;
6211
6212 /* Non zoned filesystem does not use "to_copy" flag */
6213 if (!btrfs_is_zoned(fs_info))
6214 return false;
6215
6216 cache = btrfs_lookup_block_group(fs_info, logical);
6217
6218 ret = test_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags);
6219
6220 btrfs_put_block_group(cache);
6221 return ret;
6222}
6223
6224static void handle_ops_on_dev_replace(struct btrfs_io_context *bioc,
6225 struct btrfs_dev_replace *dev_replace,
6226 u64 logical,
6227 struct btrfs_io_geometry *io_geom)
6228{
6229 u64 srcdev_devid = dev_replace->srcdev->devid;
6230 /*
6231 * At this stage, num_stripes is still the real number of stripes,
6232 * excluding the duplicated stripes.
6233 */
6234 int num_stripes = io_geom->num_stripes;
6235 int max_errors = io_geom->max_errors;
6236 int nr_extra_stripes = 0;
6237 int i;
6238
6239 /*
6240 * A block group which has "to_copy" set will eventually be copied by
6241 * the dev-replace process. We can avoid cloning IO here.
6242 */
6243 if (is_block_group_to_copy(dev_replace->srcdev->fs_info, logical))
6244 return;
6245
6246 /*
6247 * Duplicate the write operations while the dev-replace procedure is
6248 * running. Since the copying of the old disk to the new disk takes
6249 * place at run time while the filesystem is mounted writable, the
6250 * regular write operations to the old disk have to be duplicated to go
6251 * to the new disk as well.
6252 *
6253 * Note that device->missing is handled by the caller, and that the
6254 * write to the old disk is already set up in the stripes array.
6255 */
6256 for (i = 0; i < num_stripes; i++) {
6257 struct btrfs_io_stripe *old = &bioc->stripes[i];
6258 struct btrfs_io_stripe *new = &bioc->stripes[num_stripes + nr_extra_stripes];
6259
6260 if (old->dev->devid != srcdev_devid)
6261 continue;
6262
6263 new->physical = old->physical;
6264 new->dev = dev_replace->tgtdev;
6265 if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK)
6266 bioc->replace_stripe_src = i;
6267 nr_extra_stripes++;
6268 }
6269
6270 /* We can only have at most 2 extra nr_stripes (for DUP). */
6271 ASSERT(nr_extra_stripes <= 2);
6272 /*
6273 * For GET_READ_MIRRORS, we can only return at most 1 extra stripe for
6274 * replace.
6275 * If we have 2 extra stripes, only choose the one with smaller physical.
6276 */
6277 if (io_geom->op == BTRFS_MAP_GET_READ_MIRRORS && nr_extra_stripes == 2) {
6278 struct btrfs_io_stripe *first = &bioc->stripes[num_stripes];
6279 struct btrfs_io_stripe *second = &bioc->stripes[num_stripes + 1];
6280
6281 /* Only DUP can have two extra stripes. */
6282 ASSERT(bioc->map_type & BTRFS_BLOCK_GROUP_DUP);
6283
6284 /*
6285 * Swap the last stripe stripes and reduce @nr_extra_stripes.
6286 * The extra stripe would still be there, but won't be accessed.
6287 */
6288 if (first->physical > second->physical) {
6289 swap(second->physical, first->physical);
6290 swap(second->dev, first->dev);
6291 nr_extra_stripes--;
6292 }
6293 }
6294
6295 io_geom->num_stripes = num_stripes + nr_extra_stripes;
6296 io_geom->max_errors = max_errors + nr_extra_stripes;
6297 bioc->replace_nr_stripes = nr_extra_stripes;
6298}
6299
6300static u64 btrfs_max_io_len(struct btrfs_chunk_map *map, u64 offset,
6301 struct btrfs_io_geometry *io_geom)
6302{
6303 /*
6304 * Stripe_nr is the stripe where this block falls. stripe_offset is
6305 * the offset of this block in its stripe.
6306 */
6307 io_geom->stripe_offset = offset & BTRFS_STRIPE_LEN_MASK;
6308 io_geom->stripe_nr = offset >> BTRFS_STRIPE_LEN_SHIFT;
6309 ASSERT(io_geom->stripe_offset < U32_MAX);
6310
6311 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
6312 unsigned long full_stripe_len =
6313 btrfs_stripe_nr_to_offset(nr_data_stripes(map));
6314
6315 /*
6316 * For full stripe start, we use previously calculated
6317 * @stripe_nr. Align it to nr_data_stripes, then multiply with
6318 * STRIPE_LEN.
6319 *
6320 * By this we can avoid u64 division completely. And we have
6321 * to go rounddown(), not round_down(), as nr_data_stripes is
6322 * not ensured to be power of 2.
6323 */
6324 io_geom->raid56_full_stripe_start = btrfs_stripe_nr_to_offset(
6325 rounddown(io_geom->stripe_nr, nr_data_stripes(map)));
6326
6327 ASSERT(io_geom->raid56_full_stripe_start + full_stripe_len > offset);
6328 ASSERT(io_geom->raid56_full_stripe_start <= offset);
6329 /*
6330 * For writes to RAID56, allow to write a full stripe set, but
6331 * no straddling of stripe sets.
6332 */
6333 if (io_geom->op == BTRFS_MAP_WRITE)
6334 return full_stripe_len - (offset - io_geom->raid56_full_stripe_start);
6335 }
6336
6337 /*
6338 * For other RAID types and for RAID56 reads, allow a single stripe (on
6339 * a single disk).
6340 */
6341 if (map->type & BTRFS_BLOCK_GROUP_STRIPE_MASK)
6342 return BTRFS_STRIPE_LEN - io_geom->stripe_offset;
6343 return U64_MAX;
6344}
6345
6346static int set_io_stripe(struct btrfs_fs_info *fs_info, u64 logical,
6347 u64 *length, struct btrfs_io_stripe *dst,
6348 struct btrfs_chunk_map *map,
6349 struct btrfs_io_geometry *io_geom)
6350{
6351 dst->dev = map->stripes[io_geom->stripe_index].dev;
6352
6353 if (io_geom->op == BTRFS_MAP_READ &&
6354 btrfs_need_stripe_tree_update(fs_info, map->type))
6355 return btrfs_get_raid_extent_offset(fs_info, logical, length,
6356 map->type,
6357 io_geom->stripe_index, dst);
6358
6359 dst->physical = map->stripes[io_geom->stripe_index].physical +
6360 io_geom->stripe_offset +
6361 btrfs_stripe_nr_to_offset(io_geom->stripe_nr);
6362 return 0;
6363}
6364
6365static bool is_single_device_io(struct btrfs_fs_info *fs_info,
6366 const struct btrfs_io_stripe *smap,
6367 const struct btrfs_chunk_map *map,
6368 int num_alloc_stripes,
6369 enum btrfs_map_op op, int mirror_num)
6370{
6371 if (!smap)
6372 return false;
6373
6374 if (num_alloc_stripes != 1)
6375 return false;
6376
6377 if (btrfs_need_stripe_tree_update(fs_info, map->type) && op != BTRFS_MAP_READ)
6378 return false;
6379
6380 if ((map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) && mirror_num > 1)
6381 return false;
6382
6383 return true;
6384}
6385
6386static void map_blocks_raid0(const struct btrfs_chunk_map *map,
6387 struct btrfs_io_geometry *io_geom)
6388{
6389 io_geom->stripe_index = io_geom->stripe_nr % map->num_stripes;
6390 io_geom->stripe_nr /= map->num_stripes;
6391 if (io_geom->op == BTRFS_MAP_READ)
6392 io_geom->mirror_num = 1;
6393}
6394
6395static void map_blocks_raid1(struct btrfs_fs_info *fs_info,
6396 struct btrfs_chunk_map *map,
6397 struct btrfs_io_geometry *io_geom,
6398 bool dev_replace_is_ongoing)
6399{
6400 if (io_geom->op != BTRFS_MAP_READ) {
6401 io_geom->num_stripes = map->num_stripes;
6402 return;
6403 }
6404
6405 if (io_geom->mirror_num) {
6406 io_geom->stripe_index = io_geom->mirror_num - 1;
6407 return;
6408 }
6409
6410 io_geom->stripe_index = find_live_mirror(fs_info, map, 0,
6411 dev_replace_is_ongoing);
6412 io_geom->mirror_num = io_geom->stripe_index + 1;
6413}
6414
6415static void map_blocks_dup(const struct btrfs_chunk_map *map,
6416 struct btrfs_io_geometry *io_geom)
6417{
6418 if (io_geom->op != BTRFS_MAP_READ) {
6419 io_geom->num_stripes = map->num_stripes;
6420 return;
6421 }
6422
6423 if (io_geom->mirror_num) {
6424 io_geom->stripe_index = io_geom->mirror_num - 1;
6425 return;
6426 }
6427
6428 io_geom->mirror_num = 1;
6429}
6430
6431static void map_blocks_raid10(struct btrfs_fs_info *fs_info,
6432 struct btrfs_chunk_map *map,
6433 struct btrfs_io_geometry *io_geom,
6434 bool dev_replace_is_ongoing)
6435{
6436 u32 factor = map->num_stripes / map->sub_stripes;
6437 int old_stripe_index;
6438
6439 io_geom->stripe_index = (io_geom->stripe_nr % factor) * map->sub_stripes;
6440 io_geom->stripe_nr /= factor;
6441
6442 if (io_geom->op != BTRFS_MAP_READ) {
6443 io_geom->num_stripes = map->sub_stripes;
6444 return;
6445 }
6446
6447 if (io_geom->mirror_num) {
6448 io_geom->stripe_index += io_geom->mirror_num - 1;
6449 return;
6450 }
6451
6452 old_stripe_index = io_geom->stripe_index;
6453 io_geom->stripe_index = find_live_mirror(fs_info, map,
6454 io_geom->stripe_index,
6455 dev_replace_is_ongoing);
6456 io_geom->mirror_num = io_geom->stripe_index - old_stripe_index + 1;
6457}
6458
6459static void map_blocks_raid56_write(struct btrfs_chunk_map *map,
6460 struct btrfs_io_geometry *io_geom,
6461 u64 logical, u64 *length)
6462{
6463 int data_stripes = nr_data_stripes(map);
6464
6465 /*
6466 * Needs full stripe mapping.
6467 *
6468 * Push stripe_nr back to the start of the full stripe For those cases
6469 * needing a full stripe, @stripe_nr is the full stripe number.
6470 *
6471 * Originally we go raid56_full_stripe_start / full_stripe_len, but
6472 * that can be expensive. Here we just divide @stripe_nr with
6473 * @data_stripes.
6474 */
6475 io_geom->stripe_nr /= data_stripes;
6476
6477 /* RAID[56] write or recovery. Return all stripes */
6478 io_geom->num_stripes = map->num_stripes;
6479 io_geom->max_errors = btrfs_chunk_max_errors(map);
6480
6481 /* Return the length to the full stripe end. */
6482 *length = min(logical + *length,
6483 io_geom->raid56_full_stripe_start + map->start +
6484 btrfs_stripe_nr_to_offset(data_stripes)) -
6485 logical;
6486 io_geom->stripe_index = 0;
6487 io_geom->stripe_offset = 0;
6488}
6489
6490static void map_blocks_raid56_read(struct btrfs_chunk_map *map,
6491 struct btrfs_io_geometry *io_geom)
6492{
6493 int data_stripes = nr_data_stripes(map);
6494
6495 ASSERT(io_geom->mirror_num <= 1);
6496 /* Just grab the data stripe directly. */
6497 io_geom->stripe_index = io_geom->stripe_nr % data_stripes;
6498 io_geom->stripe_nr /= data_stripes;
6499
6500 /* We distribute the parity blocks across stripes. */
6501 io_geom->stripe_index =
6502 (io_geom->stripe_nr + io_geom->stripe_index) % map->num_stripes;
6503
6504 if (io_geom->op == BTRFS_MAP_READ && io_geom->mirror_num < 1)
6505 io_geom->mirror_num = 1;
6506}
6507
6508static void map_blocks_single(const struct btrfs_chunk_map *map,
6509 struct btrfs_io_geometry *io_geom)
6510{
6511 io_geom->stripe_index = io_geom->stripe_nr % map->num_stripes;
6512 io_geom->stripe_nr /= map->num_stripes;
6513 io_geom->mirror_num = io_geom->stripe_index + 1;
6514}
6515
6516/*
6517 * Map one logical range to one or more physical ranges.
6518 *
6519 * @length: (Mandatory) mapped length of this run.
6520 * One logical range can be split into different segments
6521 * due to factors like zones and RAID0/5/6/10 stripe
6522 * boundaries.
6523 *
6524 * @bioc_ret: (Mandatory) returned btrfs_io_context structure.
6525 * which has one or more physical ranges (btrfs_io_stripe)
6526 * recorded inside.
6527 * Caller should call btrfs_put_bioc() to free it after use.
6528 *
6529 * @smap: (Optional) single physical range optimization.
6530 * If the map request can be fulfilled by one single
6531 * physical range, and this is parameter is not NULL,
6532 * then @bioc_ret would be NULL, and @smap would be
6533 * updated.
6534 *
6535 * @mirror_num_ret: (Mandatory) returned mirror number if the original
6536 * value is 0.
6537 *
6538 * Mirror number 0 means to choose any live mirrors.
6539 *
6540 * For non-RAID56 profiles, non-zero mirror_num means
6541 * the Nth mirror. (e.g. mirror_num 1 means the first
6542 * copy).
6543 *
6544 * For RAID56 profile, mirror 1 means rebuild from P and
6545 * the remaining data stripes.
6546 *
6547 * For RAID6 profile, mirror > 2 means mark another
6548 * data/P stripe error and rebuild from the remaining
6549 * stripes..
6550 */
6551int btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
6552 u64 logical, u64 *length,
6553 struct btrfs_io_context **bioc_ret,
6554 struct btrfs_io_stripe *smap, int *mirror_num_ret)
6555{
6556 struct btrfs_chunk_map *map;
6557 struct btrfs_io_geometry io_geom = { 0 };
6558 u64 map_offset;
6559 int ret = 0;
6560 int num_copies;
6561 struct btrfs_io_context *bioc = NULL;
6562 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
6563 int dev_replace_is_ongoing = 0;
6564 u16 num_alloc_stripes;
6565 u64 max_len;
6566
6567 ASSERT(bioc_ret);
6568
6569 io_geom.mirror_num = (mirror_num_ret ? *mirror_num_ret : 0);
6570 io_geom.num_stripes = 1;
6571 io_geom.stripe_index = 0;
6572 io_geom.op = op;
6573
6574 map = btrfs_get_chunk_map(fs_info, logical, *length);
6575 if (IS_ERR(map))
6576 return PTR_ERR(map);
6577
6578 num_copies = btrfs_chunk_map_num_copies(map);
6579 if (io_geom.mirror_num > num_copies)
6580 return -EINVAL;
6581
6582 map_offset = logical - map->start;
6583 io_geom.raid56_full_stripe_start = (u64)-1;
6584 max_len = btrfs_max_io_len(map, map_offset, &io_geom);
6585 *length = min_t(u64, map->chunk_len - map_offset, max_len);
6586
6587 if (dev_replace->replace_task != current)
6588 down_read(&dev_replace->rwsem);
6589
6590 dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace);
6591 /*
6592 * Hold the semaphore for read during the whole operation, write is
6593 * requested at commit time but must wait.
6594 */
6595 if (!dev_replace_is_ongoing && dev_replace->replace_task != current)
6596 up_read(&dev_replace->rwsem);
6597
6598 switch (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) {
6599 case BTRFS_BLOCK_GROUP_RAID0:
6600 map_blocks_raid0(map, &io_geom);
6601 break;
6602 case BTRFS_BLOCK_GROUP_RAID1:
6603 case BTRFS_BLOCK_GROUP_RAID1C3:
6604 case BTRFS_BLOCK_GROUP_RAID1C4:
6605 map_blocks_raid1(fs_info, map, &io_geom, dev_replace_is_ongoing);
6606 break;
6607 case BTRFS_BLOCK_GROUP_DUP:
6608 map_blocks_dup(map, &io_geom);
6609 break;
6610 case BTRFS_BLOCK_GROUP_RAID10:
6611 map_blocks_raid10(fs_info, map, &io_geom, dev_replace_is_ongoing);
6612 break;
6613 case BTRFS_BLOCK_GROUP_RAID5:
6614 case BTRFS_BLOCK_GROUP_RAID6:
6615 if (op != BTRFS_MAP_READ || io_geom.mirror_num > 1)
6616 map_blocks_raid56_write(map, &io_geom, logical, length);
6617 else
6618 map_blocks_raid56_read(map, &io_geom);
6619 break;
6620 default:
6621 /*
6622 * After this, stripe_nr is the number of stripes on this
6623 * device we have to walk to find the data, and stripe_index is
6624 * the number of our device in the stripe array
6625 */
6626 map_blocks_single(map, &io_geom);
6627 break;
6628 }
6629 if (io_geom.stripe_index >= map->num_stripes) {
6630 btrfs_crit(fs_info,
6631 "stripe index math went horribly wrong, got stripe_index=%u, num_stripes=%u",
6632 io_geom.stripe_index, map->num_stripes);
6633 ret = -EINVAL;
6634 goto out;
6635 }
6636
6637 num_alloc_stripes = io_geom.num_stripes;
6638 if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL &&
6639 op != BTRFS_MAP_READ)
6640 /*
6641 * For replace case, we need to add extra stripes for extra
6642 * duplicated stripes.
6643 *
6644 * For both WRITE and GET_READ_MIRRORS, we may have at most
6645 * 2 more stripes (DUP types, otherwise 1).
6646 */
6647 num_alloc_stripes += 2;
6648
6649 /*
6650 * If this I/O maps to a single device, try to return the device and
6651 * physical block information on the stack instead of allocating an
6652 * I/O context structure.
6653 */
6654 if (is_single_device_io(fs_info, smap, map, num_alloc_stripes, op,
6655 io_geom.mirror_num)) {
6656 ret = set_io_stripe(fs_info, logical, length, smap, map, &io_geom);
6657 if (mirror_num_ret)
6658 *mirror_num_ret = io_geom.mirror_num;
6659 *bioc_ret = NULL;
6660 goto out;
6661 }
6662
6663 bioc = alloc_btrfs_io_context(fs_info, logical, num_alloc_stripes);
6664 if (!bioc) {
6665 ret = -ENOMEM;
6666 goto out;
6667 }
6668 bioc->map_type = map->type;
6669
6670 /*
6671 * For RAID56 full map, we need to make sure the stripes[] follows the
6672 * rule that data stripes are all ordered, then followed with P and Q
6673 * (if we have).
6674 *
6675 * It's still mostly the same as other profiles, just with extra rotation.
6676 */
6677 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK &&
6678 (op != BTRFS_MAP_READ || io_geom.mirror_num > 1)) {
6679 /*
6680 * For RAID56 @stripe_nr is already the number of full stripes
6681 * before us, which is also the rotation value (needs to modulo
6682 * with num_stripes).
6683 *
6684 * In this case, we just add @stripe_nr with @i, then do the
6685 * modulo, to reduce one modulo call.
6686 */
6687 bioc->full_stripe_logical = map->start +
6688 btrfs_stripe_nr_to_offset(io_geom.stripe_nr *
6689 nr_data_stripes(map));
6690 for (int i = 0; i < io_geom.num_stripes; i++) {
6691 struct btrfs_io_stripe *dst = &bioc->stripes[i];
6692 u32 stripe_index;
6693
6694 stripe_index = (i + io_geom.stripe_nr) % io_geom.num_stripes;
6695 dst->dev = map->stripes[stripe_index].dev;
6696 dst->physical =
6697 map->stripes[stripe_index].physical +
6698 io_geom.stripe_offset +
6699 btrfs_stripe_nr_to_offset(io_geom.stripe_nr);
6700 }
6701 } else {
6702 /*
6703 * For all other non-RAID56 profiles, just copy the target
6704 * stripe into the bioc.
6705 */
6706 for (int i = 0; i < io_geom.num_stripes; i++) {
6707 ret = set_io_stripe(fs_info, logical, length,
6708 &bioc->stripes[i], map, &io_geom);
6709 if (ret < 0)
6710 break;
6711 io_geom.stripe_index++;
6712 }
6713 }
6714
6715 if (ret) {
6716 *bioc_ret = NULL;
6717 btrfs_put_bioc(bioc);
6718 goto out;
6719 }
6720
6721 if (op != BTRFS_MAP_READ)
6722 io_geom.max_errors = btrfs_chunk_max_errors(map);
6723
6724 if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL &&
6725 op != BTRFS_MAP_READ) {
6726 handle_ops_on_dev_replace(bioc, dev_replace, logical, &io_geom);
6727 }
6728
6729 *bioc_ret = bioc;
6730 bioc->num_stripes = io_geom.num_stripes;
6731 bioc->max_errors = io_geom.max_errors;
6732 bioc->mirror_num = io_geom.mirror_num;
6733
6734out:
6735 if (dev_replace_is_ongoing && dev_replace->replace_task != current) {
6736 lockdep_assert_held(&dev_replace->rwsem);
6737 /* Unlock and let waiting writers proceed */
6738 up_read(&dev_replace->rwsem);
6739 }
6740 btrfs_free_chunk_map(map);
6741 return ret;
6742}
6743
6744static bool dev_args_match_fs_devices(const struct btrfs_dev_lookup_args *args,
6745 const struct btrfs_fs_devices *fs_devices)
6746{
6747 if (args->fsid == NULL)
6748 return true;
6749 if (memcmp(fs_devices->metadata_uuid, args->fsid, BTRFS_FSID_SIZE) == 0)
6750 return true;
6751 return false;
6752}
6753
6754static bool dev_args_match_device(const struct btrfs_dev_lookup_args *args,
6755 const struct btrfs_device *device)
6756{
6757 if (args->missing) {
6758 if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state) &&
6759 !device->bdev)
6760 return true;
6761 return false;
6762 }
6763
6764 if (device->devid != args->devid)
6765 return false;
6766 if (args->uuid && memcmp(device->uuid, args->uuid, BTRFS_UUID_SIZE) != 0)
6767 return false;
6768 return true;
6769}
6770
6771/*
6772 * Find a device specified by @devid or @uuid in the list of @fs_devices, or
6773 * return NULL.
6774 *
6775 * If devid and uuid are both specified, the match must be exact, otherwise
6776 * only devid is used.
6777 */
6778struct btrfs_device *btrfs_find_device(const struct btrfs_fs_devices *fs_devices,
6779 const struct btrfs_dev_lookup_args *args)
6780{
6781 struct btrfs_device *device;
6782 struct btrfs_fs_devices *seed_devs;
6783
6784 if (dev_args_match_fs_devices(args, fs_devices)) {
6785 list_for_each_entry(device, &fs_devices->devices, dev_list) {
6786 if (dev_args_match_device(args, device))
6787 return device;
6788 }
6789 }
6790
6791 list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
6792 if (!dev_args_match_fs_devices(args, seed_devs))
6793 continue;
6794 list_for_each_entry(device, &seed_devs->devices, dev_list) {
6795 if (dev_args_match_device(args, device))
6796 return device;
6797 }
6798 }
6799
6800 return NULL;
6801}
6802
6803static struct btrfs_device *add_missing_dev(struct btrfs_fs_devices *fs_devices,
6804 u64 devid, u8 *dev_uuid)
6805{
6806 struct btrfs_device *device;
6807 unsigned int nofs_flag;
6808
6809 /*
6810 * We call this under the chunk_mutex, so we want to use NOFS for this
6811 * allocation, however we don't want to change btrfs_alloc_device() to
6812 * always do NOFS because we use it in a lot of other GFP_KERNEL safe
6813 * places.
6814 */
6815
6816 nofs_flag = memalloc_nofs_save();
6817 device = btrfs_alloc_device(NULL, &devid, dev_uuid, NULL);
6818 memalloc_nofs_restore(nofs_flag);
6819 if (IS_ERR(device))
6820 return device;
6821
6822 list_add(&device->dev_list, &fs_devices->devices);
6823 device->fs_devices = fs_devices;
6824 fs_devices->num_devices++;
6825
6826 set_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
6827 fs_devices->missing_devices++;
6828
6829 return device;
6830}
6831
6832/*
6833 * Allocate new device struct, set up devid and UUID.
6834 *
6835 * @fs_info: used only for generating a new devid, can be NULL if
6836 * devid is provided (i.e. @devid != NULL).
6837 * @devid: a pointer to devid for this device. If NULL a new devid
6838 * is generated.
6839 * @uuid: a pointer to UUID for this device. If NULL a new UUID
6840 * is generated.
6841 * @path: a pointer to device path if available, NULL otherwise.
6842 *
6843 * Return: a pointer to a new &struct btrfs_device on success; ERR_PTR()
6844 * on error. Returned struct is not linked onto any lists and must be
6845 * destroyed with btrfs_free_device.
6846 */
6847struct btrfs_device *btrfs_alloc_device(struct btrfs_fs_info *fs_info,
6848 const u64 *devid, const u8 *uuid,
6849 const char *path)
6850{
6851 struct btrfs_device *dev;
6852 u64 tmp;
6853
6854 if (WARN_ON(!devid && !fs_info))
6855 return ERR_PTR(-EINVAL);
6856
6857 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
6858 if (!dev)
6859 return ERR_PTR(-ENOMEM);
6860
6861 INIT_LIST_HEAD(&dev->dev_list);
6862 INIT_LIST_HEAD(&dev->dev_alloc_list);
6863 INIT_LIST_HEAD(&dev->post_commit_list);
6864
6865 atomic_set(&dev->dev_stats_ccnt, 0);
6866 btrfs_device_data_ordered_init(dev);
6867 extent_io_tree_init(fs_info, &dev->alloc_state, IO_TREE_DEVICE_ALLOC_STATE);
6868
6869 if (devid)
6870 tmp = *devid;
6871 else {
6872 int ret;
6873
6874 ret = find_next_devid(fs_info, &tmp);
6875 if (ret) {
6876 btrfs_free_device(dev);
6877 return ERR_PTR(ret);
6878 }
6879 }
6880 dev->devid = tmp;
6881
6882 if (uuid)
6883 memcpy(dev->uuid, uuid, BTRFS_UUID_SIZE);
6884 else
6885 generate_random_uuid(dev->uuid);
6886
6887 if (path) {
6888 struct rcu_string *name;
6889
6890 name = rcu_string_strdup(path, GFP_KERNEL);
6891 if (!name) {
6892 btrfs_free_device(dev);
6893 return ERR_PTR(-ENOMEM);
6894 }
6895 rcu_assign_pointer(dev->name, name);
6896 }
6897
6898 return dev;
6899}
6900
6901static void btrfs_report_missing_device(struct btrfs_fs_info *fs_info,
6902 u64 devid, u8 *uuid, bool error)
6903{
6904 if (error)
6905 btrfs_err_rl(fs_info, "devid %llu uuid %pU is missing",
6906 devid, uuid);
6907 else
6908 btrfs_warn_rl(fs_info, "devid %llu uuid %pU is missing",
6909 devid, uuid);
6910}
6911
6912u64 btrfs_calc_stripe_length(const struct btrfs_chunk_map *map)
6913{
6914 const int data_stripes = calc_data_stripes(map->type, map->num_stripes);
6915
6916 return div_u64(map->chunk_len, data_stripes);
6917}
6918
6919#if BITS_PER_LONG == 32
6920/*
6921 * Due to page cache limit, metadata beyond BTRFS_32BIT_MAX_FILE_SIZE
6922 * can't be accessed on 32bit systems.
6923 *
6924 * This function do mount time check to reject the fs if it already has
6925 * metadata chunk beyond that limit.
6926 */
6927static int check_32bit_meta_chunk(struct btrfs_fs_info *fs_info,
6928 u64 logical, u64 length, u64 type)
6929{
6930 if (!(type & BTRFS_BLOCK_GROUP_METADATA))
6931 return 0;
6932
6933 if (logical + length < MAX_LFS_FILESIZE)
6934 return 0;
6935
6936 btrfs_err_32bit_limit(fs_info);
6937 return -EOVERFLOW;
6938}
6939
6940/*
6941 * This is to give early warning for any metadata chunk reaching
6942 * BTRFS_32BIT_EARLY_WARN_THRESHOLD.
6943 * Although we can still access the metadata, it's not going to be possible
6944 * once the limit is reached.
6945 */
6946static void warn_32bit_meta_chunk(struct btrfs_fs_info *fs_info,
6947 u64 logical, u64 length, u64 type)
6948{
6949 if (!(type & BTRFS_BLOCK_GROUP_METADATA))
6950 return;
6951
6952 if (logical + length < BTRFS_32BIT_EARLY_WARN_THRESHOLD)
6953 return;
6954
6955 btrfs_warn_32bit_limit(fs_info);
6956}
6957#endif
6958
6959static struct btrfs_device *handle_missing_device(struct btrfs_fs_info *fs_info,
6960 u64 devid, u8 *uuid)
6961{
6962 struct btrfs_device *dev;
6963
6964 if (!btrfs_test_opt(fs_info, DEGRADED)) {
6965 btrfs_report_missing_device(fs_info, devid, uuid, true);
6966 return ERR_PTR(-ENOENT);
6967 }
6968
6969 dev = add_missing_dev(fs_info->fs_devices, devid, uuid);
6970 if (IS_ERR(dev)) {
6971 btrfs_err(fs_info, "failed to init missing device %llu: %ld",
6972 devid, PTR_ERR(dev));
6973 return dev;
6974 }
6975 btrfs_report_missing_device(fs_info, devid, uuid, false);
6976
6977 return dev;
6978}
6979
6980static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf,
6981 struct btrfs_chunk *chunk)
6982{
6983 BTRFS_DEV_LOOKUP_ARGS(args);
6984 struct btrfs_fs_info *fs_info = leaf->fs_info;
6985 struct btrfs_chunk_map *map;
6986 u64 logical;
6987 u64 length;
6988 u64 devid;
6989 u64 type;
6990 u8 uuid[BTRFS_UUID_SIZE];
6991 int index;
6992 int num_stripes;
6993 int ret;
6994 int i;
6995
6996 logical = key->offset;
6997 length = btrfs_chunk_length(leaf, chunk);
6998 type = btrfs_chunk_type(leaf, chunk);
6999 index = btrfs_bg_flags_to_raid_index(type);
7000 num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
7001
7002#if BITS_PER_LONG == 32
7003 ret = check_32bit_meta_chunk(fs_info, logical, length, type);
7004 if (ret < 0)
7005 return ret;
7006 warn_32bit_meta_chunk(fs_info, logical, length, type);
7007#endif
7008
7009 /*
7010 * Only need to verify chunk item if we're reading from sys chunk array,
7011 * as chunk item in tree block is already verified by tree-checker.
7012 */
7013 if (leaf->start == BTRFS_SUPER_INFO_OFFSET) {
7014 ret = btrfs_check_chunk_valid(leaf, chunk, logical);
7015 if (ret)
7016 return ret;
7017 }
7018
7019 map = btrfs_find_chunk_map(fs_info, logical, 1);
7020
7021 /* already mapped? */
7022 if (map && map->start <= logical && map->start + map->chunk_len > logical) {
7023 btrfs_free_chunk_map(map);
7024 return 0;
7025 } else if (map) {
7026 btrfs_free_chunk_map(map);
7027 }
7028
7029 map = btrfs_alloc_chunk_map(num_stripes, GFP_NOFS);
7030 if (!map)
7031 return -ENOMEM;
7032
7033 map->start = logical;
7034 map->chunk_len = length;
7035 map->num_stripes = num_stripes;
7036 map->io_width = btrfs_chunk_io_width(leaf, chunk);
7037 map->io_align = btrfs_chunk_io_align(leaf, chunk);
7038 map->type = type;
7039 /*
7040 * We can't use the sub_stripes value, as for profiles other than
7041 * RAID10, they may have 0 as sub_stripes for filesystems created by
7042 * older mkfs (<v5.4).
7043 * In that case, it can cause divide-by-zero errors later.
7044 * Since currently sub_stripes is fixed for each profile, let's
7045 * use the trusted value instead.
7046 */
7047 map->sub_stripes = btrfs_raid_array[index].sub_stripes;
7048 map->verified_stripes = 0;
7049 map->stripe_size = btrfs_calc_stripe_length(map);
7050 for (i = 0; i < num_stripes; i++) {
7051 map->stripes[i].physical =
7052 btrfs_stripe_offset_nr(leaf, chunk, i);
7053 devid = btrfs_stripe_devid_nr(leaf, chunk, i);
7054 args.devid = devid;
7055 read_extent_buffer(leaf, uuid, (unsigned long)
7056 btrfs_stripe_dev_uuid_nr(chunk, i),
7057 BTRFS_UUID_SIZE);
7058 args.uuid = uuid;
7059 map->stripes[i].dev = btrfs_find_device(fs_info->fs_devices, &args);
7060 if (!map->stripes[i].dev) {
7061 map->stripes[i].dev = handle_missing_device(fs_info,
7062 devid, uuid);
7063 if (IS_ERR(map->stripes[i].dev)) {
7064 ret = PTR_ERR(map->stripes[i].dev);
7065 btrfs_free_chunk_map(map);
7066 return ret;
7067 }
7068 }
7069
7070 set_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
7071 &(map->stripes[i].dev->dev_state));
7072 }
7073
7074 ret = btrfs_add_chunk_map(fs_info, map);
7075 if (ret < 0) {
7076 btrfs_err(fs_info,
7077 "failed to add chunk map, start=%llu len=%llu: %d",
7078 map->start, map->chunk_len, ret);
7079 btrfs_free_chunk_map(map);
7080 }
7081
7082 return ret;
7083}
7084
7085static void fill_device_from_item(struct extent_buffer *leaf,
7086 struct btrfs_dev_item *dev_item,
7087 struct btrfs_device *device)
7088{
7089 unsigned long ptr;
7090
7091 device->devid = btrfs_device_id(leaf, dev_item);
7092 device->disk_total_bytes = btrfs_device_total_bytes(leaf, dev_item);
7093 device->total_bytes = device->disk_total_bytes;
7094 device->commit_total_bytes = device->disk_total_bytes;
7095 device->bytes_used = btrfs_device_bytes_used(leaf, dev_item);
7096 device->commit_bytes_used = device->bytes_used;
7097 device->type = btrfs_device_type(leaf, dev_item);
7098 device->io_align = btrfs_device_io_align(leaf, dev_item);
7099 device->io_width = btrfs_device_io_width(leaf, dev_item);
7100 device->sector_size = btrfs_device_sector_size(leaf, dev_item);
7101 WARN_ON(device->devid == BTRFS_DEV_REPLACE_DEVID);
7102 clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
7103
7104 ptr = btrfs_device_uuid(dev_item);
7105 read_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
7106}
7107
7108static struct btrfs_fs_devices *open_seed_devices(struct btrfs_fs_info *fs_info,
7109 u8 *fsid)
7110{
7111 struct btrfs_fs_devices *fs_devices;
7112 int ret;
7113
7114 lockdep_assert_held(&uuid_mutex);
7115 ASSERT(fsid);
7116
7117 /* This will match only for multi-device seed fs */
7118 list_for_each_entry(fs_devices, &fs_info->fs_devices->seed_list, seed_list)
7119 if (!memcmp(fs_devices->fsid, fsid, BTRFS_FSID_SIZE))
7120 return fs_devices;
7121
7122
7123 fs_devices = find_fsid(fsid, NULL);
7124 if (!fs_devices) {
7125 if (!btrfs_test_opt(fs_info, DEGRADED))
7126 return ERR_PTR(-ENOENT);
7127
7128 fs_devices = alloc_fs_devices(fsid);
7129 if (IS_ERR(fs_devices))
7130 return fs_devices;
7131
7132 fs_devices->seeding = true;
7133 fs_devices->opened = 1;
7134 return fs_devices;
7135 }
7136
7137 /*
7138 * Upon first call for a seed fs fsid, just create a private copy of the
7139 * respective fs_devices and anchor it at fs_info->fs_devices->seed_list
7140 */
7141 fs_devices = clone_fs_devices(fs_devices);
7142 if (IS_ERR(fs_devices))
7143 return fs_devices;
7144
7145 ret = open_fs_devices(fs_devices, BLK_OPEN_READ, fs_info->bdev_holder);
7146 if (ret) {
7147 free_fs_devices(fs_devices);
7148 return ERR_PTR(ret);
7149 }
7150
7151 if (!fs_devices->seeding) {
7152 close_fs_devices(fs_devices);
7153 free_fs_devices(fs_devices);
7154 return ERR_PTR(-EINVAL);
7155 }
7156
7157 list_add(&fs_devices->seed_list, &fs_info->fs_devices->seed_list);
7158
7159 return fs_devices;
7160}
7161
7162static int read_one_dev(struct extent_buffer *leaf,
7163 struct btrfs_dev_item *dev_item)
7164{
7165 BTRFS_DEV_LOOKUP_ARGS(args);
7166 struct btrfs_fs_info *fs_info = leaf->fs_info;
7167 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7168 struct btrfs_device *device;
7169 u64 devid;
7170 int ret;
7171 u8 fs_uuid[BTRFS_FSID_SIZE];
7172 u8 dev_uuid[BTRFS_UUID_SIZE];
7173
7174 devid = btrfs_device_id(leaf, dev_item);
7175 args.devid = devid;
7176 read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
7177 BTRFS_UUID_SIZE);
7178 read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
7179 BTRFS_FSID_SIZE);
7180 args.uuid = dev_uuid;
7181 args.fsid = fs_uuid;
7182
7183 if (memcmp(fs_uuid, fs_devices->metadata_uuid, BTRFS_FSID_SIZE)) {
7184 fs_devices = open_seed_devices(fs_info, fs_uuid);
7185 if (IS_ERR(fs_devices))
7186 return PTR_ERR(fs_devices);
7187 }
7188
7189 device = btrfs_find_device(fs_info->fs_devices, &args);
7190 if (!device) {
7191 if (!btrfs_test_opt(fs_info, DEGRADED)) {
7192 btrfs_report_missing_device(fs_info, devid,
7193 dev_uuid, true);
7194 return -ENOENT;
7195 }
7196
7197 device = add_missing_dev(fs_devices, devid, dev_uuid);
7198 if (IS_ERR(device)) {
7199 btrfs_err(fs_info,
7200 "failed to add missing dev %llu: %ld",
7201 devid, PTR_ERR(device));
7202 return PTR_ERR(device);
7203 }
7204 btrfs_report_missing_device(fs_info, devid, dev_uuid, false);
7205 } else {
7206 if (!device->bdev) {
7207 if (!btrfs_test_opt(fs_info, DEGRADED)) {
7208 btrfs_report_missing_device(fs_info,
7209 devid, dev_uuid, true);
7210 return -ENOENT;
7211 }
7212 btrfs_report_missing_device(fs_info, devid,
7213 dev_uuid, false);
7214 }
7215
7216 if (!device->bdev &&
7217 !test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
7218 /*
7219 * this happens when a device that was properly setup
7220 * in the device info lists suddenly goes bad.
7221 * device->bdev is NULL, and so we have to set
7222 * device->missing to one here
7223 */
7224 device->fs_devices->missing_devices++;
7225 set_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
7226 }
7227
7228 /* Move the device to its own fs_devices */
7229 if (device->fs_devices != fs_devices) {
7230 ASSERT(test_bit(BTRFS_DEV_STATE_MISSING,
7231 &device->dev_state));
7232
7233 list_move(&device->dev_list, &fs_devices->devices);
7234 device->fs_devices->num_devices--;
7235 fs_devices->num_devices++;
7236
7237 device->fs_devices->missing_devices--;
7238 fs_devices->missing_devices++;
7239
7240 device->fs_devices = fs_devices;
7241 }
7242 }
7243
7244 if (device->fs_devices != fs_info->fs_devices) {
7245 BUG_ON(test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state));
7246 if (device->generation !=
7247 btrfs_device_generation(leaf, dev_item))
7248 return -EINVAL;
7249 }
7250
7251 fill_device_from_item(leaf, dev_item, device);
7252 if (device->bdev) {
7253 u64 max_total_bytes = bdev_nr_bytes(device->bdev);
7254
7255 if (device->total_bytes > max_total_bytes) {
7256 btrfs_err(fs_info,
7257 "device total_bytes should be at most %llu but found %llu",
7258 max_total_bytes, device->total_bytes);
7259 return -EINVAL;
7260 }
7261 }
7262 set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
7263 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
7264 !test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
7265 device->fs_devices->total_rw_bytes += device->total_bytes;
7266 atomic64_add(device->total_bytes - device->bytes_used,
7267 &fs_info->free_chunk_space);
7268 }
7269 ret = 0;
7270 return ret;
7271}
7272
7273int btrfs_read_sys_array(struct btrfs_fs_info *fs_info)
7274{
7275 struct btrfs_super_block *super_copy = fs_info->super_copy;
7276 struct extent_buffer *sb;
7277 struct btrfs_disk_key *disk_key;
7278 struct btrfs_chunk *chunk;
7279 u8 *array_ptr;
7280 unsigned long sb_array_offset;
7281 int ret = 0;
7282 u32 num_stripes;
7283 u32 array_size;
7284 u32 len = 0;
7285 u32 cur_offset;
7286 u64 type;
7287 struct btrfs_key key;
7288
7289 ASSERT(BTRFS_SUPER_INFO_SIZE <= fs_info->nodesize);
7290
7291 /*
7292 * We allocated a dummy extent, just to use extent buffer accessors.
7293 * There will be unused space after BTRFS_SUPER_INFO_SIZE, but
7294 * that's fine, we will not go beyond system chunk array anyway.
7295 */
7296 sb = alloc_dummy_extent_buffer(fs_info, BTRFS_SUPER_INFO_OFFSET);
7297 if (!sb)
7298 return -ENOMEM;
7299 set_extent_buffer_uptodate(sb);
7300
7301 write_extent_buffer(sb, super_copy, 0, BTRFS_SUPER_INFO_SIZE);
7302 array_size = btrfs_super_sys_array_size(super_copy);
7303
7304 array_ptr = super_copy->sys_chunk_array;
7305 sb_array_offset = offsetof(struct btrfs_super_block, sys_chunk_array);
7306 cur_offset = 0;
7307
7308 while (cur_offset < array_size) {
7309 disk_key = (struct btrfs_disk_key *)array_ptr;
7310 len = sizeof(*disk_key);
7311 if (cur_offset + len > array_size)
7312 goto out_short_read;
7313
7314 btrfs_disk_key_to_cpu(&key, disk_key);
7315
7316 array_ptr += len;
7317 sb_array_offset += len;
7318 cur_offset += len;
7319
7320 if (key.type != BTRFS_CHUNK_ITEM_KEY) {
7321 btrfs_err(fs_info,
7322 "unexpected item type %u in sys_array at offset %u",
7323 (u32)key.type, cur_offset);
7324 ret = -EIO;
7325 break;
7326 }
7327
7328 chunk = (struct btrfs_chunk *)sb_array_offset;
7329 /*
7330 * At least one btrfs_chunk with one stripe must be present,
7331 * exact stripe count check comes afterwards
7332 */
7333 len = btrfs_chunk_item_size(1);
7334 if (cur_offset + len > array_size)
7335 goto out_short_read;
7336
7337 num_stripes = btrfs_chunk_num_stripes(sb, chunk);
7338 if (!num_stripes) {
7339 btrfs_err(fs_info,
7340 "invalid number of stripes %u in sys_array at offset %u",
7341 num_stripes, cur_offset);
7342 ret = -EIO;
7343 break;
7344 }
7345
7346 type = btrfs_chunk_type(sb, chunk);
7347 if ((type & BTRFS_BLOCK_GROUP_SYSTEM) == 0) {
7348 btrfs_err(fs_info,
7349 "invalid chunk type %llu in sys_array at offset %u",
7350 type, cur_offset);
7351 ret = -EIO;
7352 break;
7353 }
7354
7355 len = btrfs_chunk_item_size(num_stripes);
7356 if (cur_offset + len > array_size)
7357 goto out_short_read;
7358
7359 ret = read_one_chunk(&key, sb, chunk);
7360 if (ret)
7361 break;
7362
7363 array_ptr += len;
7364 sb_array_offset += len;
7365 cur_offset += len;
7366 }
7367 clear_extent_buffer_uptodate(sb);
7368 free_extent_buffer_stale(sb);
7369 return ret;
7370
7371out_short_read:
7372 btrfs_err(fs_info, "sys_array too short to read %u bytes at offset %u",
7373 len, cur_offset);
7374 clear_extent_buffer_uptodate(sb);
7375 free_extent_buffer_stale(sb);
7376 return -EIO;
7377}
7378
7379/*
7380 * Check if all chunks in the fs are OK for read-write degraded mount
7381 *
7382 * If the @failing_dev is specified, it's accounted as missing.
7383 *
7384 * Return true if all chunks meet the minimal RW mount requirements.
7385 * Return false if any chunk doesn't meet the minimal RW mount requirements.
7386 */
7387bool btrfs_check_rw_degradable(struct btrfs_fs_info *fs_info,
7388 struct btrfs_device *failing_dev)
7389{
7390 struct btrfs_chunk_map *map;
7391 u64 next_start;
7392 bool ret = true;
7393
7394 map = btrfs_find_chunk_map(fs_info, 0, U64_MAX);
7395 /* No chunk at all? Return false anyway */
7396 if (!map) {
7397 ret = false;
7398 goto out;
7399 }
7400 while (map) {
7401 int missing = 0;
7402 int max_tolerated;
7403 int i;
7404
7405 max_tolerated =
7406 btrfs_get_num_tolerated_disk_barrier_failures(
7407 map->type);
7408 for (i = 0; i < map->num_stripes; i++) {
7409 struct btrfs_device *dev = map->stripes[i].dev;
7410
7411 if (!dev || !dev->bdev ||
7412 test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) ||
7413 dev->last_flush_error)
7414 missing++;
7415 else if (failing_dev && failing_dev == dev)
7416 missing++;
7417 }
7418 if (missing > max_tolerated) {
7419 if (!failing_dev)
7420 btrfs_warn(fs_info,
7421 "chunk %llu missing %d devices, max tolerance is %d for writable mount",
7422 map->start, missing, max_tolerated);
7423 btrfs_free_chunk_map(map);
7424 ret = false;
7425 goto out;
7426 }
7427 next_start = map->start + map->chunk_len;
7428 btrfs_free_chunk_map(map);
7429
7430 map = btrfs_find_chunk_map(fs_info, next_start, U64_MAX - next_start);
7431 }
7432out:
7433 return ret;
7434}
7435
7436static void readahead_tree_node_children(struct extent_buffer *node)
7437{
7438 int i;
7439 const int nr_items = btrfs_header_nritems(node);
7440
7441 for (i = 0; i < nr_items; i++)
7442 btrfs_readahead_node_child(node, i);
7443}
7444
7445int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info)
7446{
7447 struct btrfs_root *root = fs_info->chunk_root;
7448 struct btrfs_path *path;
7449 struct extent_buffer *leaf;
7450 struct btrfs_key key;
7451 struct btrfs_key found_key;
7452 int ret;
7453 int slot;
7454 int iter_ret = 0;
7455 u64 total_dev = 0;
7456 u64 last_ra_node = 0;
7457
7458 path = btrfs_alloc_path();
7459 if (!path)
7460 return -ENOMEM;
7461
7462 /*
7463 * uuid_mutex is needed only if we are mounting a sprout FS
7464 * otherwise we don't need it.
7465 */
7466 mutex_lock(&uuid_mutex);
7467
7468 /*
7469 * It is possible for mount and umount to race in such a way that
7470 * we execute this code path, but open_fs_devices failed to clear
7471 * total_rw_bytes. We certainly want it cleared before reading the
7472 * device items, so clear it here.
7473 */
7474 fs_info->fs_devices->total_rw_bytes = 0;
7475
7476 /*
7477 * Lockdep complains about possible circular locking dependency between
7478 * a disk's open_mutex (struct gendisk.open_mutex), the rw semaphores
7479 * used for freeze procection of a fs (struct super_block.s_writers),
7480 * which we take when starting a transaction, and extent buffers of the
7481 * chunk tree if we call read_one_dev() while holding a lock on an
7482 * extent buffer of the chunk tree. Since we are mounting the filesystem
7483 * and at this point there can't be any concurrent task modifying the
7484 * chunk tree, to keep it simple, just skip locking on the chunk tree.
7485 */
7486 ASSERT(!test_bit(BTRFS_FS_OPEN, &fs_info->flags));
7487 path->skip_locking = 1;
7488
7489 /*
7490 * Read all device items, and then all the chunk items. All
7491 * device items are found before any chunk item (their object id
7492 * is smaller than the lowest possible object id for a chunk
7493 * item - BTRFS_FIRST_CHUNK_TREE_OBJECTID).
7494 */
7495 key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
7496 key.offset = 0;
7497 key.type = 0;
7498 btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
7499 struct extent_buffer *node = path->nodes[1];
7500
7501 leaf = path->nodes[0];
7502 slot = path->slots[0];
7503
7504 if (node) {
7505 if (last_ra_node != node->start) {
7506 readahead_tree_node_children(node);
7507 last_ra_node = node->start;
7508 }
7509 }
7510 if (found_key.type == BTRFS_DEV_ITEM_KEY) {
7511 struct btrfs_dev_item *dev_item;
7512 dev_item = btrfs_item_ptr(leaf, slot,
7513 struct btrfs_dev_item);
7514 ret = read_one_dev(leaf, dev_item);
7515 if (ret)
7516 goto error;
7517 total_dev++;
7518 } else if (found_key.type == BTRFS_CHUNK_ITEM_KEY) {
7519 struct btrfs_chunk *chunk;
7520
7521 /*
7522 * We are only called at mount time, so no need to take
7523 * fs_info->chunk_mutex. Plus, to avoid lockdep warnings,
7524 * we always lock first fs_info->chunk_mutex before
7525 * acquiring any locks on the chunk tree. This is a
7526 * requirement for chunk allocation, see the comment on
7527 * top of btrfs_chunk_alloc() for details.
7528 */
7529 chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
7530 ret = read_one_chunk(&found_key, leaf, chunk);
7531 if (ret)
7532 goto error;
7533 }
7534 }
7535 /* Catch error found during iteration */
7536 if (iter_ret < 0) {
7537 ret = iter_ret;
7538 goto error;
7539 }
7540
7541 /*
7542 * After loading chunk tree, we've got all device information,
7543 * do another round of validation checks.
7544 */
7545 if (total_dev != fs_info->fs_devices->total_devices) {
7546 btrfs_warn(fs_info,
7547"super block num_devices %llu mismatch with DEV_ITEM count %llu, will be repaired on next transaction commit",
7548 btrfs_super_num_devices(fs_info->super_copy),
7549 total_dev);
7550 fs_info->fs_devices->total_devices = total_dev;
7551 btrfs_set_super_num_devices(fs_info->super_copy, total_dev);
7552 }
7553 if (btrfs_super_total_bytes(fs_info->super_copy) <
7554 fs_info->fs_devices->total_rw_bytes) {
7555 btrfs_err(fs_info,
7556 "super_total_bytes %llu mismatch with fs_devices total_rw_bytes %llu",
7557 btrfs_super_total_bytes(fs_info->super_copy),
7558 fs_info->fs_devices->total_rw_bytes);
7559 ret = -EINVAL;
7560 goto error;
7561 }
7562 ret = 0;
7563error:
7564 mutex_unlock(&uuid_mutex);
7565
7566 btrfs_free_path(path);
7567 return ret;
7568}
7569
7570int btrfs_init_devices_late(struct btrfs_fs_info *fs_info)
7571{
7572 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
7573 struct btrfs_device *device;
7574 int ret = 0;
7575
7576 fs_devices->fs_info = fs_info;
7577
7578 mutex_lock(&fs_devices->device_list_mutex);
7579 list_for_each_entry(device, &fs_devices->devices, dev_list)
7580 device->fs_info = fs_info;
7581
7582 list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
7583 list_for_each_entry(device, &seed_devs->devices, dev_list) {
7584 device->fs_info = fs_info;
7585 ret = btrfs_get_dev_zone_info(device, false);
7586 if (ret)
7587 break;
7588 }
7589
7590 seed_devs->fs_info = fs_info;
7591 }
7592 mutex_unlock(&fs_devices->device_list_mutex);
7593
7594 return ret;
7595}
7596
7597static u64 btrfs_dev_stats_value(const struct extent_buffer *eb,
7598 const struct btrfs_dev_stats_item *ptr,
7599 int index)
7600{
7601 u64 val;
7602
7603 read_extent_buffer(eb, &val,
7604 offsetof(struct btrfs_dev_stats_item, values) +
7605 ((unsigned long)ptr) + (index * sizeof(u64)),
7606 sizeof(val));
7607 return val;
7608}
7609
7610static void btrfs_set_dev_stats_value(struct extent_buffer *eb,
7611 struct btrfs_dev_stats_item *ptr,
7612 int index, u64 val)
7613{
7614 write_extent_buffer(eb, &val,
7615 offsetof(struct btrfs_dev_stats_item, values) +
7616 ((unsigned long)ptr) + (index * sizeof(u64)),
7617 sizeof(val));
7618}
7619
7620static int btrfs_device_init_dev_stats(struct btrfs_device *device,
7621 struct btrfs_path *path)
7622{
7623 struct btrfs_dev_stats_item *ptr;
7624 struct extent_buffer *eb;
7625 struct btrfs_key key;
7626 int item_size;
7627 int i, ret, slot;
7628
7629 if (!device->fs_info->dev_root)
7630 return 0;
7631
7632 key.objectid = BTRFS_DEV_STATS_OBJECTID;
7633 key.type = BTRFS_PERSISTENT_ITEM_KEY;
7634 key.offset = device->devid;
7635 ret = btrfs_search_slot(NULL, device->fs_info->dev_root, &key, path, 0, 0);
7636 if (ret) {
7637 for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7638 btrfs_dev_stat_set(device, i, 0);
7639 device->dev_stats_valid = 1;
7640 btrfs_release_path(path);
7641 return ret < 0 ? ret : 0;
7642 }
7643 slot = path->slots[0];
7644 eb = path->nodes[0];
7645 item_size = btrfs_item_size(eb, slot);
7646
7647 ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_stats_item);
7648
7649 for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
7650 if (item_size >= (1 + i) * sizeof(__le64))
7651 btrfs_dev_stat_set(device, i,
7652 btrfs_dev_stats_value(eb, ptr, i));
7653 else
7654 btrfs_dev_stat_set(device, i, 0);
7655 }
7656
7657 device->dev_stats_valid = 1;
7658 btrfs_dev_stat_print_on_load(device);
7659 btrfs_release_path(path);
7660
7661 return 0;
7662}
7663
7664int btrfs_init_dev_stats(struct btrfs_fs_info *fs_info)
7665{
7666 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
7667 struct btrfs_device *device;
7668 struct btrfs_path *path = NULL;
7669 int ret = 0;
7670
7671 path = btrfs_alloc_path();
7672 if (!path)
7673 return -ENOMEM;
7674
7675 mutex_lock(&fs_devices->device_list_mutex);
7676 list_for_each_entry(device, &fs_devices->devices, dev_list) {
7677 ret = btrfs_device_init_dev_stats(device, path);
7678 if (ret)
7679 goto out;
7680 }
7681 list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
7682 list_for_each_entry(device, &seed_devs->devices, dev_list) {
7683 ret = btrfs_device_init_dev_stats(device, path);
7684 if (ret)
7685 goto out;
7686 }
7687 }
7688out:
7689 mutex_unlock(&fs_devices->device_list_mutex);
7690
7691 btrfs_free_path(path);
7692 return ret;
7693}
7694
7695static int update_dev_stat_item(struct btrfs_trans_handle *trans,
7696 struct btrfs_device *device)
7697{
7698 struct btrfs_fs_info *fs_info = trans->fs_info;
7699 struct btrfs_root *dev_root = fs_info->dev_root;
7700 struct btrfs_path *path;
7701 struct btrfs_key key;
7702 struct extent_buffer *eb;
7703 struct btrfs_dev_stats_item *ptr;
7704 int ret;
7705 int i;
7706
7707 key.objectid = BTRFS_DEV_STATS_OBJECTID;
7708 key.type = BTRFS_PERSISTENT_ITEM_KEY;
7709 key.offset = device->devid;
7710
7711 path = btrfs_alloc_path();
7712 if (!path)
7713 return -ENOMEM;
7714 ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1);
7715 if (ret < 0) {
7716 btrfs_warn_in_rcu(fs_info,
7717 "error %d while searching for dev_stats item for device %s",
7718 ret, btrfs_dev_name(device));
7719 goto out;
7720 }
7721
7722 if (ret == 0 &&
7723 btrfs_item_size(path->nodes[0], path->slots[0]) < sizeof(*ptr)) {
7724 /* need to delete old one and insert a new one */
7725 ret = btrfs_del_item(trans, dev_root, path);
7726 if (ret != 0) {
7727 btrfs_warn_in_rcu(fs_info,
7728 "delete too small dev_stats item for device %s failed %d",
7729 btrfs_dev_name(device), ret);
7730 goto out;
7731 }
7732 ret = 1;
7733 }
7734
7735 if (ret == 1) {
7736 /* need to insert a new item */
7737 btrfs_release_path(path);
7738 ret = btrfs_insert_empty_item(trans, dev_root, path,
7739 &key, sizeof(*ptr));
7740 if (ret < 0) {
7741 btrfs_warn_in_rcu(fs_info,
7742 "insert dev_stats item for device %s failed %d",
7743 btrfs_dev_name(device), ret);
7744 goto out;
7745 }
7746 }
7747
7748 eb = path->nodes[0];
7749 ptr = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dev_stats_item);
7750 for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7751 btrfs_set_dev_stats_value(eb, ptr, i,
7752 btrfs_dev_stat_read(device, i));
7753 btrfs_mark_buffer_dirty(trans, eb);
7754
7755out:
7756 btrfs_free_path(path);
7757 return ret;
7758}
7759
7760/*
7761 * called from commit_transaction. Writes all changed device stats to disk.
7762 */
7763int btrfs_run_dev_stats(struct btrfs_trans_handle *trans)
7764{
7765 struct btrfs_fs_info *fs_info = trans->fs_info;
7766 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7767 struct btrfs_device *device;
7768 int stats_cnt;
7769 int ret = 0;
7770
7771 mutex_lock(&fs_devices->device_list_mutex);
7772 list_for_each_entry(device, &fs_devices->devices, dev_list) {
7773 stats_cnt = atomic_read(&device->dev_stats_ccnt);
7774 if (!device->dev_stats_valid || stats_cnt == 0)
7775 continue;
7776
7777
7778 /*
7779 * There is a LOAD-LOAD control dependency between the value of
7780 * dev_stats_ccnt and updating the on-disk values which requires
7781 * reading the in-memory counters. Such control dependencies
7782 * require explicit read memory barriers.
7783 *
7784 * This memory barriers pairs with smp_mb__before_atomic in
7785 * btrfs_dev_stat_inc/btrfs_dev_stat_set and with the full
7786 * barrier implied by atomic_xchg in
7787 * btrfs_dev_stats_read_and_reset
7788 */
7789 smp_rmb();
7790
7791 ret = update_dev_stat_item(trans, device);
7792 if (!ret)
7793 atomic_sub(stats_cnt, &device->dev_stats_ccnt);
7794 }
7795 mutex_unlock(&fs_devices->device_list_mutex);
7796
7797 return ret;
7798}
7799
7800void btrfs_dev_stat_inc_and_print(struct btrfs_device *dev, int index)
7801{
7802 btrfs_dev_stat_inc(dev, index);
7803
7804 if (!dev->dev_stats_valid)
7805 return;
7806 btrfs_err_rl_in_rcu(dev->fs_info,
7807 "bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u",
7808 btrfs_dev_name(dev),
7809 btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
7810 btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
7811 btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
7812 btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
7813 btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
7814}
7815
7816static void btrfs_dev_stat_print_on_load(struct btrfs_device *dev)
7817{
7818 int i;
7819
7820 for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7821 if (btrfs_dev_stat_read(dev, i) != 0)
7822 break;
7823 if (i == BTRFS_DEV_STAT_VALUES_MAX)
7824 return; /* all values == 0, suppress message */
7825
7826 btrfs_info_in_rcu(dev->fs_info,
7827 "bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u",
7828 btrfs_dev_name(dev),
7829 btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
7830 btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
7831 btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
7832 btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
7833 btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
7834}
7835
7836int btrfs_get_dev_stats(struct btrfs_fs_info *fs_info,
7837 struct btrfs_ioctl_get_dev_stats *stats)
7838{
7839 BTRFS_DEV_LOOKUP_ARGS(args);
7840 struct btrfs_device *dev;
7841 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7842 int i;
7843
7844 mutex_lock(&fs_devices->device_list_mutex);
7845 args.devid = stats->devid;
7846 dev = btrfs_find_device(fs_info->fs_devices, &args);
7847 mutex_unlock(&fs_devices->device_list_mutex);
7848
7849 if (!dev) {
7850 btrfs_warn(fs_info, "get dev_stats failed, device not found");
7851 return -ENODEV;
7852 } else if (!dev->dev_stats_valid) {
7853 btrfs_warn(fs_info, "get dev_stats failed, not yet valid");
7854 return -ENODEV;
7855 } else if (stats->flags & BTRFS_DEV_STATS_RESET) {
7856 for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
7857 if (stats->nr_items > i)
7858 stats->values[i] =
7859 btrfs_dev_stat_read_and_reset(dev, i);
7860 else
7861 btrfs_dev_stat_set(dev, i, 0);
7862 }
7863 btrfs_info(fs_info, "device stats zeroed by %s (%d)",
7864 current->comm, task_pid_nr(current));
7865 } else {
7866 for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7867 if (stats->nr_items > i)
7868 stats->values[i] = btrfs_dev_stat_read(dev, i);
7869 }
7870 if (stats->nr_items > BTRFS_DEV_STAT_VALUES_MAX)
7871 stats->nr_items = BTRFS_DEV_STAT_VALUES_MAX;
7872 return 0;
7873}
7874
7875/*
7876 * Update the size and bytes used for each device where it changed. This is
7877 * delayed since we would otherwise get errors while writing out the
7878 * superblocks.
7879 *
7880 * Must be invoked during transaction commit.
7881 */
7882void btrfs_commit_device_sizes(struct btrfs_transaction *trans)
7883{
7884 struct btrfs_device *curr, *next;
7885
7886 ASSERT(trans->state == TRANS_STATE_COMMIT_DOING);
7887
7888 if (list_empty(&trans->dev_update_list))
7889 return;
7890
7891 /*
7892 * We don't need the device_list_mutex here. This list is owned by the
7893 * transaction and the transaction must complete before the device is
7894 * released.
7895 */
7896 mutex_lock(&trans->fs_info->chunk_mutex);
7897 list_for_each_entry_safe(curr, next, &trans->dev_update_list,
7898 post_commit_list) {
7899 list_del_init(&curr->post_commit_list);
7900 curr->commit_total_bytes = curr->disk_total_bytes;
7901 curr->commit_bytes_used = curr->bytes_used;
7902 }
7903 mutex_unlock(&trans->fs_info->chunk_mutex);
7904}
7905
7906/*
7907 * Multiplicity factor for simple profiles: DUP, RAID1-like and RAID10.
7908 */
7909int btrfs_bg_type_to_factor(u64 flags)
7910{
7911 const int index = btrfs_bg_flags_to_raid_index(flags);
7912
7913 return btrfs_raid_array[index].ncopies;
7914}
7915
7916
7917
7918static int verify_one_dev_extent(struct btrfs_fs_info *fs_info,
7919 u64 chunk_offset, u64 devid,
7920 u64 physical_offset, u64 physical_len)
7921{
7922 struct btrfs_dev_lookup_args args = { .devid = devid };
7923 struct btrfs_chunk_map *map;
7924 struct btrfs_device *dev;
7925 u64 stripe_len;
7926 bool found = false;
7927 int ret = 0;
7928 int i;
7929
7930 map = btrfs_find_chunk_map(fs_info, chunk_offset, 1);
7931 if (!map) {
7932 btrfs_err(fs_info,
7933"dev extent physical offset %llu on devid %llu doesn't have corresponding chunk",
7934 physical_offset, devid);
7935 ret = -EUCLEAN;
7936 goto out;
7937 }
7938
7939 stripe_len = btrfs_calc_stripe_length(map);
7940 if (physical_len != stripe_len) {
7941 btrfs_err(fs_info,
7942"dev extent physical offset %llu on devid %llu length doesn't match chunk %llu, have %llu expect %llu",
7943 physical_offset, devid, map->start, physical_len,
7944 stripe_len);
7945 ret = -EUCLEAN;
7946 goto out;
7947 }
7948
7949 /*
7950 * Very old mkfs.btrfs (before v4.1) will not respect the reserved
7951 * space. Although kernel can handle it without problem, better to warn
7952 * the users.
7953 */
7954 if (physical_offset < BTRFS_DEVICE_RANGE_RESERVED)
7955 btrfs_warn(fs_info,
7956 "devid %llu physical %llu len %llu inside the reserved space",
7957 devid, physical_offset, physical_len);
7958
7959 for (i = 0; i < map->num_stripes; i++) {
7960 if (map->stripes[i].dev->devid == devid &&
7961 map->stripes[i].physical == physical_offset) {
7962 found = true;
7963 if (map->verified_stripes >= map->num_stripes) {
7964 btrfs_err(fs_info,
7965 "too many dev extents for chunk %llu found",
7966 map->start);
7967 ret = -EUCLEAN;
7968 goto out;
7969 }
7970 map->verified_stripes++;
7971 break;
7972 }
7973 }
7974 if (!found) {
7975 btrfs_err(fs_info,
7976 "dev extent physical offset %llu devid %llu has no corresponding chunk",
7977 physical_offset, devid);
7978 ret = -EUCLEAN;
7979 }
7980
7981 /* Make sure no dev extent is beyond device boundary */
7982 dev = btrfs_find_device(fs_info->fs_devices, &args);
7983 if (!dev) {
7984 btrfs_err(fs_info, "failed to find devid %llu", devid);
7985 ret = -EUCLEAN;
7986 goto out;
7987 }
7988
7989 if (physical_offset + physical_len > dev->disk_total_bytes) {
7990 btrfs_err(fs_info,
7991"dev extent devid %llu physical offset %llu len %llu is beyond device boundary %llu",
7992 devid, physical_offset, physical_len,
7993 dev->disk_total_bytes);
7994 ret = -EUCLEAN;
7995 goto out;
7996 }
7997
7998 if (dev->zone_info) {
7999 u64 zone_size = dev->zone_info->zone_size;
8000
8001 if (!IS_ALIGNED(physical_offset, zone_size) ||
8002 !IS_ALIGNED(physical_len, zone_size)) {
8003 btrfs_err(fs_info,
8004"zoned: dev extent devid %llu physical offset %llu len %llu is not aligned to device zone",
8005 devid, physical_offset, physical_len);
8006 ret = -EUCLEAN;
8007 goto out;
8008 }
8009 }
8010
8011out:
8012 btrfs_free_chunk_map(map);
8013 return ret;
8014}
8015
8016static int verify_chunk_dev_extent_mapping(struct btrfs_fs_info *fs_info)
8017{
8018 struct rb_node *node;
8019 int ret = 0;
8020
8021 read_lock(&fs_info->mapping_tree_lock);
8022 for (node = rb_first_cached(&fs_info->mapping_tree); node; node = rb_next(node)) {
8023 struct btrfs_chunk_map *map;
8024
8025 map = rb_entry(node, struct btrfs_chunk_map, rb_node);
8026 if (map->num_stripes != map->verified_stripes) {
8027 btrfs_err(fs_info,
8028 "chunk %llu has missing dev extent, have %d expect %d",
8029 map->start, map->verified_stripes, map->num_stripes);
8030 ret = -EUCLEAN;
8031 goto out;
8032 }
8033 }
8034out:
8035 read_unlock(&fs_info->mapping_tree_lock);
8036 return ret;
8037}
8038
8039/*
8040 * Ensure that all dev extents are mapped to correct chunk, otherwise
8041 * later chunk allocation/free would cause unexpected behavior.
8042 *
8043 * NOTE: This will iterate through the whole device tree, which should be of
8044 * the same size level as the chunk tree. This slightly increases mount time.
8045 */
8046int btrfs_verify_dev_extents(struct btrfs_fs_info *fs_info)
8047{
8048 struct btrfs_path *path;
8049 struct btrfs_root *root = fs_info->dev_root;
8050 struct btrfs_key key;
8051 u64 prev_devid = 0;
8052 u64 prev_dev_ext_end = 0;
8053 int ret = 0;
8054
8055 /*
8056 * We don't have a dev_root because we mounted with ignorebadroots and
8057 * failed to load the root, so we want to skip the verification in this
8058 * case for sure.
8059 *
8060 * However if the dev root is fine, but the tree itself is corrupted
8061 * we'd still fail to mount. This verification is only to make sure
8062 * writes can happen safely, so instead just bypass this check
8063 * completely in the case of IGNOREBADROOTS.
8064 */
8065 if (btrfs_test_opt(fs_info, IGNOREBADROOTS))
8066 return 0;
8067
8068 key.objectid = 1;
8069 key.type = BTRFS_DEV_EXTENT_KEY;
8070 key.offset = 0;
8071
8072 path = btrfs_alloc_path();
8073 if (!path)
8074 return -ENOMEM;
8075
8076 path->reada = READA_FORWARD;
8077 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
8078 if (ret < 0)
8079 goto out;
8080
8081 if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
8082 ret = btrfs_next_leaf(root, path);
8083 if (ret < 0)
8084 goto out;
8085 /* No dev extents at all? Not good */
8086 if (ret > 0) {
8087 ret = -EUCLEAN;
8088 goto out;
8089 }
8090 }
8091 while (1) {
8092 struct extent_buffer *leaf = path->nodes[0];
8093 struct btrfs_dev_extent *dext;
8094 int slot = path->slots[0];
8095 u64 chunk_offset;
8096 u64 physical_offset;
8097 u64 physical_len;
8098 u64 devid;
8099
8100 btrfs_item_key_to_cpu(leaf, &key, slot);
8101 if (key.type != BTRFS_DEV_EXTENT_KEY)
8102 break;
8103 devid = key.objectid;
8104 physical_offset = key.offset;
8105
8106 dext = btrfs_item_ptr(leaf, slot, struct btrfs_dev_extent);
8107 chunk_offset = btrfs_dev_extent_chunk_offset(leaf, dext);
8108 physical_len = btrfs_dev_extent_length(leaf, dext);
8109
8110 /* Check if this dev extent overlaps with the previous one */
8111 if (devid == prev_devid && physical_offset < prev_dev_ext_end) {
8112 btrfs_err(fs_info,
8113"dev extent devid %llu physical offset %llu overlap with previous dev extent end %llu",
8114 devid, physical_offset, prev_dev_ext_end);
8115 ret = -EUCLEAN;
8116 goto out;
8117 }
8118
8119 ret = verify_one_dev_extent(fs_info, chunk_offset, devid,
8120 physical_offset, physical_len);
8121 if (ret < 0)
8122 goto out;
8123 prev_devid = devid;
8124 prev_dev_ext_end = physical_offset + physical_len;
8125
8126 ret = btrfs_next_item(root, path);
8127 if (ret < 0)
8128 goto out;
8129 if (ret > 0) {
8130 ret = 0;
8131 break;
8132 }
8133 }
8134
8135 /* Ensure all chunks have corresponding dev extents */
8136 ret = verify_chunk_dev_extent_mapping(fs_info);
8137out:
8138 btrfs_free_path(path);
8139 return ret;
8140}
8141
8142/*
8143 * Check whether the given block group or device is pinned by any inode being
8144 * used as a swapfile.
8145 */
8146bool btrfs_pinned_by_swapfile(struct btrfs_fs_info *fs_info, void *ptr)
8147{
8148 struct btrfs_swapfile_pin *sp;
8149 struct rb_node *node;
8150
8151 spin_lock(&fs_info->swapfile_pins_lock);
8152 node = fs_info->swapfile_pins.rb_node;
8153 while (node) {
8154 sp = rb_entry(node, struct btrfs_swapfile_pin, node);
8155 if (ptr < sp->ptr)
8156 node = node->rb_left;
8157 else if (ptr > sp->ptr)
8158 node = node->rb_right;
8159 else
8160 break;
8161 }
8162 spin_unlock(&fs_info->swapfile_pins_lock);
8163 return node != NULL;
8164}
8165
8166static int relocating_repair_kthread(void *data)
8167{
8168 struct btrfs_block_group *cache = data;
8169 struct btrfs_fs_info *fs_info = cache->fs_info;
8170 u64 target;
8171 int ret = 0;
8172
8173 target = cache->start;
8174 btrfs_put_block_group(cache);
8175
8176 sb_start_write(fs_info->sb);
8177 if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE)) {
8178 btrfs_info(fs_info,
8179 "zoned: skip relocating block group %llu to repair: EBUSY",
8180 target);
8181 sb_end_write(fs_info->sb);
8182 return -EBUSY;
8183 }
8184
8185 mutex_lock(&fs_info->reclaim_bgs_lock);
8186
8187 /* Ensure block group still exists */
8188 cache = btrfs_lookup_block_group(fs_info, target);
8189 if (!cache)
8190 goto out;
8191
8192 if (!test_bit(BLOCK_GROUP_FLAG_RELOCATING_REPAIR, &cache->runtime_flags))
8193 goto out;
8194
8195 ret = btrfs_may_alloc_data_chunk(fs_info, target);
8196 if (ret < 0)
8197 goto out;
8198
8199 btrfs_info(fs_info,
8200 "zoned: relocating block group %llu to repair IO failure",
8201 target);
8202 ret = btrfs_relocate_chunk(fs_info, target);
8203
8204out:
8205 if (cache)
8206 btrfs_put_block_group(cache);
8207 mutex_unlock(&fs_info->reclaim_bgs_lock);
8208 btrfs_exclop_finish(fs_info);
8209 sb_end_write(fs_info->sb);
8210
8211 return ret;
8212}
8213
8214bool btrfs_repair_one_zone(struct btrfs_fs_info *fs_info, u64 logical)
8215{
8216 struct btrfs_block_group *cache;
8217
8218 if (!btrfs_is_zoned(fs_info))
8219 return false;
8220
8221 /* Do not attempt to repair in degraded state */
8222 if (btrfs_test_opt(fs_info, DEGRADED))
8223 return true;
8224
8225 cache = btrfs_lookup_block_group(fs_info, logical);
8226 if (!cache)
8227 return true;
8228
8229 if (test_and_set_bit(BLOCK_GROUP_FLAG_RELOCATING_REPAIR, &cache->runtime_flags)) {
8230 btrfs_put_block_group(cache);
8231 return true;
8232 }
8233
8234 kthread_run(relocating_repair_kthread, cache,
8235 "btrfs-relocating-repair");
8236
8237 return true;
8238}
8239
8240static void map_raid56_repair_block(struct btrfs_io_context *bioc,
8241 struct btrfs_io_stripe *smap,
8242 u64 logical)
8243{
8244 int data_stripes = nr_bioc_data_stripes(bioc);
8245 int i;
8246
8247 for (i = 0; i < data_stripes; i++) {
8248 u64 stripe_start = bioc->full_stripe_logical +
8249 btrfs_stripe_nr_to_offset(i);
8250
8251 if (logical >= stripe_start &&
8252 logical < stripe_start + BTRFS_STRIPE_LEN)
8253 break;
8254 }
8255 ASSERT(i < data_stripes);
8256 smap->dev = bioc->stripes[i].dev;
8257 smap->physical = bioc->stripes[i].physical +
8258 ((logical - bioc->full_stripe_logical) &
8259 BTRFS_STRIPE_LEN_MASK);
8260}
8261
8262/*
8263 * Map a repair write into a single device.
8264 *
8265 * A repair write is triggered by read time repair or scrub, which would only
8266 * update the contents of a single device.
8267 * Not update any other mirrors nor go through RMW path.
8268 *
8269 * Callers should ensure:
8270 *
8271 * - Call btrfs_bio_counter_inc_blocked() first
8272 * - The range does not cross stripe boundary
8273 * - Has a valid @mirror_num passed in.
8274 */
8275int btrfs_map_repair_block(struct btrfs_fs_info *fs_info,
8276 struct btrfs_io_stripe *smap, u64 logical,
8277 u32 length, int mirror_num)
8278{
8279 struct btrfs_io_context *bioc = NULL;
8280 u64 map_length = length;
8281 int mirror_ret = mirror_num;
8282 int ret;
8283
8284 ASSERT(mirror_num > 0);
8285
8286 ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, logical, &map_length,
8287 &bioc, smap, &mirror_ret);
8288 if (ret < 0)
8289 return ret;
8290
8291 /* The map range should not cross stripe boundary. */
8292 ASSERT(map_length >= length);
8293
8294 /* Already mapped to single stripe. */
8295 if (!bioc)
8296 goto out;
8297
8298 /* Map the RAID56 multi-stripe writes to a single one. */
8299 if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
8300 map_raid56_repair_block(bioc, smap, logical);
8301 goto out;
8302 }
8303
8304 ASSERT(mirror_num <= bioc->num_stripes);
8305 smap->dev = bioc->stripes[mirror_num - 1].dev;
8306 smap->physical = bioc->stripes[mirror_num - 1].physical;
8307out:
8308 btrfs_put_bioc(bioc);
8309 ASSERT(smap->dev);
8310 return 0;
8311}