Loading...
Note: File does not exist in v4.10.11.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Code for manipulating bucket marks for garbage collection.
4 *
5 * Copyright 2014 Datera, Inc.
6 */
7
8#include "bcachefs.h"
9#include "alloc_background.h"
10#include "backpointers.h"
11#include "bset.h"
12#include "btree_gc.h"
13#include "btree_update.h"
14#include "buckets.h"
15#include "buckets_waiting_for_journal.h"
16#include "disk_accounting.h"
17#include "ec.h"
18#include "error.h"
19#include "inode.h"
20#include "movinggc.h"
21#include "recovery.h"
22#include "reflink.h"
23#include "replicas.h"
24#include "subvolume.h"
25#include "trace.h"
26
27#include <linux/preempt.h>
28
29void bch2_dev_usage_read_fast(struct bch_dev *ca, struct bch_dev_usage *usage)
30{
31 memset(usage, 0, sizeof(*usage));
32 acc_u64s_percpu((u64 *) usage, (u64 __percpu *) ca->usage, dev_usage_u64s());
33}
34
35static u64 reserve_factor(u64 r)
36{
37 return r + (round_up(r, (1 << RESERVE_FACTOR)) >> RESERVE_FACTOR);
38}
39
40static struct bch_fs_usage_short
41__bch2_fs_usage_read_short(struct bch_fs *c)
42{
43 struct bch_fs_usage_short ret;
44 u64 data, reserved;
45
46 ret.capacity = c->capacity -
47 percpu_u64_get(&c->usage->hidden);
48
49 data = percpu_u64_get(&c->usage->data) +
50 percpu_u64_get(&c->usage->btree);
51 reserved = percpu_u64_get(&c->usage->reserved) +
52 percpu_u64_get(c->online_reserved);
53
54 ret.used = min(ret.capacity, data + reserve_factor(reserved));
55 ret.free = ret.capacity - ret.used;
56
57 ret.nr_inodes = percpu_u64_get(&c->usage->nr_inodes);
58
59 return ret;
60}
61
62struct bch_fs_usage_short
63bch2_fs_usage_read_short(struct bch_fs *c)
64{
65 struct bch_fs_usage_short ret;
66
67 percpu_down_read(&c->mark_lock);
68 ret = __bch2_fs_usage_read_short(c);
69 percpu_up_read(&c->mark_lock);
70
71 return ret;
72}
73
74void bch2_dev_usage_to_text(struct printbuf *out,
75 struct bch_dev *ca,
76 struct bch_dev_usage *usage)
77{
78 if (out->nr_tabstops < 5) {
79 printbuf_tabstops_reset(out);
80 printbuf_tabstop_push(out, 12);
81 printbuf_tabstop_push(out, 16);
82 printbuf_tabstop_push(out, 16);
83 printbuf_tabstop_push(out, 16);
84 printbuf_tabstop_push(out, 16);
85 }
86
87 prt_printf(out, "\tbuckets\rsectors\rfragmented\r\n");
88
89 for (unsigned i = 0; i < BCH_DATA_NR; i++) {
90 bch2_prt_data_type(out, i);
91 prt_printf(out, "\t%llu\r%llu\r%llu\r\n",
92 usage->d[i].buckets,
93 usage->d[i].sectors,
94 usage->d[i].fragmented);
95 }
96
97 prt_printf(out, "capacity\t%llu\r\n", ca->mi.nbuckets);
98}
99
100static int bch2_check_fix_ptr(struct btree_trans *trans,
101 struct bkey_s_c k,
102 struct extent_ptr_decoded p,
103 const union bch_extent_entry *entry,
104 bool *do_update)
105{
106 struct bch_fs *c = trans->c;
107 struct printbuf buf = PRINTBUF;
108 int ret = 0;
109
110 struct bch_dev *ca = bch2_dev_tryget(c, p.ptr.dev);
111 if (!ca) {
112 if (fsck_err_on(p.ptr.dev != BCH_SB_MEMBER_INVALID,
113 trans, ptr_to_invalid_device,
114 "pointer to missing device %u\n"
115 "while marking %s",
116 p.ptr.dev,
117 (printbuf_reset(&buf),
118 bch2_bkey_val_to_text(&buf, c, k), buf.buf)))
119 *do_update = true;
120 return 0;
121 }
122
123 struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr);
124 if (!g) {
125 if (fsck_err(trans, ptr_to_invalid_device,
126 "pointer to invalid bucket on device %u\n"
127 "while marking %s",
128 p.ptr.dev,
129 (printbuf_reset(&buf),
130 bch2_bkey_val_to_text(&buf, c, k), buf.buf)))
131 *do_update = true;
132 goto out;
133 }
134
135 enum bch_data_type data_type = bch2_bkey_ptr_data_type(k, p, entry);
136
137 if (fsck_err_on(!g->gen_valid,
138 trans, ptr_to_missing_alloc_key,
139 "bucket %u:%zu data type %s ptr gen %u missing in alloc btree\n"
140 "while marking %s",
141 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),
142 bch2_data_type_str(ptr_data_type(k.k, &p.ptr)),
143 p.ptr.gen,
144 (printbuf_reset(&buf),
145 bch2_bkey_val_to_text(&buf, c, k), buf.buf))) {
146 if (!p.ptr.cached) {
147 g->gen_valid = true;
148 g->gen = p.ptr.gen;
149 } else {
150 *do_update = true;
151 }
152 }
153
154 if (fsck_err_on(gen_cmp(p.ptr.gen, g->gen) > 0,
155 trans, ptr_gen_newer_than_bucket_gen,
156 "bucket %u:%zu data type %s ptr gen in the future: %u > %u\n"
157 "while marking %s",
158 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),
159 bch2_data_type_str(ptr_data_type(k.k, &p.ptr)),
160 p.ptr.gen, g->gen,
161 (printbuf_reset(&buf),
162 bch2_bkey_val_to_text(&buf, c, k), buf.buf))) {
163 if (!p.ptr.cached &&
164 (g->data_type != BCH_DATA_btree ||
165 data_type == BCH_DATA_btree)) {
166 g->gen_valid = true;
167 g->gen = p.ptr.gen;
168 g->data_type = 0;
169 g->stripe_sectors = 0;
170 g->dirty_sectors = 0;
171 g->cached_sectors = 0;
172 } else {
173 *do_update = true;
174 }
175 }
176
177 if (fsck_err_on(gen_cmp(g->gen, p.ptr.gen) > BUCKET_GC_GEN_MAX,
178 trans, ptr_gen_newer_than_bucket_gen,
179 "bucket %u:%zu gen %u data type %s: ptr gen %u too stale\n"
180 "while marking %s",
181 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), g->gen,
182 bch2_data_type_str(ptr_data_type(k.k, &p.ptr)),
183 p.ptr.gen,
184 (printbuf_reset(&buf),
185 bch2_bkey_val_to_text(&buf, c, k), buf.buf)))
186 *do_update = true;
187
188 if (fsck_err_on(!p.ptr.cached && gen_cmp(p.ptr.gen, g->gen) < 0,
189 trans, stale_dirty_ptr,
190 "bucket %u:%zu data type %s stale dirty ptr: %u < %u\n"
191 "while marking %s",
192 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),
193 bch2_data_type_str(ptr_data_type(k.k, &p.ptr)),
194 p.ptr.gen, g->gen,
195 (printbuf_reset(&buf),
196 bch2_bkey_val_to_text(&buf, c, k), buf.buf)))
197 *do_update = true;
198
199 if (data_type != BCH_DATA_btree && p.ptr.gen != g->gen)
200 goto out;
201
202 if (fsck_err_on(bucket_data_type_mismatch(g->data_type, data_type),
203 trans, ptr_bucket_data_type_mismatch,
204 "bucket %u:%zu gen %u different types of data in same bucket: %s, %s\n"
205 "while marking %s",
206 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), g->gen,
207 bch2_data_type_str(g->data_type),
208 bch2_data_type_str(data_type),
209 (printbuf_reset(&buf),
210 bch2_bkey_val_to_text(&buf, c, k), buf.buf))) {
211 if (data_type == BCH_DATA_btree) {
212 g->gen_valid = true;
213 g->gen = p.ptr.gen;
214 g->data_type = data_type;
215 g->stripe_sectors = 0;
216 g->dirty_sectors = 0;
217 g->cached_sectors = 0;
218 } else {
219 *do_update = true;
220 }
221 }
222
223 if (p.has_ec) {
224 struct gc_stripe *m = genradix_ptr(&c->gc_stripes, p.ec.idx);
225
226 if (fsck_err_on(!m || !m->alive,
227 trans, ptr_to_missing_stripe,
228 "pointer to nonexistent stripe %llu\n"
229 "while marking %s",
230 (u64) p.ec.idx,
231 (printbuf_reset(&buf),
232 bch2_bkey_val_to_text(&buf, c, k), buf.buf)))
233 *do_update = true;
234
235 if (fsck_err_on(m && m->alive && !bch2_ptr_matches_stripe_m(m, p),
236 trans, ptr_to_incorrect_stripe,
237 "pointer does not match stripe %llu\n"
238 "while marking %s",
239 (u64) p.ec.idx,
240 (printbuf_reset(&buf),
241 bch2_bkey_val_to_text(&buf, c, k), buf.buf)))
242 *do_update = true;
243 }
244out:
245fsck_err:
246 bch2_dev_put(ca);
247 printbuf_exit(&buf);
248 return ret;
249}
250
251int bch2_check_fix_ptrs(struct btree_trans *trans,
252 enum btree_id btree, unsigned level, struct bkey_s_c k,
253 enum btree_iter_update_trigger_flags flags)
254{
255 struct bch_fs *c = trans->c;
256 struct bkey_ptrs_c ptrs_c = bch2_bkey_ptrs_c(k);
257 const union bch_extent_entry *entry_c;
258 struct extent_ptr_decoded p = { 0 };
259 bool do_update = false;
260 struct printbuf buf = PRINTBUF;
261 int ret = 0;
262
263 percpu_down_read(&c->mark_lock);
264
265 bkey_for_each_ptr_decode(k.k, ptrs_c, p, entry_c) {
266 ret = bch2_check_fix_ptr(trans, k, p, entry_c, &do_update);
267 if (ret)
268 goto err;
269 }
270
271 if (do_update) {
272 if (flags & BTREE_TRIGGER_is_root) {
273 bch_err(c, "cannot update btree roots yet");
274 ret = -EINVAL;
275 goto err;
276 }
277
278 struct bkey_i *new = bch2_bkey_make_mut_noupdate(trans, k);
279 ret = PTR_ERR_OR_ZERO(new);
280 if (ret)
281 goto err;
282
283 rcu_read_lock();
284 bch2_bkey_drop_ptrs(bkey_i_to_s(new), ptr, !bch2_dev_exists(c, ptr->dev));
285 rcu_read_unlock();
286
287 if (level) {
288 /*
289 * We don't want to drop btree node pointers - if the
290 * btree node isn't there anymore, the read path will
291 * sort it out:
292 */
293 struct bkey_ptrs ptrs = bch2_bkey_ptrs(bkey_i_to_s(new));
294 rcu_read_lock();
295 bkey_for_each_ptr(ptrs, ptr) {
296 struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev);
297 struct bucket *g = PTR_GC_BUCKET(ca, ptr);
298
299 ptr->gen = g->gen;
300 }
301 rcu_read_unlock();
302 } else {
303 struct bkey_ptrs ptrs;
304 union bch_extent_entry *entry;
305
306 rcu_read_lock();
307restart_drop_ptrs:
308 ptrs = bch2_bkey_ptrs(bkey_i_to_s(new));
309 bkey_for_each_ptr_decode(bkey_i_to_s(new).k, ptrs, p, entry) {
310 struct bch_dev *ca = bch2_dev_rcu(c, p.ptr.dev);
311 struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr);
312 enum bch_data_type data_type = bch2_bkey_ptr_data_type(bkey_i_to_s_c(new), p, entry);
313
314 if ((p.ptr.cached &&
315 (!g->gen_valid || gen_cmp(p.ptr.gen, g->gen) > 0)) ||
316 (!p.ptr.cached &&
317 gen_cmp(p.ptr.gen, g->gen) < 0) ||
318 gen_cmp(g->gen, p.ptr.gen) > BUCKET_GC_GEN_MAX ||
319 (g->data_type &&
320 g->data_type != data_type)) {
321 bch2_bkey_drop_ptr(bkey_i_to_s(new), &entry->ptr);
322 goto restart_drop_ptrs;
323 }
324 }
325 rcu_read_unlock();
326again:
327 ptrs = bch2_bkey_ptrs(bkey_i_to_s(new));
328 bkey_extent_entry_for_each(ptrs, entry) {
329 if (extent_entry_type(entry) == BCH_EXTENT_ENTRY_stripe_ptr) {
330 struct gc_stripe *m = genradix_ptr(&c->gc_stripes,
331 entry->stripe_ptr.idx);
332 union bch_extent_entry *next_ptr;
333
334 bkey_extent_entry_for_each_from(ptrs, next_ptr, entry)
335 if (extent_entry_type(next_ptr) == BCH_EXTENT_ENTRY_ptr)
336 goto found;
337 next_ptr = NULL;
338found:
339 if (!next_ptr) {
340 bch_err(c, "aieee, found stripe ptr with no data ptr");
341 continue;
342 }
343
344 if (!m || !m->alive ||
345 !__bch2_ptr_matches_stripe(&m->ptrs[entry->stripe_ptr.block],
346 &next_ptr->ptr,
347 m->sectors)) {
348 bch2_bkey_extent_entry_drop(new, entry);
349 goto again;
350 }
351 }
352 }
353 }
354
355 if (0) {
356 printbuf_reset(&buf);
357 bch2_bkey_val_to_text(&buf, c, k);
358 bch_info(c, "updated %s", buf.buf);
359
360 printbuf_reset(&buf);
361 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(new));
362 bch_info(c, "new key %s", buf.buf);
363 }
364
365 percpu_up_read(&c->mark_lock);
366 struct btree_iter iter;
367 bch2_trans_node_iter_init(trans, &iter, btree, new->k.p, 0, level,
368 BTREE_ITER_intent|BTREE_ITER_all_snapshots);
369 ret = bch2_btree_iter_traverse(&iter) ?:
370 bch2_trans_update(trans, &iter, new,
371 BTREE_UPDATE_internal_snapshot_node|
372 BTREE_TRIGGER_norun);
373 bch2_trans_iter_exit(trans, &iter);
374 percpu_down_read(&c->mark_lock);
375
376 if (ret)
377 goto err;
378
379 if (level)
380 bch2_btree_node_update_key_early(trans, btree, level - 1, k, new);
381 }
382err:
383 percpu_up_read(&c->mark_lock);
384 printbuf_exit(&buf);
385 return ret;
386}
387
388int bch2_bucket_ref_update(struct btree_trans *trans, struct bch_dev *ca,
389 struct bkey_s_c k,
390 const struct bch_extent_ptr *ptr,
391 s64 sectors, enum bch_data_type ptr_data_type,
392 u8 b_gen, u8 bucket_data_type,
393 u32 *bucket_sectors)
394{
395 struct bch_fs *c = trans->c;
396 size_t bucket_nr = PTR_BUCKET_NR(ca, ptr);
397 struct printbuf buf = PRINTBUF;
398 bool inserting = sectors > 0;
399 int ret = 0;
400
401 BUG_ON(!sectors);
402
403 if (gen_after(ptr->gen, b_gen)) {
404 bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
405 ptr_gen_newer_than_bucket_gen,
406 "bucket %u:%zu gen %u data type %s: ptr gen %u newer than bucket gen\n"
407 "while marking %s",
408 ptr->dev, bucket_nr, b_gen,
409 bch2_data_type_str(bucket_data_type ?: ptr_data_type),
410 ptr->gen,
411 (bch2_bkey_val_to_text(&buf, c, k), buf.buf));
412 if (inserting)
413 goto err;
414 goto out;
415 }
416
417 if (gen_cmp(b_gen, ptr->gen) > BUCKET_GC_GEN_MAX) {
418 bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
419 ptr_too_stale,
420 "bucket %u:%zu gen %u data type %s: ptr gen %u too stale\n"
421 "while marking %s",
422 ptr->dev, bucket_nr, b_gen,
423 bch2_data_type_str(bucket_data_type ?: ptr_data_type),
424 ptr->gen,
425 (printbuf_reset(&buf),
426 bch2_bkey_val_to_text(&buf, c, k), buf.buf));
427 if (inserting)
428 goto err;
429 goto out;
430 }
431
432 if (b_gen != ptr->gen && ptr->cached) {
433 ret = 1;
434 goto out;
435 }
436
437 if (b_gen != ptr->gen) {
438 bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
439 stale_dirty_ptr,
440 "bucket %u:%zu gen %u (mem gen %u) data type %s: stale dirty ptr (gen %u)\n"
441 "while marking %s",
442 ptr->dev, bucket_nr, b_gen,
443 bucket_gen_get(ca, bucket_nr),
444 bch2_data_type_str(bucket_data_type ?: ptr_data_type),
445 ptr->gen,
446 (printbuf_reset(&buf),
447 bch2_bkey_val_to_text(&buf, c, k), buf.buf));
448 if (inserting)
449 goto err;
450 goto out;
451 }
452
453 if (bucket_data_type_mismatch(bucket_data_type, ptr_data_type)) {
454 bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
455 ptr_bucket_data_type_mismatch,
456 "bucket %u:%zu gen %u different types of data in same bucket: %s, %s\n"
457 "while marking %s",
458 ptr->dev, bucket_nr, b_gen,
459 bch2_data_type_str(bucket_data_type),
460 bch2_data_type_str(ptr_data_type),
461 (printbuf_reset(&buf),
462 bch2_bkey_val_to_text(&buf, c, k), buf.buf));
463 if (inserting)
464 goto err;
465 goto out;
466 }
467
468 if ((u64) *bucket_sectors + sectors > U32_MAX) {
469 bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
470 bucket_sector_count_overflow,
471 "bucket %u:%zu gen %u data type %s sector count overflow: %u + %lli > U32_MAX\n"
472 "while marking %s",
473 ptr->dev, bucket_nr, b_gen,
474 bch2_data_type_str(bucket_data_type ?: ptr_data_type),
475 *bucket_sectors, sectors,
476 (printbuf_reset(&buf),
477 bch2_bkey_val_to_text(&buf, c, k), buf.buf));
478 if (inserting)
479 goto err;
480 sectors = -*bucket_sectors;
481 }
482
483 *bucket_sectors += sectors;
484out:
485 printbuf_exit(&buf);
486 return ret;
487err:
488 bch2_dump_trans_updates(trans);
489 ret = -BCH_ERR_bucket_ref_update;
490 goto out;
491}
492
493void bch2_trans_account_disk_usage_change(struct btree_trans *trans)
494{
495 struct bch_fs *c = trans->c;
496 u64 disk_res_sectors = trans->disk_res ? trans->disk_res->sectors : 0;
497 static int warned_disk_usage = 0;
498 bool warn = false;
499
500 percpu_down_read(&c->mark_lock);
501 struct bch_fs_usage_base *src = &trans->fs_usage_delta;
502
503 s64 added = src->btree + src->data + src->reserved;
504
505 /*
506 * Not allowed to reduce sectors_available except by getting a
507 * reservation:
508 */
509 s64 should_not_have_added = added - (s64) disk_res_sectors;
510 if (unlikely(should_not_have_added > 0)) {
511 u64 old, new;
512
513 old = atomic64_read(&c->sectors_available);
514 do {
515 new = max_t(s64, 0, old - should_not_have_added);
516 } while (!atomic64_try_cmpxchg(&c->sectors_available,
517 &old, new));
518
519 added -= should_not_have_added;
520 warn = true;
521 }
522
523 if (added > 0) {
524 trans->disk_res->sectors -= added;
525 this_cpu_sub(*c->online_reserved, added);
526 }
527
528 preempt_disable();
529 struct bch_fs_usage_base *dst = this_cpu_ptr(c->usage);
530 acc_u64s((u64 *) dst, (u64 *) src, sizeof(*src) / sizeof(u64));
531 preempt_enable();
532 percpu_up_read(&c->mark_lock);
533
534 if (unlikely(warn) && !xchg(&warned_disk_usage, 1))
535 bch2_trans_inconsistent(trans,
536 "disk usage increased %lli more than %llu sectors reserved)",
537 should_not_have_added, disk_res_sectors);
538}
539
540/* KEY_TYPE_extent: */
541
542static int __mark_pointer(struct btree_trans *trans, struct bch_dev *ca,
543 struct bkey_s_c k,
544 const struct extent_ptr_decoded *p,
545 s64 sectors, enum bch_data_type ptr_data_type,
546 struct bch_alloc_v4 *a)
547{
548 u32 *dst_sectors = p->has_ec ? &a->stripe_sectors :
549 !p->ptr.cached ? &a->dirty_sectors :
550 &a->cached_sectors;
551 int ret = bch2_bucket_ref_update(trans, ca, k, &p->ptr, sectors, ptr_data_type,
552 a->gen, a->data_type, dst_sectors);
553
554 if (ret)
555 return ret;
556
557 alloc_data_type_set(a, ptr_data_type);
558 return 0;
559}
560
561static int bch2_trigger_pointer(struct btree_trans *trans,
562 enum btree_id btree_id, unsigned level,
563 struct bkey_s_c k, struct extent_ptr_decoded p,
564 const union bch_extent_entry *entry,
565 s64 *sectors,
566 enum btree_iter_update_trigger_flags flags)
567{
568 struct bch_fs *c = trans->c;
569 bool insert = !(flags & BTREE_TRIGGER_overwrite);
570 struct printbuf buf = PRINTBUF;
571 int ret = 0;
572
573 u64 abs_sectors = ptr_disk_sectors(level ? btree_sectors(c) : k.k->size, p);
574 *sectors = insert ? abs_sectors : -abs_sectors;
575
576 struct bch_dev *ca = bch2_dev_tryget(c, p.ptr.dev);
577 if (unlikely(!ca)) {
578 if (insert && p.ptr.dev != BCH_SB_MEMBER_INVALID)
579 ret = -BCH_ERR_trigger_pointer;
580 goto err;
581 }
582
583 struct bpos bucket;
584 struct bch_backpointer bp;
585 __bch2_extent_ptr_to_bp(trans->c, ca, btree_id, level, k, p, entry, &bucket, &bp, abs_sectors);
586
587 if (flags & BTREE_TRIGGER_transactional) {
588 struct bkey_i_alloc_v4 *a = bch2_trans_start_alloc_update(trans, bucket, 0);
589 ret = PTR_ERR_OR_ZERO(a) ?:
590 __mark_pointer(trans, ca, k, &p, *sectors, bp.data_type, &a->v);
591 if (ret)
592 goto err;
593
594 if (!p.ptr.cached) {
595 ret = bch2_bucket_backpointer_mod(trans, ca, bucket, bp, k, insert);
596 if (ret)
597 goto err;
598 }
599 }
600
601 if (flags & BTREE_TRIGGER_gc) {
602 percpu_down_read(&c->mark_lock);
603 struct bucket *g = gc_bucket(ca, bucket.offset);
604 if (bch2_fs_inconsistent_on(!g, c, "reference to invalid bucket on device %u\n %s",
605 p.ptr.dev,
606 (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) {
607 ret = -BCH_ERR_trigger_pointer;
608 goto err_unlock;
609 }
610
611 bucket_lock(g);
612 struct bch_alloc_v4 old = bucket_m_to_alloc(*g), new = old;
613 ret = __mark_pointer(trans, ca, k, &p, *sectors, bp.data_type, &new);
614 alloc_to_bucket(g, new);
615 bucket_unlock(g);
616err_unlock:
617 percpu_up_read(&c->mark_lock);
618
619 if (!ret)
620 ret = bch2_alloc_key_to_dev_counters(trans, ca, &old, &new, flags);
621 }
622err:
623 bch2_dev_put(ca);
624 printbuf_exit(&buf);
625 return ret;
626}
627
628static int bch2_trigger_stripe_ptr(struct btree_trans *trans,
629 struct bkey_s_c k,
630 struct extent_ptr_decoded p,
631 enum bch_data_type data_type,
632 s64 sectors,
633 enum btree_iter_update_trigger_flags flags)
634{
635 if (flags & BTREE_TRIGGER_transactional) {
636 struct btree_iter iter;
637 struct bkey_i_stripe *s = bch2_bkey_get_mut_typed(trans, &iter,
638 BTREE_ID_stripes, POS(0, p.ec.idx),
639 BTREE_ITER_with_updates, stripe);
640 int ret = PTR_ERR_OR_ZERO(s);
641 if (unlikely(ret)) {
642 bch2_trans_inconsistent_on(bch2_err_matches(ret, ENOENT), trans,
643 "pointer to nonexistent stripe %llu",
644 (u64) p.ec.idx);
645 goto err;
646 }
647
648 if (!bch2_ptr_matches_stripe(&s->v, p)) {
649 bch2_trans_inconsistent(trans,
650 "stripe pointer doesn't match stripe %llu",
651 (u64) p.ec.idx);
652 ret = -BCH_ERR_trigger_stripe_pointer;
653 goto err;
654 }
655
656 stripe_blockcount_set(&s->v, p.ec.block,
657 stripe_blockcount_get(&s->v, p.ec.block) +
658 sectors);
659
660 struct disk_accounting_pos acc = {
661 .type = BCH_DISK_ACCOUNTING_replicas,
662 };
663 bch2_bkey_to_replicas(&acc.replicas, bkey_i_to_s_c(&s->k_i));
664 acc.replicas.data_type = data_type;
665 ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, false);
666err:
667 bch2_trans_iter_exit(trans, &iter);
668 return ret;
669 }
670
671 if (flags & BTREE_TRIGGER_gc) {
672 struct bch_fs *c = trans->c;
673
674 struct gc_stripe *m = genradix_ptr_alloc(&c->gc_stripes, p.ec.idx, GFP_KERNEL);
675 if (!m) {
676 bch_err(c, "error allocating memory for gc_stripes, idx %llu",
677 (u64) p.ec.idx);
678 return -BCH_ERR_ENOMEM_mark_stripe_ptr;
679 }
680
681 mutex_lock(&c->ec_stripes_heap_lock);
682
683 if (!m || !m->alive) {
684 mutex_unlock(&c->ec_stripes_heap_lock);
685 struct printbuf buf = PRINTBUF;
686 bch2_bkey_val_to_text(&buf, c, k);
687 bch_err_ratelimited(c, "pointer to nonexistent stripe %llu\n while marking %s",
688 (u64) p.ec.idx, buf.buf);
689 printbuf_exit(&buf);
690 bch2_inconsistent_error(c);
691 return -BCH_ERR_trigger_stripe_pointer;
692 }
693
694 m->block_sectors[p.ec.block] += sectors;
695
696 struct disk_accounting_pos acc = {
697 .type = BCH_DISK_ACCOUNTING_replicas,
698 };
699 memcpy(&acc.replicas, &m->r.e, replicas_entry_bytes(&m->r.e));
700 mutex_unlock(&c->ec_stripes_heap_lock);
701
702 acc.replicas.data_type = data_type;
703 int ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, true);
704 if (ret)
705 return ret;
706 }
707
708 return 0;
709}
710
711static int __trigger_extent(struct btree_trans *trans,
712 enum btree_id btree_id, unsigned level,
713 struct bkey_s_c k,
714 enum btree_iter_update_trigger_flags flags,
715 s64 *replicas_sectors)
716{
717 bool gc = flags & BTREE_TRIGGER_gc;
718 struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
719 const union bch_extent_entry *entry;
720 struct extent_ptr_decoded p;
721 enum bch_data_type data_type = bkey_is_btree_ptr(k.k)
722 ? BCH_DATA_btree
723 : BCH_DATA_user;
724 int ret = 0;
725
726 struct disk_accounting_pos acc_replicas_key = {
727 .type = BCH_DISK_ACCOUNTING_replicas,
728 .replicas.data_type = data_type,
729 .replicas.nr_devs = 0,
730 .replicas.nr_required = 1,
731 };
732
733 struct disk_accounting_pos acct_compression_key = {
734 .type = BCH_DISK_ACCOUNTING_compression,
735 };
736 u64 compression_acct[3] = { 1, 0, 0 };
737
738 bkey_for_each_ptr_decode(k.k, ptrs, p, entry) {
739 s64 disk_sectors = 0;
740 ret = bch2_trigger_pointer(trans, btree_id, level, k, p, entry, &disk_sectors, flags);
741 if (ret < 0)
742 return ret;
743
744 bool stale = ret > 0;
745
746 if (p.ptr.cached && stale)
747 continue;
748
749 if (p.ptr.cached) {
750 ret = bch2_mod_dev_cached_sectors(trans, p.ptr.dev, disk_sectors, gc);
751 if (ret)
752 return ret;
753 } else if (!p.has_ec) {
754 *replicas_sectors += disk_sectors;
755 replicas_entry_add_dev(&acc_replicas_key.replicas, p.ptr.dev);
756 } else {
757 ret = bch2_trigger_stripe_ptr(trans, k, p, data_type, disk_sectors, flags);
758 if (ret)
759 return ret;
760
761 /*
762 * There may be other dirty pointers in this extent, but
763 * if so they're not required for mounting if we have an
764 * erasure coded pointer in this extent:
765 */
766 acc_replicas_key.replicas.nr_required = 0;
767 }
768
769 if (acct_compression_key.compression.type &&
770 acct_compression_key.compression.type != p.crc.compression_type) {
771 if (flags & BTREE_TRIGGER_overwrite)
772 bch2_u64s_neg(compression_acct, ARRAY_SIZE(compression_acct));
773
774 ret = bch2_disk_accounting_mod(trans, &acct_compression_key, compression_acct,
775 ARRAY_SIZE(compression_acct), gc);
776 if (ret)
777 return ret;
778
779 compression_acct[0] = 1;
780 compression_acct[1] = 0;
781 compression_acct[2] = 0;
782 }
783
784 acct_compression_key.compression.type = p.crc.compression_type;
785 if (p.crc.compression_type) {
786 compression_acct[1] += p.crc.uncompressed_size;
787 compression_acct[2] += p.crc.compressed_size;
788 }
789 }
790
791 if (acc_replicas_key.replicas.nr_devs) {
792 ret = bch2_disk_accounting_mod(trans, &acc_replicas_key, replicas_sectors, 1, gc);
793 if (ret)
794 return ret;
795 }
796
797 if (acc_replicas_key.replicas.nr_devs && !level && k.k->p.snapshot) {
798 struct disk_accounting_pos acc_snapshot_key = {
799 .type = BCH_DISK_ACCOUNTING_snapshot,
800 .snapshot.id = k.k->p.snapshot,
801 };
802 ret = bch2_disk_accounting_mod(trans, &acc_snapshot_key, replicas_sectors, 1, gc);
803 if (ret)
804 return ret;
805 }
806
807 if (acct_compression_key.compression.type) {
808 if (flags & BTREE_TRIGGER_overwrite)
809 bch2_u64s_neg(compression_acct, ARRAY_SIZE(compression_acct));
810
811 ret = bch2_disk_accounting_mod(trans, &acct_compression_key, compression_acct,
812 ARRAY_SIZE(compression_acct), gc);
813 if (ret)
814 return ret;
815 }
816
817 if (level) {
818 struct disk_accounting_pos acc_btree_key = {
819 .type = BCH_DISK_ACCOUNTING_btree,
820 .btree.id = btree_id,
821 };
822 ret = bch2_disk_accounting_mod(trans, &acc_btree_key, replicas_sectors, 1, gc);
823 if (ret)
824 return ret;
825 } else {
826 bool insert = !(flags & BTREE_TRIGGER_overwrite);
827 struct disk_accounting_pos acc_inum_key = {
828 .type = BCH_DISK_ACCOUNTING_inum,
829 .inum.inum = k.k->p.inode,
830 };
831 s64 v[3] = {
832 insert ? 1 : -1,
833 insert ? k.k->size : -((s64) k.k->size),
834 *replicas_sectors,
835 };
836 ret = bch2_disk_accounting_mod(trans, &acc_inum_key, v, ARRAY_SIZE(v), gc);
837 if (ret)
838 return ret;
839 }
840
841 return 0;
842}
843
844int bch2_trigger_extent(struct btree_trans *trans,
845 enum btree_id btree, unsigned level,
846 struct bkey_s_c old, struct bkey_s new,
847 enum btree_iter_update_trigger_flags flags)
848{
849 struct bch_fs *c = trans->c;
850 struct bkey_ptrs_c new_ptrs = bch2_bkey_ptrs_c(new.s_c);
851 struct bkey_ptrs_c old_ptrs = bch2_bkey_ptrs_c(old);
852 unsigned new_ptrs_bytes = (void *) new_ptrs.end - (void *) new_ptrs.start;
853 unsigned old_ptrs_bytes = (void *) old_ptrs.end - (void *) old_ptrs.start;
854
855 if (unlikely(flags & BTREE_TRIGGER_check_repair))
856 return bch2_check_fix_ptrs(trans, btree, level, new.s_c, flags);
857
858 /* if pointers aren't changing - nothing to do: */
859 if (new_ptrs_bytes == old_ptrs_bytes &&
860 !memcmp(new_ptrs.start,
861 old_ptrs.start,
862 new_ptrs_bytes))
863 return 0;
864
865 if (flags & (BTREE_TRIGGER_transactional|BTREE_TRIGGER_gc)) {
866 s64 old_replicas_sectors = 0, new_replicas_sectors = 0;
867
868 if (old.k->type) {
869 int ret = __trigger_extent(trans, btree, level, old,
870 flags & ~BTREE_TRIGGER_insert,
871 &old_replicas_sectors);
872 if (ret)
873 return ret;
874 }
875
876 if (new.k->type) {
877 int ret = __trigger_extent(trans, btree, level, new.s_c,
878 flags & ~BTREE_TRIGGER_overwrite,
879 &new_replicas_sectors);
880 if (ret)
881 return ret;
882 }
883
884 int need_rebalance_delta = 0;
885 s64 need_rebalance_sectors_delta = 0;
886
887 s64 s = bch2_bkey_sectors_need_rebalance(c, old);
888 need_rebalance_delta -= s != 0;
889 need_rebalance_sectors_delta -= s;
890
891 s = bch2_bkey_sectors_need_rebalance(c, new.s_c);
892 need_rebalance_delta += s != 0;
893 need_rebalance_sectors_delta += s;
894
895 if ((flags & BTREE_TRIGGER_transactional) && need_rebalance_delta) {
896 int ret = bch2_btree_bit_mod_buffered(trans, BTREE_ID_rebalance_work,
897 new.k->p, need_rebalance_delta > 0);
898 if (ret)
899 return ret;
900 }
901
902 if (need_rebalance_sectors_delta) {
903 struct disk_accounting_pos acc = {
904 .type = BCH_DISK_ACCOUNTING_rebalance_work,
905 };
906 int ret = bch2_disk_accounting_mod(trans, &acc, &need_rebalance_sectors_delta, 1,
907 flags & BTREE_TRIGGER_gc);
908 if (ret)
909 return ret;
910 }
911 }
912
913 return 0;
914}
915
916/* KEY_TYPE_reservation */
917
918static int __trigger_reservation(struct btree_trans *trans,
919 enum btree_id btree_id, unsigned level, struct bkey_s_c k,
920 enum btree_iter_update_trigger_flags flags)
921{
922 if (flags & (BTREE_TRIGGER_transactional|BTREE_TRIGGER_gc)) {
923 s64 sectors = k.k->size;
924
925 if (flags & BTREE_TRIGGER_overwrite)
926 sectors = -sectors;
927
928 struct disk_accounting_pos acc = {
929 .type = BCH_DISK_ACCOUNTING_persistent_reserved,
930 .persistent_reserved.nr_replicas = bkey_s_c_to_reservation(k).v->nr_replicas,
931 };
932
933 return bch2_disk_accounting_mod(trans, &acc, §ors, 1, flags & BTREE_TRIGGER_gc);
934 }
935
936 return 0;
937}
938
939int bch2_trigger_reservation(struct btree_trans *trans,
940 enum btree_id btree_id, unsigned level,
941 struct bkey_s_c old, struct bkey_s new,
942 enum btree_iter_update_trigger_flags flags)
943{
944 return trigger_run_overwrite_then_insert(__trigger_reservation, trans, btree_id, level, old, new, flags);
945}
946
947/* Mark superblocks: */
948
949static int __bch2_trans_mark_metadata_bucket(struct btree_trans *trans,
950 struct bch_dev *ca, u64 b,
951 enum bch_data_type type,
952 unsigned sectors)
953{
954 struct btree_iter iter;
955 int ret = 0;
956
957 struct bkey_i_alloc_v4 *a =
958 bch2_trans_start_alloc_update_noupdate(trans, &iter, POS(ca->dev_idx, b));
959 if (IS_ERR(a))
960 return PTR_ERR(a);
961
962 if (a->v.data_type && type && a->v.data_type != type) {
963 bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
964 bucket_metadata_type_mismatch,
965 "bucket %llu:%llu gen %u different types of data in same bucket: %s, %s\n"
966 "while marking %s",
967 iter.pos.inode, iter.pos.offset, a->v.gen,
968 bch2_data_type_str(a->v.data_type),
969 bch2_data_type_str(type),
970 bch2_data_type_str(type));
971 ret = -BCH_ERR_metadata_bucket_inconsistency;
972 goto err;
973 }
974
975 if (a->v.data_type != type ||
976 a->v.dirty_sectors != sectors) {
977 a->v.data_type = type;
978 a->v.dirty_sectors = sectors;
979 ret = bch2_trans_update(trans, &iter, &a->k_i, 0);
980 }
981err:
982 bch2_trans_iter_exit(trans, &iter);
983 return ret;
984}
985
986static int bch2_mark_metadata_bucket(struct btree_trans *trans, struct bch_dev *ca,
987 u64 b, enum bch_data_type data_type, unsigned sectors,
988 enum btree_iter_update_trigger_flags flags)
989{
990 struct bch_fs *c = trans->c;
991 int ret = 0;
992
993 percpu_down_read(&c->mark_lock);
994 struct bucket *g = gc_bucket(ca, b);
995 if (bch2_fs_inconsistent_on(!g, c, "reference to invalid bucket on device %u when marking metadata type %s",
996 ca->dev_idx, bch2_data_type_str(data_type)))
997 goto err_unlock;
998
999 bucket_lock(g);
1000 struct bch_alloc_v4 old = bucket_m_to_alloc(*g);
1001
1002 if (bch2_fs_inconsistent_on(g->data_type &&
1003 g->data_type != data_type, c,
1004 "different types of data in same bucket: %s, %s",
1005 bch2_data_type_str(g->data_type),
1006 bch2_data_type_str(data_type)))
1007 goto err;
1008
1009 if (bch2_fs_inconsistent_on((u64) g->dirty_sectors + sectors > ca->mi.bucket_size, c,
1010 "bucket %u:%llu gen %u data type %s sector count overflow: %u + %u > bucket size",
1011 ca->dev_idx, b, g->gen,
1012 bch2_data_type_str(g->data_type ?: data_type),
1013 g->dirty_sectors, sectors))
1014 goto err;
1015
1016 g->data_type = data_type;
1017 g->dirty_sectors += sectors;
1018 struct bch_alloc_v4 new = bucket_m_to_alloc(*g);
1019 bucket_unlock(g);
1020 percpu_up_read(&c->mark_lock);
1021 ret = bch2_alloc_key_to_dev_counters(trans, ca, &old, &new, flags);
1022 return ret;
1023err:
1024 bucket_unlock(g);
1025err_unlock:
1026 percpu_up_read(&c->mark_lock);
1027 return -BCH_ERR_metadata_bucket_inconsistency;
1028}
1029
1030int bch2_trans_mark_metadata_bucket(struct btree_trans *trans,
1031 struct bch_dev *ca, u64 b,
1032 enum bch_data_type type, unsigned sectors,
1033 enum btree_iter_update_trigger_flags flags)
1034{
1035 BUG_ON(type != BCH_DATA_free &&
1036 type != BCH_DATA_sb &&
1037 type != BCH_DATA_journal);
1038
1039 /*
1040 * Backup superblock might be past the end of our normal usable space:
1041 */
1042 if (b >= ca->mi.nbuckets)
1043 return 0;
1044
1045 if (flags & BTREE_TRIGGER_gc)
1046 return bch2_mark_metadata_bucket(trans, ca, b, type, sectors, flags);
1047 else if (flags & BTREE_TRIGGER_transactional)
1048 return commit_do(trans, NULL, NULL, 0,
1049 __bch2_trans_mark_metadata_bucket(trans, ca, b, type, sectors));
1050 else
1051 BUG();
1052}
1053
1054static int bch2_trans_mark_metadata_sectors(struct btree_trans *trans,
1055 struct bch_dev *ca, u64 start, u64 end,
1056 enum bch_data_type type, u64 *bucket, unsigned *bucket_sectors,
1057 enum btree_iter_update_trigger_flags flags)
1058{
1059 do {
1060 u64 b = sector_to_bucket(ca, start);
1061 unsigned sectors =
1062 min_t(u64, bucket_to_sector(ca, b + 1), end) - start;
1063
1064 if (b != *bucket && *bucket_sectors) {
1065 int ret = bch2_trans_mark_metadata_bucket(trans, ca, *bucket,
1066 type, *bucket_sectors, flags);
1067 if (ret)
1068 return ret;
1069
1070 *bucket_sectors = 0;
1071 }
1072
1073 *bucket = b;
1074 *bucket_sectors += sectors;
1075 start += sectors;
1076 } while (start < end);
1077
1078 return 0;
1079}
1080
1081static int __bch2_trans_mark_dev_sb(struct btree_trans *trans, struct bch_dev *ca,
1082 enum btree_iter_update_trigger_flags flags)
1083{
1084 struct bch_fs *c = trans->c;
1085
1086 mutex_lock(&c->sb_lock);
1087 struct bch_sb_layout layout = ca->disk_sb.sb->layout;
1088 mutex_unlock(&c->sb_lock);
1089
1090 u64 bucket = 0;
1091 unsigned i, bucket_sectors = 0;
1092 int ret;
1093
1094 for (i = 0; i < layout.nr_superblocks; i++) {
1095 u64 offset = le64_to_cpu(layout.sb_offset[i]);
1096
1097 if (offset == BCH_SB_SECTOR) {
1098 ret = bch2_trans_mark_metadata_sectors(trans, ca,
1099 0, BCH_SB_SECTOR,
1100 BCH_DATA_sb, &bucket, &bucket_sectors, flags);
1101 if (ret)
1102 return ret;
1103 }
1104
1105 ret = bch2_trans_mark_metadata_sectors(trans, ca, offset,
1106 offset + (1 << layout.sb_max_size_bits),
1107 BCH_DATA_sb, &bucket, &bucket_sectors, flags);
1108 if (ret)
1109 return ret;
1110 }
1111
1112 if (bucket_sectors) {
1113 ret = bch2_trans_mark_metadata_bucket(trans, ca,
1114 bucket, BCH_DATA_sb, bucket_sectors, flags);
1115 if (ret)
1116 return ret;
1117 }
1118
1119 for (i = 0; i < ca->journal.nr; i++) {
1120 ret = bch2_trans_mark_metadata_bucket(trans, ca,
1121 ca->journal.buckets[i],
1122 BCH_DATA_journal, ca->mi.bucket_size, flags);
1123 if (ret)
1124 return ret;
1125 }
1126
1127 return 0;
1128}
1129
1130int bch2_trans_mark_dev_sb(struct bch_fs *c, struct bch_dev *ca,
1131 enum btree_iter_update_trigger_flags flags)
1132{
1133 int ret = bch2_trans_run(c,
1134 __bch2_trans_mark_dev_sb(trans, ca, flags));
1135 bch_err_fn(c, ret);
1136 return ret;
1137}
1138
1139int bch2_trans_mark_dev_sbs_flags(struct bch_fs *c,
1140 enum btree_iter_update_trigger_flags flags)
1141{
1142 for_each_online_member(c, ca) {
1143 int ret = bch2_trans_mark_dev_sb(c, ca, flags);
1144 if (ret) {
1145 percpu_ref_put(&ca->io_ref);
1146 return ret;
1147 }
1148 }
1149
1150 return 0;
1151}
1152
1153int bch2_trans_mark_dev_sbs(struct bch_fs *c)
1154{
1155 return bch2_trans_mark_dev_sbs_flags(c, BTREE_TRIGGER_transactional);
1156}
1157
1158/* Disk reservations: */
1159
1160#define SECTORS_CACHE 1024
1161
1162int __bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res,
1163 u64 sectors, enum bch_reservation_flags flags)
1164{
1165 struct bch_fs_pcpu *pcpu;
1166 u64 old, get;
1167 u64 sectors_available;
1168 int ret;
1169
1170 percpu_down_read(&c->mark_lock);
1171 preempt_disable();
1172 pcpu = this_cpu_ptr(c->pcpu);
1173
1174 if (sectors <= pcpu->sectors_available)
1175 goto out;
1176
1177 old = atomic64_read(&c->sectors_available);
1178 do {
1179 get = min((u64) sectors + SECTORS_CACHE, old);
1180
1181 if (get < sectors) {
1182 preempt_enable();
1183 goto recalculate;
1184 }
1185 } while (!atomic64_try_cmpxchg(&c->sectors_available,
1186 &old, old - get));
1187
1188 pcpu->sectors_available += get;
1189
1190out:
1191 pcpu->sectors_available -= sectors;
1192 this_cpu_add(*c->online_reserved, sectors);
1193 res->sectors += sectors;
1194
1195 preempt_enable();
1196 percpu_up_read(&c->mark_lock);
1197 return 0;
1198
1199recalculate:
1200 mutex_lock(&c->sectors_available_lock);
1201
1202 percpu_u64_set(&c->pcpu->sectors_available, 0);
1203 sectors_available = avail_factor(__bch2_fs_usage_read_short(c).free);
1204
1205 if (sectors_available && (flags & BCH_DISK_RESERVATION_PARTIAL))
1206 sectors = min(sectors, sectors_available);
1207
1208 if (sectors <= sectors_available ||
1209 (flags & BCH_DISK_RESERVATION_NOFAIL)) {
1210 atomic64_set(&c->sectors_available,
1211 max_t(s64, 0, sectors_available - sectors));
1212 this_cpu_add(*c->online_reserved, sectors);
1213 res->sectors += sectors;
1214 ret = 0;
1215 } else {
1216 atomic64_set(&c->sectors_available, sectors_available);
1217 ret = -BCH_ERR_ENOSPC_disk_reservation;
1218 }
1219
1220 mutex_unlock(&c->sectors_available_lock);
1221 percpu_up_read(&c->mark_lock);
1222
1223 return ret;
1224}
1225
1226/* Startup/shutdown: */
1227
1228void bch2_buckets_nouse_free(struct bch_fs *c)
1229{
1230 for_each_member_device(c, ca) {
1231 kvfree_rcu_mightsleep(ca->buckets_nouse);
1232 ca->buckets_nouse = NULL;
1233 }
1234}
1235
1236int bch2_buckets_nouse_alloc(struct bch_fs *c)
1237{
1238 for_each_member_device(c, ca) {
1239 BUG_ON(ca->buckets_nouse);
1240
1241 ca->buckets_nouse = kvmalloc(BITS_TO_LONGS(ca->mi.nbuckets) *
1242 sizeof(unsigned long),
1243 GFP_KERNEL|__GFP_ZERO);
1244 if (!ca->buckets_nouse) {
1245 bch2_dev_put(ca);
1246 return -BCH_ERR_ENOMEM_buckets_nouse;
1247 }
1248 }
1249
1250 return 0;
1251}
1252
1253static void bucket_gens_free_rcu(struct rcu_head *rcu)
1254{
1255 struct bucket_gens *buckets =
1256 container_of(rcu, struct bucket_gens, rcu);
1257
1258 kvfree(buckets);
1259}
1260
1261int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
1262{
1263 struct bucket_gens *bucket_gens = NULL, *old_bucket_gens = NULL;
1264 bool resize = ca->bucket_gens != NULL;
1265 int ret;
1266
1267 BUG_ON(resize && ca->buckets_nouse);
1268
1269 if (!(bucket_gens = kvmalloc(sizeof(struct bucket_gens) + nbuckets,
1270 GFP_KERNEL|__GFP_ZERO))) {
1271 ret = -BCH_ERR_ENOMEM_bucket_gens;
1272 goto err;
1273 }
1274
1275 bucket_gens->first_bucket = ca->mi.first_bucket;
1276 bucket_gens->nbuckets = nbuckets;
1277 bucket_gens->nbuckets_minus_first =
1278 bucket_gens->nbuckets - bucket_gens->first_bucket;
1279
1280 if (resize) {
1281 down_write(&ca->bucket_lock);
1282 percpu_down_write(&c->mark_lock);
1283 }
1284
1285 old_bucket_gens = rcu_dereference_protected(ca->bucket_gens, 1);
1286
1287 if (resize) {
1288 size_t n = min(bucket_gens->nbuckets, old_bucket_gens->nbuckets);
1289
1290 memcpy(bucket_gens->b,
1291 old_bucket_gens->b,
1292 n);
1293 }
1294
1295 rcu_assign_pointer(ca->bucket_gens, bucket_gens);
1296 bucket_gens = old_bucket_gens;
1297
1298 nbuckets = ca->mi.nbuckets;
1299
1300 if (resize) {
1301 percpu_up_write(&c->mark_lock);
1302 up_write(&ca->bucket_lock);
1303 }
1304
1305 ret = 0;
1306err:
1307 if (bucket_gens)
1308 call_rcu(&bucket_gens->rcu, bucket_gens_free_rcu);
1309
1310 return ret;
1311}
1312
1313void bch2_dev_buckets_free(struct bch_dev *ca)
1314{
1315 kvfree(ca->buckets_nouse);
1316 kvfree(rcu_dereference_protected(ca->bucket_gens, 1));
1317 free_percpu(ca->usage);
1318}
1319
1320int bch2_dev_buckets_alloc(struct bch_fs *c, struct bch_dev *ca)
1321{
1322 ca->usage = alloc_percpu(struct bch_dev_usage);
1323 if (!ca->usage)
1324 return -BCH_ERR_ENOMEM_usage_init;
1325
1326 return bch2_dev_buckets_resize(c, ca, ca->mi.nbuckets);
1327}