1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) STRATO AG 2011.  All rights reserved.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
   4 */
   5
   6/*
   7 * This module can be used to catch cases when the btrfs kernel
   8 * code executes write requests to the disk that bring the file
   9 * system in an inconsistent state. In such a state, a power-loss
  10 * or kernel panic event would cause that the data on disk is
  11 * lost or at least damaged.
  12 *
  13 * Code is added that examines all block write requests during
  14 * runtime (including writes of the super block). Three rules
  15 * are verified and an error is printed on violation of the
  16 * rules:
  17 * 1. It is not allowed to write a disk block which is
  18 *    currently referenced by the super block (either directly
  19 *    or indirectly).
  20 * 2. When a super block is written, it is verified that all
  21 *    referenced (directly or indirectly) blocks fulfill the
  22 *    following requirements:
  23 *    2a. All referenced blocks have either been present when
  24 *        the file system was mounted, (i.e., they have been
  25 *        referenced by the super block) or they have been
  26 *        written since then and the write completion callback
  27 *        was called and no write error was indicated and a
  28 *        FLUSH request to the device where these blocks are
  29 *        located was received and completed.
  30 *    2b. All referenced blocks need to have a generation
  31 *        number which is equal to the parent's number.
  32 *
  33 * One issue that was found using this module was that the log
  34 * tree on disk became temporarily corrupted because disk blocks
  35 * that had been in use for the log tree had been freed and
  36 * reused too early, while being referenced by the written super
  37 * block.
  38 *
  39 * The search term in the kernel log that can be used to filter
  40 * on the existence of detected integrity issues is
  41 * "btrfs: attempt".
  42 *
  43 * The integrity check is enabled via mount options. These
  44 * mount options are only supported if the integrity check
  45 * tool is compiled by defining BTRFS_FS_CHECK_INTEGRITY.
  46 *
  47 * Example #1, apply integrity checks to all metadata:
  48 * mount /dev/sdb1 /mnt -o check_int
  49 *
  50 * Example #2, apply integrity checks to all metadata and
  51 * to data extents:
  52 * mount /dev/sdb1 /mnt -o check_int_data
  53 *
  54 * Example #3, apply integrity checks to all metadata and dump
  55 * the tree that the super block references to kernel messages
  56 * each time after a super block was written:
  57 * mount /dev/sdb1 /mnt -o check_int,check_int_print_mask=263
  58 *
  59 * If the integrity check tool is included and activated in
  60 * the mount options, plenty of kernel memory is used, and
  61 * plenty of additional CPU cycles are spent. Enabling this
  62 * functionality is not intended for normal use. In most
  63 * cases, unless you are a btrfs developer who needs to verify
  64 * the integrity of (super)-block write requests, do not
  65 * enable the config option BTRFS_FS_CHECK_INTEGRITY to
  66 * include and compile the integrity check tool.
  67 *
  68 * Expect millions of lines of information in the kernel log with an
  69 * enabled check_int_print_mask. Therefore set LOG_BUF_SHIFT in the
  70 * kernel config to at least 26 (which is 64MB). Usually the value is
  71 * limited to 21 (which is 2MB) in init/Kconfig. The file needs to be
  72 * changed like this before LOG_BUF_SHIFT can be set to a high value:
  73 * config LOG_BUF_SHIFT
  74 *       int "Kernel log buffer size (16 => 64KB, 17 => 128KB)"
  75 *       range 12 30
  76 */
  77
  78#include <linux/sched.h>
  79#include <linux/slab.h>
  80#include <linux/buffer_head.h>
  81#include <linux/mutex.h>
 
  82#include <linux/genhd.h>
  83#include <linux/blkdev.h>
  84#include <linux/mm.h>
  85#include <linux/string.h>
  86#include <crypto/hash.h>
  87#include "ctree.h"
  88#include "disk-io.h"
  89#include "transaction.h"
  90#include "extent_io.h"
  91#include "volumes.h"
  92#include "print-tree.h"
  93#include "locking.h"
  94#include "check-integrity.h"
  95#include "rcu-string.h"
  96#include "compression.h"
  97
  98#define BTRFSIC_BLOCK_HASHTABLE_SIZE 0x10000
  99#define BTRFSIC_BLOCK_LINK_HASHTABLE_SIZE 0x10000
 100#define BTRFSIC_DEV2STATE_HASHTABLE_SIZE 0x100
 101#define BTRFSIC_BLOCK_MAGIC_NUMBER 0x14491051
 102#define BTRFSIC_BLOCK_LINK_MAGIC_NUMBER 0x11070807
 103#define BTRFSIC_DEV2STATE_MAGIC_NUMBER 0x20111530
 104#define BTRFSIC_BLOCK_STACK_FRAME_MAGIC_NUMBER 20111300
 105#define BTRFSIC_TREE_DUMP_MAX_INDENT_LEVEL (200 - 6)	/* in characters,
 106							 * excluding " [...]" */
 107#define BTRFSIC_GENERATION_UNKNOWN ((u64)-1)
 108
 109/*
 110 * The definition of the bitmask fields for the print_mask.
 111 * They are specified with the mount option check_integrity_print_mask.
 112 */
 113#define BTRFSIC_PRINT_MASK_SUPERBLOCK_WRITE			0x00000001
 114#define BTRFSIC_PRINT_MASK_ROOT_CHUNK_LOG_TREE_LOCATION		0x00000002
 115#define BTRFSIC_PRINT_MASK_TREE_AFTER_SB_WRITE			0x00000004
 116#define BTRFSIC_PRINT_MASK_TREE_BEFORE_SB_WRITE			0x00000008
 117#define BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH			0x00000010
 118#define BTRFSIC_PRINT_MASK_END_IO_BIO_BH			0x00000020
 119#define BTRFSIC_PRINT_MASK_VERBOSE				0x00000040
 120#define BTRFSIC_PRINT_MASK_VERY_VERBOSE				0x00000080
 121#define BTRFSIC_PRINT_MASK_INITIAL_TREE				0x00000100
 122#define BTRFSIC_PRINT_MASK_INITIAL_ALL_TREES			0x00000200
 123#define BTRFSIC_PRINT_MASK_INITIAL_DATABASE			0x00000400
 124#define BTRFSIC_PRINT_MASK_NUM_COPIES				0x00000800
 125#define BTRFSIC_PRINT_MASK_TREE_WITH_ALL_MIRRORS		0x00001000
 126#define BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH_VERBOSE		0x00002000
 127
 128struct btrfsic_dev_state;
 129struct btrfsic_state;
 130
 131struct btrfsic_block {
 132	u32 magic_num;		/* only used for debug purposes */
 133	unsigned int is_metadata:1;	/* if it is meta-data, not data-data */
 134	unsigned int is_superblock:1;	/* if it is one of the superblocks */
 135	unsigned int is_iodone:1;	/* if is done by lower subsystem */
 136	unsigned int iodone_w_error:1;	/* error was indicated to endio */
 137	unsigned int never_written:1;	/* block was added because it was
 138					 * referenced, not because it was
 139					 * written */
 140	unsigned int mirror_num;	/* large enough to hold
 141					 * BTRFS_SUPER_MIRROR_MAX */
 142	struct btrfsic_dev_state *dev_state;
 143	u64 dev_bytenr;		/* key, physical byte num on disk */
 144	u64 logical_bytenr;	/* logical byte num on disk */
 145	u64 generation;
 146	struct btrfs_disk_key disk_key;	/* extra info to print in case of
 147					 * issues, will not always be correct */
 148	struct list_head collision_resolving_node;	/* list node */
 149	struct list_head all_blocks_node;	/* list node */
 150
 151	/* the following two lists contain block_link items */
 152	struct list_head ref_to_list;	/* list */
 153	struct list_head ref_from_list;	/* list */
 154	struct btrfsic_block *next_in_same_bio;
 155	void *orig_bio_bh_private;
 156	union {
 157		bio_end_io_t *bio;
 158		bh_end_io_t *bh;
 159	} orig_bio_bh_end_io;
 160	int submit_bio_bh_rw;
 161	u64 flush_gen; /* only valid if !never_written */
 162};
 163
 164/*
 165 * Elements of this type are allocated dynamically and required because
 166 * each block object can refer to and can be ref from multiple blocks.
 167 * The key to lookup them in the hashtable is the dev_bytenr of
 168 * the block ref to plus the one from the block referred from.
 169 * The fact that they are searchable via a hashtable and that a
 170 * ref_cnt is maintained is not required for the btrfs integrity
 171 * check algorithm itself, it is only used to make the output more
 172 * beautiful in case that an error is detected (an error is defined
 173 * as a write operation to a block while that block is still referenced).
 174 */
 175struct btrfsic_block_link {
 176	u32 magic_num;		/* only used for debug purposes */
 177	u32 ref_cnt;
 178	struct list_head node_ref_to;	/* list node */
 179	struct list_head node_ref_from;	/* list node */
 180	struct list_head collision_resolving_node;	/* list node */
 181	struct btrfsic_block *block_ref_to;
 182	struct btrfsic_block *block_ref_from;
 183	u64 parent_generation;
 184};
 185
 186struct btrfsic_dev_state {
 187	u32 magic_num;		/* only used for debug purposes */
 188	struct block_device *bdev;
 189	struct btrfsic_state *state;
 190	struct list_head collision_resolving_node;	/* list node */
 191	struct btrfsic_block dummy_block_for_bio_bh_flush;
 192	u64 last_flush_gen;
 193	char name[BDEVNAME_SIZE];
 194};
 195
 196struct btrfsic_block_hashtable {
 197	struct list_head table[BTRFSIC_BLOCK_HASHTABLE_SIZE];
 198};
 199
 200struct btrfsic_block_link_hashtable {
 201	struct list_head table[BTRFSIC_BLOCK_LINK_HASHTABLE_SIZE];
 202};
 203
 204struct btrfsic_dev_state_hashtable {
 205	struct list_head table[BTRFSIC_DEV2STATE_HASHTABLE_SIZE];
 206};
 207
 208struct btrfsic_block_data_ctx {
 209	u64 start;		/* virtual bytenr */
 210	u64 dev_bytenr;		/* physical bytenr on device */
 211	u32 len;
 212	struct btrfsic_dev_state *dev;
 213	char **datav;
 214	struct page **pagev;
 215	void *mem_to_free;
 216};
 217
 218/* This structure is used to implement recursion without occupying
 219 * any stack space, refer to btrfsic_process_metablock() */
 220struct btrfsic_stack_frame {
 221	u32 magic;
 222	u32 nr;
 223	int error;
 224	int i;
 225	int limit_nesting;
 226	int num_copies;
 227	int mirror_num;
 228	struct btrfsic_block *block;
 229	struct btrfsic_block_data_ctx *block_ctx;
 230	struct btrfsic_block *next_block;
 231	struct btrfsic_block_data_ctx next_block_ctx;
 232	struct btrfs_header *hdr;
 233	struct btrfsic_stack_frame *prev;
 234};
 235
 236/* Some state per mounted filesystem */
 237struct btrfsic_state {
 238	u32 print_mask;
 239	int include_extent_data;
 240	int csum_size;
 241	struct list_head all_blocks_list;
 242	struct btrfsic_block_hashtable block_hashtable;
 243	struct btrfsic_block_link_hashtable block_link_hashtable;
 244	struct btrfs_fs_info *fs_info;
 245	u64 max_superblock_generation;
 246	struct btrfsic_block *latest_superblock;
 247	u32 metablock_size;
 248	u32 datablock_size;
 249};
 250
 251static void btrfsic_block_init(struct btrfsic_block *b);
 252static struct btrfsic_block *btrfsic_block_alloc(void);
 253static void btrfsic_block_free(struct btrfsic_block *b);
 254static void btrfsic_block_link_init(struct btrfsic_block_link *n);
 255static struct btrfsic_block_link *btrfsic_block_link_alloc(void);
 256static void btrfsic_block_link_free(struct btrfsic_block_link *n);
 257static void btrfsic_dev_state_init(struct btrfsic_dev_state *ds);
 258static struct btrfsic_dev_state *btrfsic_dev_state_alloc(void);
 259static void btrfsic_dev_state_free(struct btrfsic_dev_state *ds);
 260static void btrfsic_block_hashtable_init(struct btrfsic_block_hashtable *h);
 261static void btrfsic_block_hashtable_add(struct btrfsic_block *b,
 262					struct btrfsic_block_hashtable *h);
 263static void btrfsic_block_hashtable_remove(struct btrfsic_block *b);
 264static struct btrfsic_block *btrfsic_block_hashtable_lookup(
 265		struct block_device *bdev,
 266		u64 dev_bytenr,
 267		struct btrfsic_block_hashtable *h);
 268static void btrfsic_block_link_hashtable_init(
 269		struct btrfsic_block_link_hashtable *h);
 270static void btrfsic_block_link_hashtable_add(
 271		struct btrfsic_block_link *l,
 272		struct btrfsic_block_link_hashtable *h);
 273static void btrfsic_block_link_hashtable_remove(struct btrfsic_block_link *l);
 274static struct btrfsic_block_link *btrfsic_block_link_hashtable_lookup(
 275		struct block_device *bdev_ref_to,
 276		u64 dev_bytenr_ref_to,
 277		struct block_device *bdev_ref_from,
 278		u64 dev_bytenr_ref_from,
 279		struct btrfsic_block_link_hashtable *h);
 280static void btrfsic_dev_state_hashtable_init(
 281		struct btrfsic_dev_state_hashtable *h);
 282static void btrfsic_dev_state_hashtable_add(
 283		struct btrfsic_dev_state *ds,
 284		struct btrfsic_dev_state_hashtable *h);
 285static void btrfsic_dev_state_hashtable_remove(struct btrfsic_dev_state *ds);
 286static struct btrfsic_dev_state *btrfsic_dev_state_hashtable_lookup(dev_t dev,
 
 287		struct btrfsic_dev_state_hashtable *h);
 288static struct btrfsic_stack_frame *btrfsic_stack_frame_alloc(void);
 289static void btrfsic_stack_frame_free(struct btrfsic_stack_frame *sf);
 290static int btrfsic_process_superblock(struct btrfsic_state *state,
 291				      struct btrfs_fs_devices *fs_devices);
 292static int btrfsic_process_metablock(struct btrfsic_state *state,
 293				     struct btrfsic_block *block,
 294				     struct btrfsic_block_data_ctx *block_ctx,
 295				     int limit_nesting, int force_iodone_flag);
 296static void btrfsic_read_from_block_data(
 297	struct btrfsic_block_data_ctx *block_ctx,
 298	void *dst, u32 offset, size_t len);
 299static int btrfsic_create_link_to_next_block(
 300		struct btrfsic_state *state,
 301		struct btrfsic_block *block,
 302		struct btrfsic_block_data_ctx
 303		*block_ctx, u64 next_bytenr,
 304		int limit_nesting,
 305		struct btrfsic_block_data_ctx *next_block_ctx,
 306		struct btrfsic_block **next_blockp,
 307		int force_iodone_flag,
 308		int *num_copiesp, int *mirror_nump,
 309		struct btrfs_disk_key *disk_key,
 310		u64 parent_generation);
 311static int btrfsic_handle_extent_data(struct btrfsic_state *state,
 312				      struct btrfsic_block *block,
 313				      struct btrfsic_block_data_ctx *block_ctx,
 314				      u32 item_offset, int force_iodone_flag);
 315static int btrfsic_map_block(struct btrfsic_state *state, u64 bytenr, u32 len,
 316			     struct btrfsic_block_data_ctx *block_ctx_out,
 317			     int mirror_num);
 
 
 
 318static void btrfsic_release_block_ctx(struct btrfsic_block_data_ctx *block_ctx);
 319static int btrfsic_read_block(struct btrfsic_state *state,
 320			      struct btrfsic_block_data_ctx *block_ctx);
 321static void btrfsic_dump_database(struct btrfsic_state *state);
 
 322static int btrfsic_test_for_metadata(struct btrfsic_state *state,
 323				     char **datav, unsigned int num_pages);
 324static void btrfsic_process_written_block(struct btrfsic_dev_state *dev_state,
 325					  u64 dev_bytenr, char **mapped_datav,
 326					  unsigned int num_pages,
 327					  struct bio *bio, int *bio_is_patched,
 328					  struct buffer_head *bh,
 329					  int submit_bio_bh_rw);
 330static int btrfsic_process_written_superblock(
 331		struct btrfsic_state *state,
 332		struct btrfsic_block *const block,
 333		struct btrfs_super_block *const super_hdr);
 334static void btrfsic_bio_end_io(struct bio *bp);
 335static void btrfsic_bh_end_io(struct buffer_head *bh, int uptodate);
 336static int btrfsic_is_block_ref_by_superblock(const struct btrfsic_state *state,
 337					      const struct btrfsic_block *block,
 338					      int recursion_level);
 339static int btrfsic_check_all_ref_blocks(struct btrfsic_state *state,
 340					struct btrfsic_block *const block,
 341					int recursion_level);
 342static void btrfsic_print_add_link(const struct btrfsic_state *state,
 343				   const struct btrfsic_block_link *l);
 344static void btrfsic_print_rem_link(const struct btrfsic_state *state,
 345				   const struct btrfsic_block_link *l);
 346static char btrfsic_get_block_type(const struct btrfsic_state *state,
 347				   const struct btrfsic_block *block);
 348static void btrfsic_dump_tree(const struct btrfsic_state *state);
 349static void btrfsic_dump_tree_sub(const struct btrfsic_state *state,
 350				  const struct btrfsic_block *block,
 351				  int indent_level);
 352static struct btrfsic_block_link *btrfsic_block_link_lookup_or_add(
 353		struct btrfsic_state *state,
 354		struct btrfsic_block_data_ctx *next_block_ctx,
 355		struct btrfsic_block *next_block,
 356		struct btrfsic_block *from_block,
 357		u64 parent_generation);
 358static struct btrfsic_block *btrfsic_block_lookup_or_add(
 359		struct btrfsic_state *state,
 360		struct btrfsic_block_data_ctx *block_ctx,
 361		const char *additional_string,
 362		int is_metadata,
 363		int is_iodone,
 364		int never_written,
 365		int mirror_num,
 366		int *was_created);
 367static int btrfsic_process_superblock_dev_mirror(
 368		struct btrfsic_state *state,
 369		struct btrfsic_dev_state *dev_state,
 370		struct btrfs_device *device,
 371		int superblock_mirror_num,
 372		struct btrfsic_dev_state **selected_dev_state,
 373		struct btrfs_super_block *selected_super);
 374static struct btrfsic_dev_state *btrfsic_dev_state_lookup(dev_t dev);
 
 375static void btrfsic_cmp_log_and_dev_bytenr(struct btrfsic_state *state,
 376					   u64 bytenr,
 377					   struct btrfsic_dev_state *dev_state,
 378					   u64 dev_bytenr);
 379
 380static struct mutex btrfsic_mutex;
 381static int btrfsic_is_initialized;
 382static struct btrfsic_dev_state_hashtable btrfsic_dev_state_hashtable;
 383
 384
 385static void btrfsic_block_init(struct btrfsic_block *b)
 386{
 387	b->magic_num = BTRFSIC_BLOCK_MAGIC_NUMBER;
 388	b->dev_state = NULL;
 389	b->dev_bytenr = 0;
 390	b->logical_bytenr = 0;
 391	b->generation = BTRFSIC_GENERATION_UNKNOWN;
 392	b->disk_key.objectid = 0;
 393	b->disk_key.type = 0;
 394	b->disk_key.offset = 0;
 395	b->is_metadata = 0;
 396	b->is_superblock = 0;
 397	b->is_iodone = 0;
 398	b->iodone_w_error = 0;
 399	b->never_written = 0;
 400	b->mirror_num = 0;
 401	b->next_in_same_bio = NULL;
 402	b->orig_bio_bh_private = NULL;
 403	b->orig_bio_bh_end_io.bio = NULL;
 404	INIT_LIST_HEAD(&b->collision_resolving_node);
 405	INIT_LIST_HEAD(&b->all_blocks_node);
 406	INIT_LIST_HEAD(&b->ref_to_list);
 407	INIT_LIST_HEAD(&b->ref_from_list);
 408	b->submit_bio_bh_rw = 0;
 409	b->flush_gen = 0;
 410}
 411
 412static struct btrfsic_block *btrfsic_block_alloc(void)
 413{
 414	struct btrfsic_block *b;
 415
 416	b = kzalloc(sizeof(*b), GFP_NOFS);
 417	if (NULL != b)
 418		btrfsic_block_init(b);
 419
 420	return b;
 421}
 422
 423static void btrfsic_block_free(struct btrfsic_block *b)
 424{
 425	BUG_ON(!(NULL == b || BTRFSIC_BLOCK_MAGIC_NUMBER == b->magic_num));
 426	kfree(b);
 427}
 428
 429static void btrfsic_block_link_init(struct btrfsic_block_link *l)
 430{
 431	l->magic_num = BTRFSIC_BLOCK_LINK_MAGIC_NUMBER;
 432	l->ref_cnt = 1;
 433	INIT_LIST_HEAD(&l->node_ref_to);
 434	INIT_LIST_HEAD(&l->node_ref_from);
 435	INIT_LIST_HEAD(&l->collision_resolving_node);
 436	l->block_ref_to = NULL;
 437	l->block_ref_from = NULL;
 438}
 439
 440static struct btrfsic_block_link *btrfsic_block_link_alloc(void)
 441{
 442	struct btrfsic_block_link *l;
 443
 444	l = kzalloc(sizeof(*l), GFP_NOFS);
 445	if (NULL != l)
 446		btrfsic_block_link_init(l);
 447
 448	return l;
 449}
 450
 451static void btrfsic_block_link_free(struct btrfsic_block_link *l)
 452{
 453	BUG_ON(!(NULL == l || BTRFSIC_BLOCK_LINK_MAGIC_NUMBER == l->magic_num));
 454	kfree(l);
 455}
 456
 457static void btrfsic_dev_state_init(struct btrfsic_dev_state *ds)
 458{
 459	ds->magic_num = BTRFSIC_DEV2STATE_MAGIC_NUMBER;
 460	ds->bdev = NULL;
 461	ds->state = NULL;
 462	ds->name[0] = '\0';
 463	INIT_LIST_HEAD(&ds->collision_resolving_node);
 464	ds->last_flush_gen = 0;
 465	btrfsic_block_init(&ds->dummy_block_for_bio_bh_flush);
 466	ds->dummy_block_for_bio_bh_flush.is_iodone = 1;
 467	ds->dummy_block_for_bio_bh_flush.dev_state = ds;
 468}
 469
 470static struct btrfsic_dev_state *btrfsic_dev_state_alloc(void)
 471{
 472	struct btrfsic_dev_state *ds;
 473
 474	ds = kzalloc(sizeof(*ds), GFP_NOFS);
 475	if (NULL != ds)
 476		btrfsic_dev_state_init(ds);
 477
 478	return ds;
 479}
 480
 481static void btrfsic_dev_state_free(struct btrfsic_dev_state *ds)
 482{
 483	BUG_ON(!(NULL == ds ||
 484		 BTRFSIC_DEV2STATE_MAGIC_NUMBER == ds->magic_num));
 485	kfree(ds);
 486}
 487
 488static void btrfsic_block_hashtable_init(struct btrfsic_block_hashtable *h)
 489{
 490	int i;
 491
 492	for (i = 0; i < BTRFSIC_BLOCK_HASHTABLE_SIZE; i++)
 493		INIT_LIST_HEAD(h->table + i);
 494}
 495
 496static void btrfsic_block_hashtable_add(struct btrfsic_block *b,
 497					struct btrfsic_block_hashtable *h)
 498{
 499	const unsigned int hashval =
 500	    (((unsigned int)(b->dev_bytenr >> 16)) ^
 501	     ((unsigned int)((uintptr_t)b->dev_state->bdev))) &
 502	     (BTRFSIC_BLOCK_HASHTABLE_SIZE - 1);
 503
 504	list_add(&b->collision_resolving_node, h->table + hashval);
 505}
 506
 507static void btrfsic_block_hashtable_remove(struct btrfsic_block *b)
 508{
 509	list_del(&b->collision_resolving_node);
 510}
 511
 512static struct btrfsic_block *btrfsic_block_hashtable_lookup(
 513		struct block_device *bdev,
 514		u64 dev_bytenr,
 515		struct btrfsic_block_hashtable *h)
 516{
 517	const unsigned int hashval =
 518	    (((unsigned int)(dev_bytenr >> 16)) ^
 519	     ((unsigned int)((uintptr_t)bdev))) &
 520	     (BTRFSIC_BLOCK_HASHTABLE_SIZE - 1);
 521	struct btrfsic_block *b;
 
 
 
 
 
 522
 523	list_for_each_entry(b, h->table + hashval, collision_resolving_node) {
 524		if (b->dev_state->bdev == bdev && b->dev_bytenr == dev_bytenr)
 525			return b;
 526	}
 527
 528	return NULL;
 529}
 530
 531static void btrfsic_block_link_hashtable_init(
 532		struct btrfsic_block_link_hashtable *h)
 533{
 534	int i;
 535
 536	for (i = 0; i < BTRFSIC_BLOCK_LINK_HASHTABLE_SIZE; i++)
 537		INIT_LIST_HEAD(h->table + i);
 538}
 539
 540static void btrfsic_block_link_hashtable_add(
 541		struct btrfsic_block_link *l,
 542		struct btrfsic_block_link_hashtable *h)
 543{
 544	const unsigned int hashval =
 545	    (((unsigned int)(l->block_ref_to->dev_bytenr >> 16)) ^
 546	     ((unsigned int)(l->block_ref_from->dev_bytenr >> 16)) ^
 547	     ((unsigned int)((uintptr_t)l->block_ref_to->dev_state->bdev)) ^
 548	     ((unsigned int)((uintptr_t)l->block_ref_from->dev_state->bdev)))
 549	     & (BTRFSIC_BLOCK_LINK_HASHTABLE_SIZE - 1);
 550
 551	BUG_ON(NULL == l->block_ref_to);
 552	BUG_ON(NULL == l->block_ref_from);
 553	list_add(&l->collision_resolving_node, h->table + hashval);
 554}
 555
 556static void btrfsic_block_link_hashtable_remove(struct btrfsic_block_link *l)
 557{
 558	list_del(&l->collision_resolving_node);
 559}
 560
 561static struct btrfsic_block_link *btrfsic_block_link_hashtable_lookup(
 562		struct block_device *bdev_ref_to,
 563		u64 dev_bytenr_ref_to,
 564		struct block_device *bdev_ref_from,
 565		u64 dev_bytenr_ref_from,
 566		struct btrfsic_block_link_hashtable *h)
 567{
 568	const unsigned int hashval =
 569	    (((unsigned int)(dev_bytenr_ref_to >> 16)) ^
 570	     ((unsigned int)(dev_bytenr_ref_from >> 16)) ^
 571	     ((unsigned int)((uintptr_t)bdev_ref_to)) ^
 572	     ((unsigned int)((uintptr_t)bdev_ref_from))) &
 573	     (BTRFSIC_BLOCK_LINK_HASHTABLE_SIZE - 1);
 574	struct btrfsic_block_link *l;
 
 
 
 
 
 575
 576	list_for_each_entry(l, h->table + hashval, collision_resolving_node) {
 577		BUG_ON(NULL == l->block_ref_to);
 578		BUG_ON(NULL == l->block_ref_from);
 579		if (l->block_ref_to->dev_state->bdev == bdev_ref_to &&
 580		    l->block_ref_to->dev_bytenr == dev_bytenr_ref_to &&
 581		    l->block_ref_from->dev_state->bdev == bdev_ref_from &&
 582		    l->block_ref_from->dev_bytenr == dev_bytenr_ref_from)
 583			return l;
 584	}
 585
 586	return NULL;
 587}
 588
 589static void btrfsic_dev_state_hashtable_init(
 590		struct btrfsic_dev_state_hashtable *h)
 591{
 592	int i;
 593
 594	for (i = 0; i < BTRFSIC_DEV2STATE_HASHTABLE_SIZE; i++)
 595		INIT_LIST_HEAD(h->table + i);
 596}
 597
 598static void btrfsic_dev_state_hashtable_add(
 599		struct btrfsic_dev_state *ds,
 600		struct btrfsic_dev_state_hashtable *h)
 601{
 602	const unsigned int hashval =
 603	    (((unsigned int)((uintptr_t)ds->bdev->bd_dev)) &
 604	     (BTRFSIC_DEV2STATE_HASHTABLE_SIZE - 1));
 605
 606	list_add(&ds->collision_resolving_node, h->table + hashval);
 607}
 608
 609static void btrfsic_dev_state_hashtable_remove(struct btrfsic_dev_state *ds)
 610{
 611	list_del(&ds->collision_resolving_node);
 612}
 613
 614static struct btrfsic_dev_state *btrfsic_dev_state_hashtable_lookup(dev_t dev,
 
 615		struct btrfsic_dev_state_hashtable *h)
 616{
 617	const unsigned int hashval =
 618		dev & (BTRFSIC_DEV2STATE_HASHTABLE_SIZE - 1);
 619	struct btrfsic_dev_state *ds;
 
 620
 621	list_for_each_entry(ds, h->table + hashval, collision_resolving_node) {
 622		if (ds->bdev->bd_dev == dev)
 
 
 
 
 623			return ds;
 624	}
 625
 626	return NULL;
 627}
 628
 629static int btrfsic_process_superblock(struct btrfsic_state *state,
 630				      struct btrfs_fs_devices *fs_devices)
 631{
 632	struct btrfs_fs_info *fs_info = state->fs_info;
 633	struct btrfs_super_block *selected_super;
 634	struct list_head *dev_head = &fs_devices->devices;
 635	struct btrfs_device *device;
 636	struct btrfsic_dev_state *selected_dev_state = NULL;
 637	int ret = 0;
 638	int pass;
 639
 640	BUG_ON(NULL == state);
 641	selected_super = kzalloc(sizeof(*selected_super), GFP_NOFS);
 642	if (NULL == selected_super) {
 643		pr_info("btrfsic: error, kmalloc failed!\n");
 644		return -ENOMEM;
 645	}
 646
 647	list_for_each_entry(device, dev_head, dev_list) {
 648		int i;
 649		struct btrfsic_dev_state *dev_state;
 650
 651		if (!device->bdev || !device->name)
 652			continue;
 653
 654		dev_state = btrfsic_dev_state_lookup(device->bdev->bd_dev);
 655		BUG_ON(NULL == dev_state);
 656		for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
 657			ret = btrfsic_process_superblock_dev_mirror(
 658					state, dev_state, device, i,
 659					&selected_dev_state, selected_super);
 660			if (0 != ret && 0 == i) {
 661				kfree(selected_super);
 662				return ret;
 663			}
 664		}
 665	}
 666
 667	if (NULL == state->latest_superblock) {
 668		pr_info("btrfsic: no superblock found!\n");
 669		kfree(selected_super);
 670		return -1;
 671	}
 672
 673	state->csum_size = btrfs_super_csum_size(selected_super);
 674
 675	for (pass = 0; pass < 3; pass++) {
 676		int num_copies;
 677		int mirror_num;
 678		u64 next_bytenr;
 679
 680		switch (pass) {
 681		case 0:
 682			next_bytenr = btrfs_super_root(selected_super);
 683			if (state->print_mask &
 684			    BTRFSIC_PRINT_MASK_ROOT_CHUNK_LOG_TREE_LOCATION)
 685				pr_info("root@%llu\n", next_bytenr);
 
 686			break;
 687		case 1:
 688			next_bytenr = btrfs_super_chunk_root(selected_super);
 689			if (state->print_mask &
 690			    BTRFSIC_PRINT_MASK_ROOT_CHUNK_LOG_TREE_LOCATION)
 691				pr_info("chunk@%llu\n", next_bytenr);
 
 692			break;
 693		case 2:
 694			next_bytenr = btrfs_super_log_root(selected_super);
 695			if (0 == next_bytenr)
 696				continue;
 697			if (state->print_mask &
 698			    BTRFSIC_PRINT_MASK_ROOT_CHUNK_LOG_TREE_LOCATION)
 699				pr_info("log@%llu\n", next_bytenr);
 
 700			break;
 701		}
 702
 703		num_copies = btrfs_num_copies(fs_info, next_bytenr,
 704					      state->metablock_size);
 
 705		if (state->print_mask & BTRFSIC_PRINT_MASK_NUM_COPIES)
 706			pr_info("num_copies(log_bytenr=%llu) = %d\n",
 707			       next_bytenr, num_copies);
 708
 709		for (mirror_num = 1; mirror_num <= num_copies; mirror_num++) {
 710			struct btrfsic_block *next_block;
 711			struct btrfsic_block_data_ctx tmp_next_block_ctx;
 712			struct btrfsic_block_link *l;
 713
 714			ret = btrfsic_map_block(state, next_bytenr,
 715						state->metablock_size,
 716						&tmp_next_block_ctx,
 717						mirror_num);
 718			if (ret) {
 719				pr_info("btrfsic: btrfsic_map_block(root @%llu, mirror %d) failed!\n",
 720				       next_bytenr, mirror_num);
 
 
 
 721				kfree(selected_super);
 722				return -1;
 723			}
 724
 725			next_block = btrfsic_block_hashtable_lookup(
 726					tmp_next_block_ctx.dev->bdev,
 727					tmp_next_block_ctx.dev_bytenr,
 728					&state->block_hashtable);
 729			BUG_ON(NULL == next_block);
 730
 731			l = btrfsic_block_link_hashtable_lookup(
 732					tmp_next_block_ctx.dev->bdev,
 733					tmp_next_block_ctx.dev_bytenr,
 734					state->latest_superblock->dev_state->
 735					bdev,
 736					state->latest_superblock->dev_bytenr,
 737					&state->block_link_hashtable);
 738			BUG_ON(NULL == l);
 739
 740			ret = btrfsic_read_block(state, &tmp_next_block_ctx);
 741			if (ret < (int)PAGE_SIZE) {
 742				pr_info("btrfsic: read @logical %llu failed!\n",
 
 
 743				       tmp_next_block_ctx.start);
 744				btrfsic_release_block_ctx(&tmp_next_block_ctx);
 745				kfree(selected_super);
 746				return -1;
 747			}
 748
 749			ret = btrfsic_process_metablock(state,
 750							next_block,
 751							&tmp_next_block_ctx,
 752							BTRFS_MAX_LEVEL + 3, 1);
 753			btrfsic_release_block_ctx(&tmp_next_block_ctx);
 754		}
 755	}
 756
 757	kfree(selected_super);
 758	return ret;
 759}
 760
 761static int btrfsic_process_superblock_dev_mirror(
 762		struct btrfsic_state *state,
 763		struct btrfsic_dev_state *dev_state,
 764		struct btrfs_device *device,
 765		int superblock_mirror_num,
 766		struct btrfsic_dev_state **selected_dev_state,
 767		struct btrfs_super_block *selected_super)
 768{
 769	struct btrfs_fs_info *fs_info = state->fs_info;
 770	struct btrfs_super_block *super_tmp;
 771	u64 dev_bytenr;
 772	struct buffer_head *bh;
 773	struct btrfsic_block *superblock_tmp;
 774	int pass;
 775	struct block_device *const superblock_bdev = device->bdev;
 776
 777	/* super block bytenr is always the unmapped device bytenr */
 778	dev_bytenr = btrfs_sb_offset(superblock_mirror_num);
 779	if (dev_bytenr + BTRFS_SUPER_INFO_SIZE > device->commit_total_bytes)
 780		return -1;
 781	bh = __bread(superblock_bdev, dev_bytenr / BTRFS_BDEV_BLOCKSIZE,
 782		     BTRFS_SUPER_INFO_SIZE);
 783	if (NULL == bh)
 784		return -1;
 785	super_tmp = (struct btrfs_super_block *)
 786	    (bh->b_data + (dev_bytenr & (BTRFS_BDEV_BLOCKSIZE - 1)));
 787
 788	if (btrfs_super_bytenr(super_tmp) != dev_bytenr ||
 789	    btrfs_super_magic(super_tmp) != BTRFS_MAGIC ||
 
 790	    memcmp(device->uuid, super_tmp->dev_item.uuid, BTRFS_UUID_SIZE) ||
 791	    btrfs_super_nodesize(super_tmp) != state->metablock_size ||
 
 792	    btrfs_super_sectorsize(super_tmp) != state->datablock_size) {
 793		brelse(bh);
 794		return 0;
 795	}
 796
 797	superblock_tmp =
 798	    btrfsic_block_hashtable_lookup(superblock_bdev,
 799					   dev_bytenr,
 800					   &state->block_hashtable);
 801	if (NULL == superblock_tmp) {
 802		superblock_tmp = btrfsic_block_alloc();
 803		if (NULL == superblock_tmp) {
 804			pr_info("btrfsic: error, kmalloc failed!\n");
 805			brelse(bh);
 806			return -1;
 807		}
 808		/* for superblock, only the dev_bytenr makes sense */
 809		superblock_tmp->dev_bytenr = dev_bytenr;
 810		superblock_tmp->dev_state = dev_state;
 811		superblock_tmp->logical_bytenr = dev_bytenr;
 812		superblock_tmp->generation = btrfs_super_generation(super_tmp);
 813		superblock_tmp->is_metadata = 1;
 814		superblock_tmp->is_superblock = 1;
 815		superblock_tmp->is_iodone = 1;
 816		superblock_tmp->never_written = 0;
 817		superblock_tmp->mirror_num = 1 + superblock_mirror_num;
 818		if (state->print_mask & BTRFSIC_PRINT_MASK_SUPERBLOCK_WRITE)
 819			btrfs_info_in_rcu(fs_info,
 820				"new initial S-block (bdev %p, %s) @%llu (%s/%llu/%d)",
 821				     superblock_bdev,
 822				     rcu_str_deref(device->name), dev_bytenr,
 823				     dev_state->name, dev_bytenr,
 
 
 824				     superblock_mirror_num);
 825		list_add(&superblock_tmp->all_blocks_node,
 826			 &state->all_blocks_list);
 827		btrfsic_block_hashtable_add(superblock_tmp,
 828					    &state->block_hashtable);
 829	}
 830
 831	/* select the one with the highest generation field */
 832	if (btrfs_super_generation(super_tmp) >
 833	    state->max_superblock_generation ||
 834	    0 == state->max_superblock_generation) {
 835		memcpy(selected_super, super_tmp, sizeof(*selected_super));
 836		*selected_dev_state = dev_state;
 837		state->max_superblock_generation =
 838		    btrfs_super_generation(super_tmp);
 839		state->latest_superblock = superblock_tmp;
 840	}
 841
 842	for (pass = 0; pass < 3; pass++) {
 843		u64 next_bytenr;
 844		int num_copies;
 845		int mirror_num;
 846		const char *additional_string = NULL;
 847		struct btrfs_disk_key tmp_disk_key;
 848
 849		tmp_disk_key.type = BTRFS_ROOT_ITEM_KEY;
 850		tmp_disk_key.offset = 0;
 851		switch (pass) {
 852		case 0:
 853			btrfs_set_disk_key_objectid(&tmp_disk_key,
 854						    BTRFS_ROOT_TREE_OBJECTID);
 855			additional_string = "initial root ";
 856			next_bytenr = btrfs_super_root(super_tmp);
 857			break;
 858		case 1:
 859			btrfs_set_disk_key_objectid(&tmp_disk_key,
 860						    BTRFS_CHUNK_TREE_OBJECTID);
 861			additional_string = "initial chunk ";
 862			next_bytenr = btrfs_super_chunk_root(super_tmp);
 863			break;
 864		case 2:
 865			btrfs_set_disk_key_objectid(&tmp_disk_key,
 866						    BTRFS_TREE_LOG_OBJECTID);
 867			additional_string = "initial log ";
 868			next_bytenr = btrfs_super_log_root(super_tmp);
 869			if (0 == next_bytenr)
 870				continue;
 871			break;
 872		}
 873
 874		num_copies = btrfs_num_copies(fs_info, next_bytenr,
 875					      state->metablock_size);
 
 876		if (state->print_mask & BTRFSIC_PRINT_MASK_NUM_COPIES)
 877			pr_info("num_copies(log_bytenr=%llu) = %d\n",
 878			       next_bytenr, num_copies);
 879		for (mirror_num = 1; mirror_num <= num_copies; mirror_num++) {
 880			struct btrfsic_block *next_block;
 881			struct btrfsic_block_data_ctx tmp_next_block_ctx;
 882			struct btrfsic_block_link *l;
 883
 884			if (btrfsic_map_block(state, next_bytenr,
 885					      state->metablock_size,
 886					      &tmp_next_block_ctx,
 887					      mirror_num)) {
 888				pr_info("btrfsic: btrfsic_map_block(bytenr @%llu, mirror %d) failed!\n",
 889				       next_bytenr, mirror_num);
 
 
 890				brelse(bh);
 891				return -1;
 892			}
 893
 894			next_block = btrfsic_block_lookup_or_add(
 895					state, &tmp_next_block_ctx,
 896					additional_string, 1, 1, 0,
 897					mirror_num, NULL);
 898			if (NULL == next_block) {
 899				btrfsic_release_block_ctx(&tmp_next_block_ctx);
 900				brelse(bh);
 901				return -1;
 902			}
 903
 904			next_block->disk_key = tmp_disk_key;
 905			next_block->generation = BTRFSIC_GENERATION_UNKNOWN;
 906			l = btrfsic_block_link_lookup_or_add(
 907					state, &tmp_next_block_ctx,
 908					next_block, superblock_tmp,
 909					BTRFSIC_GENERATION_UNKNOWN);
 910			btrfsic_release_block_ctx(&tmp_next_block_ctx);
 911			if (NULL == l) {
 912				brelse(bh);
 913				return -1;
 914			}
 915		}
 916	}
 917	if (state->print_mask & BTRFSIC_PRINT_MASK_INITIAL_ALL_TREES)
 918		btrfsic_dump_tree_sub(state, superblock_tmp, 0);
 919
 920	brelse(bh);
 921	return 0;
 922}
 923
 924static struct btrfsic_stack_frame *btrfsic_stack_frame_alloc(void)
 925{
 926	struct btrfsic_stack_frame *sf;
 927
 928	sf = kzalloc(sizeof(*sf), GFP_NOFS);
 929	if (NULL == sf)
 930		pr_info("btrfsic: alloc memory failed!\n");
 931	else
 932		sf->magic = BTRFSIC_BLOCK_STACK_FRAME_MAGIC_NUMBER;
 933	return sf;
 934}
 935
 936static void btrfsic_stack_frame_free(struct btrfsic_stack_frame *sf)
 937{
 938	BUG_ON(!(NULL == sf ||
 939		 BTRFSIC_BLOCK_STACK_FRAME_MAGIC_NUMBER == sf->magic));
 940	kfree(sf);
 941}
 942
 943static noinline_for_stack int btrfsic_process_metablock(
 944		struct btrfsic_state *state,
 945		struct btrfsic_block *const first_block,
 946		struct btrfsic_block_data_ctx *const first_block_ctx,
 947		int first_limit_nesting, int force_iodone_flag)
 948{
 949	struct btrfsic_stack_frame initial_stack_frame = { 0 };
 950	struct btrfsic_stack_frame *sf;
 951	struct btrfsic_stack_frame *next_stack;
 952	struct btrfs_header *const first_hdr =
 953		(struct btrfs_header *)first_block_ctx->datav[0];
 954
 955	BUG_ON(!first_hdr);
 956	sf = &initial_stack_frame;
 957	sf->error = 0;
 958	sf->i = -1;
 959	sf->limit_nesting = first_limit_nesting;
 960	sf->block = first_block;
 961	sf->block_ctx = first_block_ctx;
 962	sf->next_block = NULL;
 963	sf->hdr = first_hdr;
 964	sf->prev = NULL;
 965
 966continue_with_new_stack_frame:
 967	sf->block->generation = le64_to_cpu(sf->hdr->generation);
 968	if (0 == sf->hdr->level) {
 969		struct btrfs_leaf *const leafhdr =
 970		    (struct btrfs_leaf *)sf->hdr;
 971
 972		if (-1 == sf->i) {
 973			sf->nr = btrfs_stack_header_nritems(&leafhdr->header);
 974
 975			if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
 976				pr_info("leaf %llu items %d generation %llu owner %llu\n",
 977				       sf->block_ctx->start, sf->nr,
 978				       btrfs_stack_header_generation(
 979					       &leafhdr->header),
 980				       btrfs_stack_header_owner(
 981					       &leafhdr->header));
 
 
 
 
 982		}
 983
 984continue_with_current_leaf_stack_frame:
 985		if (0 == sf->num_copies || sf->mirror_num > sf->num_copies) {
 986			sf->i++;
 987			sf->num_copies = 0;
 988		}
 989
 990		if (sf->i < sf->nr) {
 991			struct btrfs_item disk_item;
 992			u32 disk_item_offset =
 993				(uintptr_t)(leafhdr->items + sf->i) -
 994				(uintptr_t)leafhdr;
 995			struct btrfs_disk_key *disk_key;
 996			u8 type;
 997			u32 item_offset;
 998			u32 item_size;
 999
1000			if (disk_item_offset + sizeof(struct btrfs_item) >
1001			    sf->block_ctx->len) {
1002leaf_item_out_of_bounce_error:
1003				pr_info("btrfsic: leaf item out of bounce at logical %llu, dev %s\n",
 
1004				       sf->block_ctx->start,
1005				       sf->block_ctx->dev->name);
1006				goto one_stack_frame_backwards;
1007			}
1008			btrfsic_read_from_block_data(sf->block_ctx,
1009						     &disk_item,
1010						     disk_item_offset,
1011						     sizeof(struct btrfs_item));
1012			item_offset = btrfs_stack_item_offset(&disk_item);
1013			item_size = btrfs_stack_item_size(&disk_item);
1014			disk_key = &disk_item.key;
1015			type = btrfs_disk_key_type(disk_key);
1016
1017			if (BTRFS_ROOT_ITEM_KEY == type) {
1018				struct btrfs_root_item root_item;
1019				u32 root_item_offset;
1020				u64 next_bytenr;
1021
1022				root_item_offset = item_offset +
1023					offsetof(struct btrfs_leaf, items);
1024				if (root_item_offset + item_size >
 
1025				    sf->block_ctx->len)
1026					goto leaf_item_out_of_bounce_error;
1027				btrfsic_read_from_block_data(
1028					sf->block_ctx, &root_item,
1029					root_item_offset,
1030					item_size);
1031				next_bytenr = btrfs_root_bytenr(&root_item);
1032
1033				sf->error =
1034				    btrfsic_create_link_to_next_block(
1035						state,
1036						sf->block,
1037						sf->block_ctx,
1038						next_bytenr,
1039						sf->limit_nesting,
1040						&sf->next_block_ctx,
1041						&sf->next_block,
1042						force_iodone_flag,
1043						&sf->num_copies,
1044						&sf->mirror_num,
1045						disk_key,
1046						btrfs_root_generation(
1047						&root_item));
1048				if (sf->error)
1049					goto one_stack_frame_backwards;
1050
1051				if (NULL != sf->next_block) {
1052					struct btrfs_header *const next_hdr =
1053					    (struct btrfs_header *)
1054					    sf->next_block_ctx.datav[0];
1055
1056					next_stack =
1057					    btrfsic_stack_frame_alloc();
1058					if (NULL == next_stack) {
1059						sf->error = -1;
1060						btrfsic_release_block_ctx(
1061								&sf->
1062								next_block_ctx);
1063						goto one_stack_frame_backwards;
1064					}
1065
1066					next_stack->i = -1;
1067					next_stack->block = sf->next_block;
1068					next_stack->block_ctx =
1069					    &sf->next_block_ctx;
1070					next_stack->next_block = NULL;
1071					next_stack->hdr = next_hdr;
1072					next_stack->limit_nesting =
1073					    sf->limit_nesting - 1;
1074					next_stack->prev = sf;
1075					sf = next_stack;
1076					goto continue_with_new_stack_frame;
1077				}
1078			} else if (BTRFS_EXTENT_DATA_KEY == type &&
1079				   state->include_extent_data) {
1080				sf->error = btrfsic_handle_extent_data(
1081						state,
1082						sf->block,
1083						sf->block_ctx,
1084						item_offset,
1085						force_iodone_flag);
1086				if (sf->error)
1087					goto one_stack_frame_backwards;
1088			}
1089
1090			goto continue_with_current_leaf_stack_frame;
1091		}
1092	} else {
1093		struct btrfs_node *const nodehdr = (struct btrfs_node *)sf->hdr;
1094
1095		if (-1 == sf->i) {
1096			sf->nr = btrfs_stack_header_nritems(&nodehdr->header);
1097
1098			if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
1099				pr_info("node %llu level %d items %d generation %llu owner %llu\n",
 
 
1100				       sf->block_ctx->start,
1101				       nodehdr->header.level, sf->nr,
1102				       btrfs_stack_header_generation(
1103				       &nodehdr->header),
1104				       btrfs_stack_header_owner(
1105				       &nodehdr->header));
1106		}
1107
1108continue_with_current_node_stack_frame:
1109		if (0 == sf->num_copies || sf->mirror_num > sf->num_copies) {
1110			sf->i++;
1111			sf->num_copies = 0;
1112		}
1113
1114		if (sf->i < sf->nr) {
1115			struct btrfs_key_ptr key_ptr;
1116			u32 key_ptr_offset;
1117			u64 next_bytenr;
1118
1119			key_ptr_offset = (uintptr_t)(nodehdr->ptrs + sf->i) -
1120					  (uintptr_t)nodehdr;
1121			if (key_ptr_offset + sizeof(struct btrfs_key_ptr) >
1122			    sf->block_ctx->len) {
1123				pr_info("btrfsic: node item out of bounce at logical %llu, dev %s\n",
 
1124				       sf->block_ctx->start,
1125				       sf->block_ctx->dev->name);
1126				goto one_stack_frame_backwards;
1127			}
1128			btrfsic_read_from_block_data(
1129				sf->block_ctx, &key_ptr, key_ptr_offset,
1130				sizeof(struct btrfs_key_ptr));
1131			next_bytenr = btrfs_stack_key_blockptr(&key_ptr);
1132
1133			sf->error = btrfsic_create_link_to_next_block(
1134					state,
1135					sf->block,
1136					sf->block_ctx,
1137					next_bytenr,
1138					sf->limit_nesting,
1139					&sf->next_block_ctx,
1140					&sf->next_block,
1141					force_iodone_flag,
1142					&sf->num_copies,
1143					&sf->mirror_num,
1144					&key_ptr.key,
1145					btrfs_stack_key_generation(&key_ptr));
1146			if (sf->error)
1147				goto one_stack_frame_backwards;
1148
1149			if (NULL != sf->next_block) {
1150				struct btrfs_header *const next_hdr =
1151				    (struct btrfs_header *)
1152				    sf->next_block_ctx.datav[0];
1153
1154				next_stack = btrfsic_stack_frame_alloc();
1155				if (NULL == next_stack) {
1156					sf->error = -1;
1157					goto one_stack_frame_backwards;
1158				}
1159
1160				next_stack->i = -1;
1161				next_stack->block = sf->next_block;
1162				next_stack->block_ctx = &sf->next_block_ctx;
1163				next_stack->next_block = NULL;
1164				next_stack->hdr = next_hdr;
1165				next_stack->limit_nesting =
1166				    sf->limit_nesting - 1;
1167				next_stack->prev = sf;
1168				sf = next_stack;
1169				goto continue_with_new_stack_frame;
1170			}
1171
1172			goto continue_with_current_node_stack_frame;
1173		}
1174	}
1175
1176one_stack_frame_backwards:
1177	if (NULL != sf->prev) {
1178		struct btrfsic_stack_frame *const prev = sf->prev;
1179
1180		/* the one for the initial block is freed in the caller */
1181		btrfsic_release_block_ctx(sf->block_ctx);
1182
1183		if (sf->error) {
1184			prev->error = sf->error;
1185			btrfsic_stack_frame_free(sf);
1186			sf = prev;
1187			goto one_stack_frame_backwards;
1188		}
1189
1190		btrfsic_stack_frame_free(sf);
1191		sf = prev;
1192		goto continue_with_new_stack_frame;
1193	} else {
1194		BUG_ON(&initial_stack_frame != sf);
1195	}
1196
1197	return sf->error;
1198}
1199
1200static void btrfsic_read_from_block_data(
1201	struct btrfsic_block_data_ctx *block_ctx,
1202	void *dstv, u32 offset, size_t len)
1203{
1204	size_t cur;
1205	size_t pgoff;
1206	char *kaddr;
1207	char *dst = (char *)dstv;
1208	size_t start_offset = offset_in_page(block_ctx->start);
1209	unsigned long i = (start_offset + offset) >> PAGE_SHIFT;
1210
1211	WARN_ON(offset + len > block_ctx->len);
1212	pgoff = offset_in_page(start_offset + offset);
 
1213
1214	while (len > 0) {
1215		cur = min(len, ((size_t)PAGE_SIZE - pgoff));
1216		BUG_ON(i >= DIV_ROUND_UP(block_ctx->len, PAGE_SIZE));
 
1217		kaddr = block_ctx->datav[i];
1218		memcpy(dst, kaddr + pgoff, cur);
1219
1220		dst += cur;
1221		len -= cur;
1222		pgoff = 0;
1223		i++;
1224	}
1225}
1226
1227static int btrfsic_create_link_to_next_block(
1228		struct btrfsic_state *state,
1229		struct btrfsic_block *block,
1230		struct btrfsic_block_data_ctx *block_ctx,
1231		u64 next_bytenr,
1232		int limit_nesting,
1233		struct btrfsic_block_data_ctx *next_block_ctx,
1234		struct btrfsic_block **next_blockp,
1235		int force_iodone_flag,
1236		int *num_copiesp, int *mirror_nump,
1237		struct btrfs_disk_key *disk_key,
1238		u64 parent_generation)
1239{
1240	struct btrfs_fs_info *fs_info = state->fs_info;
1241	struct btrfsic_block *next_block = NULL;
1242	int ret;
1243	struct btrfsic_block_link *l;
1244	int did_alloc_block_link;
1245	int block_was_created;
1246
1247	*next_blockp = NULL;
1248	if (0 == *num_copiesp) {
1249		*num_copiesp = btrfs_num_copies(fs_info, next_bytenr,
1250						state->metablock_size);
 
1251		if (state->print_mask & BTRFSIC_PRINT_MASK_NUM_COPIES)
1252			pr_info("num_copies(log_bytenr=%llu) = %d\n",
1253			       next_bytenr, *num_copiesp);
1254		*mirror_nump = 1;
1255	}
1256
1257	if (*mirror_nump > *num_copiesp)
1258		return 0;
1259
1260	if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
1261		pr_info("btrfsic_create_link_to_next_block(mirror_num=%d)\n",
 
1262		       *mirror_nump);
1263	ret = btrfsic_map_block(state, next_bytenr,
1264				state->metablock_size,
1265				next_block_ctx, *mirror_nump);
1266	if (ret) {
1267		pr_info("btrfsic: btrfsic_map_block(@%llu, mirror=%d) failed!\n",
1268		       next_bytenr, *mirror_nump);
 
1269		btrfsic_release_block_ctx(next_block_ctx);
1270		*next_blockp = NULL;
1271		return -1;
1272	}
1273
1274	next_block = btrfsic_block_lookup_or_add(state,
1275						 next_block_ctx, "referenced ",
1276						 1, force_iodone_flag,
1277						 !force_iodone_flag,
1278						 *mirror_nump,
1279						 &block_was_created);
1280	if (NULL == next_block) {
1281		btrfsic_release_block_ctx(next_block_ctx);
1282		*next_blockp = NULL;
1283		return -1;
1284	}
1285	if (block_was_created) {
1286		l = NULL;
1287		next_block->generation = BTRFSIC_GENERATION_UNKNOWN;
1288	} else {
1289		if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE) {
1290			if (next_block->logical_bytenr != next_bytenr &&
1291			    !(!next_block->is_metadata &&
1292			      0 == next_block->logical_bytenr))
1293				pr_info("Referenced block @%llu (%s/%llu/%d) found in hash table, %c, bytenr mismatch (!= stored %llu).\n",
1294				       next_bytenr, next_block_ctx->dev->name,
1295				       next_block_ctx->dev_bytenr, *mirror_nump,
1296				       btrfsic_get_block_type(state,
1297							      next_block),
1298				       next_block->logical_bytenr);
1299			else
1300				pr_info("Referenced block @%llu (%s/%llu/%d) found in hash table, %c.\n",
1301				       next_bytenr, next_block_ctx->dev->name,
1302				       next_block_ctx->dev_bytenr, *mirror_nump,
1303				       btrfsic_get_block_type(state,
1304							      next_block));
1305		}
 
 
 
 
 
1306		next_block->logical_bytenr = next_bytenr;
1307
1308		next_block->mirror_num = *mirror_nump;
1309		l = btrfsic_block_link_hashtable_lookup(
1310				next_block_ctx->dev->bdev,
1311				next_block_ctx->dev_bytenr,
1312				block_ctx->dev->bdev,
1313				block_ctx->dev_bytenr,
1314				&state->block_link_hashtable);
1315	}
1316
1317	next_block->disk_key = *disk_key;
1318	if (NULL == l) {
1319		l = btrfsic_block_link_alloc();
1320		if (NULL == l) {
1321			pr_info("btrfsic: error, kmalloc failed!\n");
1322			btrfsic_release_block_ctx(next_block_ctx);
1323			*next_blockp = NULL;
1324			return -1;
1325		}
1326
1327		did_alloc_block_link = 1;
1328		l->block_ref_to = next_block;
1329		l->block_ref_from = block;
1330		l->ref_cnt = 1;
1331		l->parent_generation = parent_generation;
1332
1333		if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
1334			btrfsic_print_add_link(state, l);
1335
1336		list_add(&l->node_ref_to, &block->ref_to_list);
1337		list_add(&l->node_ref_from, &next_block->ref_from_list);
1338
1339		btrfsic_block_link_hashtable_add(l,
1340						 &state->block_link_hashtable);
1341	} else {
1342		did_alloc_block_link = 0;
1343		if (0 == limit_nesting) {
1344			l->ref_cnt++;
1345			l->parent_generation = parent_generation;
1346			if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
1347				btrfsic_print_add_link(state, l);
1348		}
1349	}
1350
1351	if (limit_nesting > 0 && did_alloc_block_link) {
1352		ret = btrfsic_read_block(state, next_block_ctx);
1353		if (ret < (int)next_block_ctx->len) {
1354			pr_info("btrfsic: read block @logical %llu failed!\n",
1355			       next_bytenr);
 
1356			btrfsic_release_block_ctx(next_block_ctx);
1357			*next_blockp = NULL;
1358			return -1;
1359		}
1360
1361		*next_blockp = next_block;
1362	} else {
1363		*next_blockp = NULL;
1364	}
1365	(*mirror_nump)++;
1366
1367	return 0;
1368}
1369
1370static int btrfsic_handle_extent_data(
1371		struct btrfsic_state *state,
1372		struct btrfsic_block *block,
1373		struct btrfsic_block_data_ctx *block_ctx,
1374		u32 item_offset, int force_iodone_flag)
1375{
1376	struct btrfs_fs_info *fs_info = state->fs_info;
1377	struct btrfs_file_extent_item file_extent_item;
1378	u64 file_extent_item_offset;
1379	u64 next_bytenr;
1380	u64 num_bytes;
1381	u64 generation;
1382	struct btrfsic_block_link *l;
1383	int ret;
1384
1385	file_extent_item_offset = offsetof(struct btrfs_leaf, items) +
1386				  item_offset;
1387	if (file_extent_item_offset +
1388	    offsetof(struct btrfs_file_extent_item, disk_num_bytes) >
1389	    block_ctx->len) {
1390		pr_info("btrfsic: file item out of bounce at logical %llu, dev %s\n",
 
1391		       block_ctx->start, block_ctx->dev->name);
1392		return -1;
1393	}
1394
1395	btrfsic_read_from_block_data(block_ctx, &file_extent_item,
1396		file_extent_item_offset,
1397		offsetof(struct btrfs_file_extent_item, disk_num_bytes));
1398	if (BTRFS_FILE_EXTENT_REG != file_extent_item.type ||
1399	    btrfs_stack_file_extent_disk_bytenr(&file_extent_item) == 0) {
1400		if (state->print_mask & BTRFSIC_PRINT_MASK_VERY_VERBOSE)
1401			pr_info("extent_data: type %u, disk_bytenr = %llu\n",
1402			       file_extent_item.type,
1403			       btrfs_stack_file_extent_disk_bytenr(
1404			       &file_extent_item));
1405		return 0;
1406	}
1407
1408	if (file_extent_item_offset + sizeof(struct btrfs_file_extent_item) >
1409	    block_ctx->len) {
1410		pr_info("btrfsic: file item out of bounce at logical %llu, dev %s\n",
 
1411		       block_ctx->start, block_ctx->dev->name);
1412		return -1;
1413	}
1414	btrfsic_read_from_block_data(block_ctx, &file_extent_item,
1415				     file_extent_item_offset,
1416				     sizeof(struct btrfs_file_extent_item));
1417	next_bytenr = btrfs_stack_file_extent_disk_bytenr(&file_extent_item);
1418	if (btrfs_stack_file_extent_compression(&file_extent_item) ==
1419	    BTRFS_COMPRESS_NONE) {
1420		next_bytenr += btrfs_stack_file_extent_offset(&file_extent_item);
1421		num_bytes = btrfs_stack_file_extent_num_bytes(&file_extent_item);
1422	} else {
1423		num_bytes = btrfs_stack_file_extent_disk_num_bytes(&file_extent_item);
1424	}
1425	generation = btrfs_stack_file_extent_generation(&file_extent_item);
1426
1427	if (state->print_mask & BTRFSIC_PRINT_MASK_VERY_VERBOSE)
1428		pr_info("extent_data: type %u, disk_bytenr = %llu, offset = %llu, num_bytes = %llu\n",
 
1429		       file_extent_item.type,
1430		       btrfs_stack_file_extent_disk_bytenr(&file_extent_item),
1431		       btrfs_stack_file_extent_offset(&file_extent_item),
1432		       num_bytes);
 
1433	while (num_bytes > 0) {
1434		u32 chunk_len;
1435		int num_copies;
1436		int mirror_num;
1437
1438		if (num_bytes > state->datablock_size)
1439			chunk_len = state->datablock_size;
1440		else
1441			chunk_len = num_bytes;
1442
1443		num_copies = btrfs_num_copies(fs_info, next_bytenr,
1444					      state->datablock_size);
 
1445		if (state->print_mask & BTRFSIC_PRINT_MASK_NUM_COPIES)
1446			pr_info("num_copies(log_bytenr=%llu) = %d\n",
1447			       next_bytenr, num_copies);
1448		for (mirror_num = 1; mirror_num <= num_copies; mirror_num++) {
1449			struct btrfsic_block_data_ctx next_block_ctx;
1450			struct btrfsic_block *next_block;
1451			int block_was_created;
1452
1453			if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
1454				pr_info("btrfsic_handle_extent_data(mirror_num=%d)\n",
1455					mirror_num);
1456			if (state->print_mask & BTRFSIC_PRINT_MASK_VERY_VERBOSE)
1457				pr_info("\tdisk_bytenr = %llu, num_bytes %u\n",
1458				       next_bytenr, chunk_len);
 
 
1459			ret = btrfsic_map_block(state, next_bytenr,
1460						chunk_len, &next_block_ctx,
1461						mirror_num);
1462			if (ret) {
1463				pr_info("btrfsic: btrfsic_map_block(@%llu, mirror=%d) failed!\n",
1464				       next_bytenr, mirror_num);
 
 
 
1465				return -1;
1466			}
1467
1468			next_block = btrfsic_block_lookup_or_add(
1469					state,
1470					&next_block_ctx,
1471					"referenced ",
1472					0,
1473					force_iodone_flag,
1474					!force_iodone_flag,
1475					mirror_num,
1476					&block_was_created);
1477			if (NULL == next_block) {
1478				pr_info("btrfsic: error, kmalloc failed!\n");
 
1479				btrfsic_release_block_ctx(&next_block_ctx);
1480				return -1;
1481			}
1482			if (!block_was_created) {
1483				if ((state->print_mask &
1484				     BTRFSIC_PRINT_MASK_VERBOSE) &&
1485				    next_block->logical_bytenr != next_bytenr &&
1486				    !(!next_block->is_metadata &&
1487				      0 == next_block->logical_bytenr)) {
1488					pr_info("Referenced block @%llu (%s/%llu/%d) found in hash table, D, bytenr mismatch (!= stored %llu).\n",
1489					       next_bytenr,
 
 
 
 
 
1490					       next_block_ctx.dev->name,
 
1491					       next_block_ctx.dev_bytenr,
1492					       mirror_num,
 
1493					       next_block->logical_bytenr);
1494				}
1495				next_block->logical_bytenr = next_bytenr;
1496				next_block->mirror_num = mirror_num;
1497			}
1498
1499			l = btrfsic_block_link_lookup_or_add(state,
1500							     &next_block_ctx,
1501							     next_block, block,
1502							     generation);
1503			btrfsic_release_block_ctx(&next_block_ctx);
1504			if (NULL == l)
1505				return -1;
1506		}
1507
1508		next_bytenr += chunk_len;
1509		num_bytes -= chunk_len;
1510	}
1511
1512	return 0;
1513}
1514
1515static int btrfsic_map_block(struct btrfsic_state *state, u64 bytenr, u32 len,
1516			     struct btrfsic_block_data_ctx *block_ctx_out,
1517			     int mirror_num)
1518{
1519	struct btrfs_fs_info *fs_info = state->fs_info;
1520	int ret;
1521	u64 length;
1522	struct btrfs_bio *multi = NULL;
1523	struct btrfs_device *device;
1524
1525	length = len;
1526	ret = btrfs_map_block(fs_info, BTRFS_MAP_READ,
1527			      bytenr, &length, &multi, mirror_num);
1528
1529	if (ret) {
1530		block_ctx_out->start = 0;
1531		block_ctx_out->dev_bytenr = 0;
1532		block_ctx_out->len = 0;
1533		block_ctx_out->dev = NULL;
1534		block_ctx_out->datav = NULL;
1535		block_ctx_out->pagev = NULL;
1536		block_ctx_out->mem_to_free = NULL;
1537
1538		return ret;
1539	}
1540
1541	device = multi->stripes[0].dev;
1542	if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state) ||
1543	    !device->bdev || !device->name)
1544		block_ctx_out->dev = NULL;
1545	else
1546		block_ctx_out->dev = btrfsic_dev_state_lookup(
1547							device->bdev->bd_dev);
1548	block_ctx_out->dev_bytenr = multi->stripes[0].physical;
1549	block_ctx_out->start = bytenr;
1550	block_ctx_out->len = len;
1551	block_ctx_out->datav = NULL;
1552	block_ctx_out->pagev = NULL;
1553	block_ctx_out->mem_to_free = NULL;
1554
1555	kfree(multi);
 
1556	if (NULL == block_ctx_out->dev) {
1557		ret = -ENXIO;
1558		pr_info("btrfsic: error, cannot lookup dev (#1)!\n");
1559	}
1560
1561	return ret;
1562}
1563
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1564static void btrfsic_release_block_ctx(struct btrfsic_block_data_ctx *block_ctx)
1565{
1566	if (block_ctx->mem_to_free) {
1567		unsigned int num_pages;
1568
1569		BUG_ON(!block_ctx->datav);
1570		BUG_ON(!block_ctx->pagev);
1571		num_pages = (block_ctx->len + (u64)PAGE_SIZE - 1) >>
1572			    PAGE_SHIFT;
1573		while (num_pages > 0) {
1574			num_pages--;
1575			if (block_ctx->datav[num_pages]) {
1576				kunmap(block_ctx->pagev[num_pages]);
1577				block_ctx->datav[num_pages] = NULL;
1578			}
1579			if (block_ctx->pagev[num_pages]) {
1580				__free_page(block_ctx->pagev[num_pages]);
1581				block_ctx->pagev[num_pages] = NULL;
1582			}
1583		}
1584
1585		kfree(block_ctx->mem_to_free);
1586		block_ctx->mem_to_free = NULL;
1587		block_ctx->pagev = NULL;
1588		block_ctx->datav = NULL;
1589	}
1590}
1591
1592static int btrfsic_read_block(struct btrfsic_state *state,
1593			      struct btrfsic_block_data_ctx *block_ctx)
1594{
1595	unsigned int num_pages;
1596	unsigned int i;
1597	size_t size;
1598	u64 dev_bytenr;
1599	int ret;
1600
1601	BUG_ON(block_ctx->datav);
1602	BUG_ON(block_ctx->pagev);
1603	BUG_ON(block_ctx->mem_to_free);
1604	if (!PAGE_ALIGNED(block_ctx->dev_bytenr)) {
1605		pr_info("btrfsic: read_block() with unaligned bytenr %llu\n",
1606		       block_ctx->dev_bytenr);
 
1607		return -1;
1608	}
1609
1610	num_pages = (block_ctx->len + (u64)PAGE_SIZE - 1) >>
1611		    PAGE_SHIFT;
1612	size = sizeof(*block_ctx->datav) + sizeof(*block_ctx->pagev);
1613	block_ctx->mem_to_free = kcalloc(num_pages, size, GFP_NOFS);
 
1614	if (!block_ctx->mem_to_free)
1615		return -ENOMEM;
1616	block_ctx->datav = block_ctx->mem_to_free;
1617	block_ctx->pagev = (struct page **)(block_ctx->datav + num_pages);
1618	for (i = 0; i < num_pages; i++) {
1619		block_ctx->pagev[i] = alloc_page(GFP_NOFS);
1620		if (!block_ctx->pagev[i])
1621			return -1;
1622	}
1623
1624	dev_bytenr = block_ctx->dev_bytenr;
1625	for (i = 0; i < num_pages;) {
1626		struct bio *bio;
1627		unsigned int j;
 
1628
1629		bio = btrfs_io_bio_alloc(num_pages - i);
1630		bio_set_dev(bio, block_ctx->dev->bdev);
1631		bio->bi_iter.bi_sector = dev_bytenr >> 9;
1632		bio->bi_opf = REQ_OP_READ;
 
 
 
 
 
 
 
1633
1634		for (j = i; j < num_pages; j++) {
1635			ret = bio_add_page(bio, block_ctx->pagev[j],
1636					   PAGE_SIZE, 0);
1637			if (PAGE_SIZE != ret)
1638				break;
1639		}
1640		if (j == i) {
1641			pr_info("btrfsic: error, failed to add a single page!\n");
 
1642			return -1;
1643		}
1644		if (submit_bio_wait(bio)) {
1645			pr_info("btrfsic: read error at logical %llu dev %s!\n",
 
 
 
 
 
 
1646			       block_ctx->start, block_ctx->dev->name);
1647			bio_put(bio);
1648			return -1;
1649		}
1650		bio_put(bio);
1651		dev_bytenr += (j - i) * PAGE_SIZE;
1652		i = j;
1653	}
1654	for (i = 0; i < num_pages; i++)
1655		block_ctx->datav[i] = kmap(block_ctx->pagev[i]);
 
 
 
 
 
 
1656
1657	return block_ctx->len;
1658}
1659
 
 
 
 
 
1660static void btrfsic_dump_database(struct btrfsic_state *state)
1661{
1662	const struct btrfsic_block *b_all;
1663
1664	BUG_ON(NULL == state);
1665
1666	pr_info("all_blocks_list:\n");
1667	list_for_each_entry(b_all, &state->all_blocks_list, all_blocks_node) {
1668		const struct btrfsic_block_link *l;
 
 
 
 
1669
1670		pr_info("%c-block @%llu (%s/%llu/%d)\n",
1671		       btrfsic_get_block_type(state, b_all),
1672		       b_all->logical_bytenr, b_all->dev_state->name,
1673		       b_all->dev_bytenr, b_all->mirror_num);
1674
1675		list_for_each_entry(l, &b_all->ref_to_list, node_ref_to) {
1676			pr_info(" %c @%llu (%s/%llu/%d) refers %u* to %c @%llu (%s/%llu/%d)\n",
 
 
 
 
 
 
 
 
 
1677			       btrfsic_get_block_type(state, b_all),
1678			       b_all->logical_bytenr, b_all->dev_state->name,
1679			       b_all->dev_bytenr, b_all->mirror_num,
 
 
1680			       l->ref_cnt,
1681			       btrfsic_get_block_type(state, l->block_ref_to),
 
1682			       l->block_ref_to->logical_bytenr,
1683			       l->block_ref_to->dev_state->name,
1684			       l->block_ref_to->dev_bytenr,
1685			       l->block_ref_to->mirror_num);
1686		}
1687
1688		list_for_each_entry(l, &b_all->ref_from_list, node_ref_from) {
1689			pr_info(" %c @%llu (%s/%llu/%d) is ref %u* from %c @%llu (%s/%llu/%d)\n",
 
 
 
 
 
 
 
1690			       btrfsic_get_block_type(state, b_all),
1691			       b_all->logical_bytenr, b_all->dev_state->name,
1692			       b_all->dev_bytenr, b_all->mirror_num,
 
 
1693			       l->ref_cnt,
1694			       btrfsic_get_block_type(state, l->block_ref_from),
 
1695			       l->block_ref_from->logical_bytenr,
1696			       l->block_ref_from->dev_state->name,
 
1697			       l->block_ref_from->dev_bytenr,
1698			       l->block_ref_from->mirror_num);
1699		}
1700
1701		pr_info("\n");
1702	}
1703}
1704
1705/*
1706 * Test whether the disk block contains a tree block (leaf or node)
1707 * (note that this test fails for the super block)
1708 */
1709static noinline_for_stack int btrfsic_test_for_metadata(
1710		struct btrfsic_state *state,
1711		char **datav, unsigned int num_pages)
1712{
1713	struct btrfs_fs_info *fs_info = state->fs_info;
1714	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
1715	struct btrfs_header *h;
1716	u8 csum[BTRFS_CSUM_SIZE];
 
1717	unsigned int i;
1718
1719	if (num_pages * PAGE_SIZE < state->metablock_size)
1720		return 1; /* not metadata */
1721	num_pages = state->metablock_size >> PAGE_SHIFT;
1722	h = (struct btrfs_header *)datav[0];
1723
1724	if (memcmp(h->fsid, fs_info->fs_devices->fsid, BTRFS_FSID_SIZE))
1725		return 1;
1726
1727	shash->tfm = fs_info->csum_shash;
1728	crypto_shash_init(shash);
1729
1730	for (i = 0; i < num_pages; i++) {
1731		u8 *data = i ? datav[i] : (datav[i] + BTRFS_CSUM_SIZE);
1732		size_t sublen = i ? PAGE_SIZE :
1733				    (PAGE_SIZE - BTRFS_CSUM_SIZE);
1734
1735		crypto_shash_update(shash, data, sublen);
1736	}
1737	crypto_shash_final(shash, csum);
1738	if (memcmp(csum, h->csum, state->csum_size))
1739		return 1;
1740
1741	return 0; /* is metadata */
1742}
1743
1744static void btrfsic_process_written_block(struct btrfsic_dev_state *dev_state,
1745					  u64 dev_bytenr, char **mapped_datav,
1746					  unsigned int num_pages,
1747					  struct bio *bio, int *bio_is_patched,
1748					  struct buffer_head *bh,
1749					  int submit_bio_bh_rw)
1750{
1751	int is_metadata;
1752	struct btrfsic_block *block;
1753	struct btrfsic_block_data_ctx block_ctx;
1754	int ret;
1755	struct btrfsic_state *state = dev_state->state;
1756	struct block_device *bdev = dev_state->bdev;
1757	unsigned int processed_len;
1758
1759	if (NULL != bio_is_patched)
1760		*bio_is_patched = 0;
1761
1762again:
1763	if (num_pages == 0)
1764		return;
1765
1766	processed_len = 0;
1767	is_metadata = (0 == btrfsic_test_for_metadata(state, mapped_datav,
1768						      num_pages));
1769
1770	block = btrfsic_block_hashtable_lookup(bdev, dev_bytenr,
1771					       &state->block_hashtable);
1772	if (NULL != block) {
1773		u64 bytenr = 0;
1774		struct btrfsic_block_link *l, *tmp;
 
1775
1776		if (block->is_superblock) {
1777			bytenr = btrfs_super_bytenr((struct btrfs_super_block *)
1778						    mapped_datav[0]);
1779			if (num_pages * PAGE_SIZE <
1780			    BTRFS_SUPER_INFO_SIZE) {
1781				pr_info("btrfsic: cannot work with too short bios!\n");
 
1782				return;
1783			}
1784			is_metadata = 1;
1785			BUG_ON(!PAGE_ALIGNED(BTRFS_SUPER_INFO_SIZE));
1786			processed_len = BTRFS_SUPER_INFO_SIZE;
1787			if (state->print_mask &
1788			    BTRFSIC_PRINT_MASK_TREE_BEFORE_SB_WRITE) {
1789				pr_info("[before new superblock is written]:\n");
 
1790				btrfsic_dump_tree_sub(state, block, 0);
1791			}
1792		}
1793		if (is_metadata) {
1794			if (!block->is_superblock) {
1795				if (num_pages * PAGE_SIZE <
1796				    state->metablock_size) {
1797					pr_info("btrfsic: cannot work with too short bios!\n");
 
1798					return;
1799				}
1800				processed_len = state->metablock_size;
1801				bytenr = btrfs_stack_header_bytenr(
1802						(struct btrfs_header *)
1803						mapped_datav[0]);
1804				btrfsic_cmp_log_and_dev_bytenr(state, bytenr,
1805							       dev_state,
1806							       dev_bytenr);
1807			}
1808			if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE) {
1809				if (block->logical_bytenr != bytenr &&
1810				    !(!block->is_metadata &&
1811				      block->logical_bytenr == 0))
1812					pr_info("Written block @%llu (%s/%llu/%d) found in hash table, %c, bytenr mismatch (!= stored %llu).\n",
1813					       bytenr, dev_state->name,
1814					       dev_bytenr,
1815					       block->mirror_num,
1816					       btrfsic_get_block_type(state,
1817								      block),
1818					       block->logical_bytenr);
1819				else
1820					pr_info("Written block @%llu (%s/%llu/%d) found in hash table, %c.\n",
1821					       bytenr, dev_state->name,
1822					       dev_bytenr, block->mirror_num,
1823					       btrfsic_get_block_type(state,
1824								      block));
1825			}
1826			block->logical_bytenr = bytenr;
 
 
 
 
 
1827		} else {
1828			if (num_pages * PAGE_SIZE <
1829			    state->datablock_size) {
1830				pr_info("btrfsic: cannot work with too short bios!\n");
 
1831				return;
1832			}
1833			processed_len = state->datablock_size;
1834			bytenr = block->logical_bytenr;
1835			if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
1836				pr_info("Written block @%llu (%s/%llu/%d) found in hash table, %c.\n",
1837				       bytenr, dev_state->name, dev_bytenr,
 
 
 
 
1838				       block->mirror_num,
1839				       btrfsic_get_block_type(state, block));
1840		}
1841
1842		if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
1843			pr_info("ref_to_list: %cE, ref_from_list: %cE\n",
 
1844			       list_empty(&block->ref_to_list) ? ' ' : '!',
1845			       list_empty(&block->ref_from_list) ? ' ' : '!');
1846		if (btrfsic_is_block_ref_by_superblock(state, block, 0)) {
1847			pr_info("btrfs: attempt to overwrite %c-block @%llu (%s/%llu/%d), old(gen=%llu, objectid=%llu, type=%d, offset=%llu), new(gen=%llu), which is referenced by most recent superblock (superblockgen=%llu)!\n",
1848			       btrfsic_get_block_type(state, block), bytenr,
1849			       dev_state->name, dev_bytenr, block->mirror_num,
1850			       block->generation,
1851			       btrfs_disk_key_objectid(&block->disk_key),
 
 
 
 
 
 
 
 
 
1852			       block->disk_key.type,
1853			       btrfs_disk_key_offset(&block->disk_key),
1854			       btrfs_stack_header_generation(
1855				       (struct btrfs_header *) mapped_datav[0]),
 
 
 
1856			       state->max_superblock_generation);
1857			btrfsic_dump_tree(state);
1858		}
1859
1860		if (!block->is_iodone && !block->never_written) {
1861			pr_info("btrfs: attempt to overwrite %c-block @%llu (%s/%llu/%d), oldgen=%llu, newgen=%llu, which is not yet iodone!\n",
1862			       btrfsic_get_block_type(state, block), bytenr,
1863			       dev_state->name, dev_bytenr, block->mirror_num,
1864			       block->generation,
1865			       btrfs_stack_header_generation(
1866				       (struct btrfs_header *)
1867				       mapped_datav[0]));
 
 
 
 
 
1868			/* it would not be safe to go on */
1869			btrfsic_dump_tree(state);
1870			goto continue_loop;
1871		}
1872
1873		/*
1874		 * Clear all references of this block. Do not free
1875		 * the block itself even if is not referenced anymore
1876		 * because it still carries valuable information
1877		 * like whether it was ever written and IO completed.
1878		 */
1879		list_for_each_entry_safe(l, tmp, &block->ref_to_list,
1880					 node_ref_to) {
 
 
 
 
 
1881			if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
1882				btrfsic_print_rem_link(state, l);
1883			l->ref_cnt--;
1884			if (0 == l->ref_cnt) {
1885				list_del(&l->node_ref_to);
1886				list_del(&l->node_ref_from);
1887				btrfsic_block_link_hashtable_remove(l);
1888				btrfsic_block_link_free(l);
1889			}
1890		}
1891
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1892		block_ctx.dev = dev_state;
1893		block_ctx.dev_bytenr = dev_bytenr;
1894		block_ctx.start = bytenr;
1895		block_ctx.len = processed_len;
1896		block_ctx.pagev = NULL;
1897		block_ctx.mem_to_free = NULL;
1898		block_ctx.datav = mapped_datav;
1899
1900		if (is_metadata || state->include_extent_data) {
1901			block->never_written = 0;
1902			block->iodone_w_error = 0;
1903			if (NULL != bio) {
1904				block->is_iodone = 0;
1905				BUG_ON(NULL == bio_is_patched);
1906				if (!*bio_is_patched) {
1907					block->orig_bio_bh_private =
1908					    bio->bi_private;
1909					block->orig_bio_bh_end_io.bio =
1910					    bio->bi_end_io;
1911					block->next_in_same_bio = NULL;
1912					bio->bi_private = block;
1913					bio->bi_end_io = btrfsic_bio_end_io;
1914					*bio_is_patched = 1;
1915				} else {
1916					struct btrfsic_block *chained_block =
1917					    (struct btrfsic_block *)
1918					    bio->bi_private;
1919
1920					BUG_ON(NULL == chained_block);
1921					block->orig_bio_bh_private =
1922					    chained_block->orig_bio_bh_private;
1923					block->orig_bio_bh_end_io.bio =
1924					    chained_block->orig_bio_bh_end_io.
1925					    bio;
1926					block->next_in_same_bio = chained_block;
1927					bio->bi_private = block;
1928				}
1929			} else if (NULL != bh) {
1930				block->is_iodone = 0;
1931				block->orig_bio_bh_private = bh->b_private;
1932				block->orig_bio_bh_end_io.bh = bh->b_end_io;
1933				block->next_in_same_bio = NULL;
1934				bh->b_private = block;
1935				bh->b_end_io = btrfsic_bh_end_io;
1936			} else {
1937				block->is_iodone = 1;
1938				block->orig_bio_bh_private = NULL;
1939				block->orig_bio_bh_end_io.bio = NULL;
1940				block->next_in_same_bio = NULL;
1941			}
1942		}
1943
1944		block->flush_gen = dev_state->last_flush_gen + 1;
1945		block->submit_bio_bh_rw = submit_bio_bh_rw;
1946		if (is_metadata) {
1947			block->logical_bytenr = bytenr;
1948			block->is_metadata = 1;
1949			if (block->is_superblock) {
1950				BUG_ON(PAGE_SIZE !=
1951				       BTRFS_SUPER_INFO_SIZE);
1952				ret = btrfsic_process_written_superblock(
1953						state,
1954						block,
1955						(struct btrfs_super_block *)
1956						mapped_datav[0]);
1957				if (state->print_mask &
1958				    BTRFSIC_PRINT_MASK_TREE_AFTER_SB_WRITE) {
1959					pr_info("[after new superblock is written]:\n");
 
1960					btrfsic_dump_tree_sub(state, block, 0);
1961				}
1962			} else {
1963				block->mirror_num = 0;	/* unknown */
1964				ret = btrfsic_process_metablock(
1965						state,
1966						block,
1967						&block_ctx,
1968						0, 0);
1969			}
1970			if (ret)
1971				pr_info("btrfsic: btrfsic_process_metablock(root @%llu) failed!\n",
1972				       dev_bytenr);
 
 
1973		} else {
1974			block->is_metadata = 0;
1975			block->mirror_num = 0;	/* unknown */
1976			block->generation = BTRFSIC_GENERATION_UNKNOWN;
1977			if (!state->include_extent_data
1978			    && list_empty(&block->ref_from_list)) {
1979				/*
1980				 * disk block is overwritten with extent
1981				 * data (not meta data) and we are configured
1982				 * to not include extent data: take the
1983				 * chance and free the block's memory
1984				 */
1985				btrfsic_block_hashtable_remove(block);
1986				list_del(&block->all_blocks_node);
1987				btrfsic_block_free(block);
1988			}
1989		}
1990		btrfsic_release_block_ctx(&block_ctx);
1991	} else {
1992		/* block has not been found in hash table */
1993		u64 bytenr;
1994
1995		if (!is_metadata) {
1996			processed_len = state->datablock_size;
1997			if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
1998				pr_info("Written block (%s/%llu/?) !found in hash table, D.\n",
1999				       dev_state->name, dev_bytenr);
 
 
2000			if (!state->include_extent_data) {
2001				/* ignore that written D block */
2002				goto continue_loop;
2003			}
2004
2005			/* this is getting ugly for the
2006			 * include_extent_data case... */
2007			bytenr = 0;	/* unknown */
 
 
 
 
2008		} else {
2009			processed_len = state->metablock_size;
2010			bytenr = btrfs_stack_header_bytenr(
2011					(struct btrfs_header *)
2012					mapped_datav[0]);
2013			btrfsic_cmp_log_and_dev_bytenr(state, bytenr, dev_state,
2014						       dev_bytenr);
2015			if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
2016				pr_info("Written block @%llu (%s/%llu/?) !found in hash table, M.\n",
2017				       bytenr, dev_state->name, dev_bytenr);
2018		}
 
 
 
2019
 
 
 
 
 
 
 
 
 
 
 
 
 
2020		block_ctx.dev = dev_state;
2021		block_ctx.dev_bytenr = dev_bytenr;
2022		block_ctx.start = bytenr;
2023		block_ctx.len = processed_len;
2024		block_ctx.pagev = NULL;
2025		block_ctx.mem_to_free = NULL;
2026		block_ctx.datav = mapped_datav;
2027
2028		block = btrfsic_block_alloc();
2029		if (NULL == block) {
2030			pr_info("btrfsic: error, kmalloc failed!\n");
2031			btrfsic_release_block_ctx(&block_ctx);
2032			goto continue_loop;
2033		}
2034		block->dev_state = dev_state;
2035		block->dev_bytenr = dev_bytenr;
2036		block->logical_bytenr = bytenr;
2037		block->is_metadata = is_metadata;
2038		block->never_written = 0;
2039		block->iodone_w_error = 0;
2040		block->mirror_num = 0;	/* unknown */
2041		block->flush_gen = dev_state->last_flush_gen + 1;
2042		block->submit_bio_bh_rw = submit_bio_bh_rw;
2043		if (NULL != bio) {
2044			block->is_iodone = 0;
2045			BUG_ON(NULL == bio_is_patched);
2046			if (!*bio_is_patched) {
2047				block->orig_bio_bh_private = bio->bi_private;
2048				block->orig_bio_bh_end_io.bio = bio->bi_end_io;
2049				block->next_in_same_bio = NULL;
2050				bio->bi_private = block;
2051				bio->bi_end_io = btrfsic_bio_end_io;
2052				*bio_is_patched = 1;
2053			} else {
2054				struct btrfsic_block *chained_block =
2055				    (struct btrfsic_block *)
2056				    bio->bi_private;
2057
2058				BUG_ON(NULL == chained_block);
2059				block->orig_bio_bh_private =
2060				    chained_block->orig_bio_bh_private;
2061				block->orig_bio_bh_end_io.bio =
2062				    chained_block->orig_bio_bh_end_io.bio;
2063				block->next_in_same_bio = chained_block;
2064				bio->bi_private = block;
2065			}
2066		} else if (NULL != bh) {
2067			block->is_iodone = 0;
2068			block->orig_bio_bh_private = bh->b_private;
2069			block->orig_bio_bh_end_io.bh = bh->b_end_io;
2070			block->next_in_same_bio = NULL;
2071			bh->b_private = block;
2072			bh->b_end_io = btrfsic_bh_end_io;
2073		} else {
2074			block->is_iodone = 1;
2075			block->orig_bio_bh_private = NULL;
2076			block->orig_bio_bh_end_io.bio = NULL;
2077			block->next_in_same_bio = NULL;
2078		}
2079		if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
2080			pr_info("New written %c-block @%llu (%s/%llu/%d)\n",
 
2081			       is_metadata ? 'M' : 'D',
2082			       block->logical_bytenr, block->dev_state->name,
2083			       block->dev_bytenr, block->mirror_num);
 
 
2084		list_add(&block->all_blocks_node, &state->all_blocks_list);
2085		btrfsic_block_hashtable_add(block, &state->block_hashtable);
2086
2087		if (is_metadata) {
2088			ret = btrfsic_process_metablock(state, block,
2089							&block_ctx, 0, 0);
2090			if (ret)
2091				pr_info("btrfsic: process_metablock(root @%llu) failed!\n",
2092				       dev_bytenr);
 
 
2093		}
2094		btrfsic_release_block_ctx(&block_ctx);
2095	}
2096
2097continue_loop:
2098	BUG_ON(!processed_len);
2099	dev_bytenr += processed_len;
2100	mapped_datav += processed_len >> PAGE_SHIFT;
2101	num_pages -= processed_len >> PAGE_SHIFT;
2102	goto again;
2103}
2104
2105static void btrfsic_bio_end_io(struct bio *bp)
2106{
2107	struct btrfsic_block *block = (struct btrfsic_block *)bp->bi_private;
2108	int iodone_w_error;
2109
2110	/* mutex is not held! This is not save if IO is not yet completed
2111	 * on umount */
2112	iodone_w_error = 0;
2113	if (bp->bi_status)
2114		iodone_w_error = 1;
2115
2116	BUG_ON(NULL == block);
2117	bp->bi_private = block->orig_bio_bh_private;
2118	bp->bi_end_io = block->orig_bio_bh_end_io.bio;
2119
2120	do {
2121		struct btrfsic_block *next_block;
2122		struct btrfsic_dev_state *const dev_state = block->dev_state;
2123
2124		if ((dev_state->state->print_mask &
2125		     BTRFSIC_PRINT_MASK_END_IO_BIO_BH))
2126			pr_info("bio_end_io(err=%d) for %c @%llu (%s/%llu/%d)\n",
2127			       bp->bi_status,
 
2128			       btrfsic_get_block_type(dev_state->state, block),
2129			       block->logical_bytenr, dev_state->name,
2130			       block->dev_bytenr, block->mirror_num);
 
 
2131		next_block = block->next_in_same_bio;
2132		block->iodone_w_error = iodone_w_error;
2133		if (block->submit_bio_bh_rw & REQ_PREFLUSH) {
2134			dev_state->last_flush_gen++;
2135			if ((dev_state->state->print_mask &
2136			     BTRFSIC_PRINT_MASK_END_IO_BIO_BH))
2137				pr_info("bio_end_io() new %s flush_gen=%llu\n",
 
2138				       dev_state->name,
 
2139				       dev_state->last_flush_gen);
2140		}
2141		if (block->submit_bio_bh_rw & REQ_FUA)
2142			block->flush_gen = 0; /* FUA completed means block is
2143					       * on disk */
2144		block->is_iodone = 1; /* for FLUSH, this releases the block */
2145		block = next_block;
2146	} while (NULL != block);
2147
2148	bp->bi_end_io(bp);
2149}
2150
2151static void btrfsic_bh_end_io(struct buffer_head *bh, int uptodate)
2152{
2153	struct btrfsic_block *block = (struct btrfsic_block *)bh->b_private;
2154	int iodone_w_error = !uptodate;
2155	struct btrfsic_dev_state *dev_state;
2156
2157	BUG_ON(NULL == block);
2158	dev_state = block->dev_state;
2159	if ((dev_state->state->print_mask & BTRFSIC_PRINT_MASK_END_IO_BIO_BH))
2160		pr_info("bh_end_io(error=%d) for %c @%llu (%s/%llu/%d)\n",
 
2161		       iodone_w_error,
2162		       btrfsic_get_block_type(dev_state->state, block),
2163		       block->logical_bytenr, block->dev_state->name,
2164		       block->dev_bytenr, block->mirror_num);
 
 
2165
2166	block->iodone_w_error = iodone_w_error;
2167	if (block->submit_bio_bh_rw & REQ_PREFLUSH) {
2168		dev_state->last_flush_gen++;
2169		if ((dev_state->state->print_mask &
2170		     BTRFSIC_PRINT_MASK_END_IO_BIO_BH))
2171			pr_info("bh_end_io() new %s flush_gen=%llu\n",
2172			       dev_state->name, dev_state->last_flush_gen);
 
 
2173	}
2174	if (block->submit_bio_bh_rw & REQ_FUA)
2175		block->flush_gen = 0; /* FUA completed means block is on disk */
2176
2177	bh->b_private = block->orig_bio_bh_private;
2178	bh->b_end_io = block->orig_bio_bh_end_io.bh;
2179	block->is_iodone = 1; /* for FLUSH, this releases the block */
2180	bh->b_end_io(bh, uptodate);
2181}
2182
2183static int btrfsic_process_written_superblock(
2184		struct btrfsic_state *state,
2185		struct btrfsic_block *const superblock,
2186		struct btrfs_super_block *const super_hdr)
2187{
2188	struct btrfs_fs_info *fs_info = state->fs_info;
2189	int pass;
2190
2191	superblock->generation = btrfs_super_generation(super_hdr);
2192	if (!(superblock->generation > state->max_superblock_generation ||
2193	      0 == state->max_superblock_generation)) {
2194		if (state->print_mask & BTRFSIC_PRINT_MASK_SUPERBLOCK_WRITE)
2195			pr_info("btrfsic: superblock @%llu (%s/%llu/%d) with old gen %llu <= %llu\n",
2196			       superblock->logical_bytenr,
 
 
2197			       superblock->dev_state->name,
2198			       superblock->dev_bytenr, superblock->mirror_num,
 
 
2199			       btrfs_super_generation(super_hdr),
 
2200			       state->max_superblock_generation);
2201	} else {
2202		if (state->print_mask & BTRFSIC_PRINT_MASK_SUPERBLOCK_WRITE)
2203			pr_info("btrfsic: got new superblock @%llu (%s/%llu/%d) with new gen %llu > %llu\n",
2204			       superblock->logical_bytenr,
 
 
2205			       superblock->dev_state->name,
2206			       superblock->dev_bytenr, superblock->mirror_num,
 
 
2207			       btrfs_super_generation(super_hdr),
 
2208			       state->max_superblock_generation);
2209
2210		state->max_superblock_generation =
2211		    btrfs_super_generation(super_hdr);
2212		state->latest_superblock = superblock;
2213	}
2214
2215	for (pass = 0; pass < 3; pass++) {
2216		int ret;
2217		u64 next_bytenr;
2218		struct btrfsic_block *next_block;
2219		struct btrfsic_block_data_ctx tmp_next_block_ctx;
2220		struct btrfsic_block_link *l;
2221		int num_copies;
2222		int mirror_num;
2223		const char *additional_string = NULL;
2224		struct btrfs_disk_key tmp_disk_key = {0};
2225
2226		btrfs_set_disk_key_objectid(&tmp_disk_key,
2227					    BTRFS_ROOT_ITEM_KEY);
2228		btrfs_set_disk_key_objectid(&tmp_disk_key, 0);
2229
2230		switch (pass) {
2231		case 0:
2232			btrfs_set_disk_key_objectid(&tmp_disk_key,
2233						    BTRFS_ROOT_TREE_OBJECTID);
2234			additional_string = "root ";
2235			next_bytenr = btrfs_super_root(super_hdr);
2236			if (state->print_mask &
2237			    BTRFSIC_PRINT_MASK_ROOT_CHUNK_LOG_TREE_LOCATION)
2238				pr_info("root@%llu\n", next_bytenr);
 
2239			break;
2240		case 1:
2241			btrfs_set_disk_key_objectid(&tmp_disk_key,
2242						    BTRFS_CHUNK_TREE_OBJECTID);
2243			additional_string = "chunk ";
2244			next_bytenr = btrfs_super_chunk_root(super_hdr);
2245			if (state->print_mask &
2246			    BTRFSIC_PRINT_MASK_ROOT_CHUNK_LOG_TREE_LOCATION)
2247				pr_info("chunk@%llu\n", next_bytenr);
 
2248			break;
2249		case 2:
2250			btrfs_set_disk_key_objectid(&tmp_disk_key,
2251						    BTRFS_TREE_LOG_OBJECTID);
2252			additional_string = "log ";
2253			next_bytenr = btrfs_super_log_root(super_hdr);
2254			if (0 == next_bytenr)
2255				continue;
2256			if (state->print_mask &
2257			    BTRFSIC_PRINT_MASK_ROOT_CHUNK_LOG_TREE_LOCATION)
2258				pr_info("log@%llu\n", next_bytenr);
 
2259			break;
2260		}
2261
2262		num_copies = btrfs_num_copies(fs_info, next_bytenr,
2263					      BTRFS_SUPER_INFO_SIZE);
 
2264		if (state->print_mask & BTRFSIC_PRINT_MASK_NUM_COPIES)
2265			pr_info("num_copies(log_bytenr=%llu) = %d\n",
2266			       next_bytenr, num_copies);
2267		for (mirror_num = 1; mirror_num <= num_copies; mirror_num++) {
2268			int was_created;
2269
2270			if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
2271				pr_info("btrfsic_process_written_superblock(mirror_num=%d)\n", mirror_num);
 
 
2272			ret = btrfsic_map_block(state, next_bytenr,
2273						BTRFS_SUPER_INFO_SIZE,
2274						&tmp_next_block_ctx,
2275						mirror_num);
2276			if (ret) {
2277				pr_info("btrfsic: btrfsic_map_block(@%llu, mirror=%d) failed!\n",
2278				       next_bytenr, mirror_num);
 
 
 
2279				return -1;
2280			}
2281
2282			next_block = btrfsic_block_lookup_or_add(
2283					state,
2284					&tmp_next_block_ctx,
2285					additional_string,
2286					1, 0, 1,
2287					mirror_num,
2288					&was_created);
2289			if (NULL == next_block) {
2290				pr_info("btrfsic: error, kmalloc failed!\n");
 
2291				btrfsic_release_block_ctx(&tmp_next_block_ctx);
2292				return -1;
2293			}
2294
2295			next_block->disk_key = tmp_disk_key;
2296			if (was_created)
2297				next_block->generation =
2298				    BTRFSIC_GENERATION_UNKNOWN;
2299			l = btrfsic_block_link_lookup_or_add(
2300					state,
2301					&tmp_next_block_ctx,
2302					next_block,
2303					superblock,
2304					BTRFSIC_GENERATION_UNKNOWN);
2305			btrfsic_release_block_ctx(&tmp_next_block_ctx);
2306			if (NULL == l)
2307				return -1;
2308		}
2309	}
2310
2311	if (WARN_ON(-1 == btrfsic_check_all_ref_blocks(state, superblock, 0)))
 
2312		btrfsic_dump_tree(state);
 
2313
2314	return 0;
2315}
2316
2317static int btrfsic_check_all_ref_blocks(struct btrfsic_state *state,
2318					struct btrfsic_block *const block,
2319					int recursion_level)
2320{
2321	const struct btrfsic_block_link *l;
2322	int ret = 0;
2323
2324	if (recursion_level >= 3 + BTRFS_MAX_LEVEL) {
2325		/*
2326		 * Note that this situation can happen and does not
2327		 * indicate an error in regular cases. It happens
2328		 * when disk blocks are freed and later reused.
2329		 * The check-integrity module is not aware of any
2330		 * block free operations, it just recognizes block
2331		 * write operations. Therefore it keeps the linkage
2332		 * information for a block until a block is
2333		 * rewritten. This can temporarily cause incorrect
2334		 * and even circular linkage information. This
2335		 * causes no harm unless such blocks are referenced
2336		 * by the most recent super block.
2337		 */
2338		if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
2339			pr_info("btrfsic: abort cyclic linkage (case 1).\n");
 
2340
2341		return ret;
2342	}
2343
2344	/*
2345	 * This algorithm is recursive because the amount of used stack
2346	 * space is very small and the max recursion depth is limited.
2347	 */
2348	list_for_each_entry(l, &block->ref_to_list, node_ref_to) {
 
 
 
 
2349		if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
2350			pr_info("rl=%d, %c @%llu (%s/%llu/%d) %u* refers to %c @%llu (%s/%llu/%d)\n",
 
 
2351			       recursion_level,
2352			       btrfsic_get_block_type(state, block),
2353			       block->logical_bytenr, block->dev_state->name,
2354			       block->dev_bytenr, block->mirror_num,
 
 
2355			       l->ref_cnt,
2356			       btrfsic_get_block_type(state, l->block_ref_to),
 
2357			       l->block_ref_to->logical_bytenr,
2358			       l->block_ref_to->dev_state->name,
2359			       l->block_ref_to->dev_bytenr,
2360			       l->block_ref_to->mirror_num);
2361		if (l->block_ref_to->never_written) {
2362			pr_info("btrfs: attempt to write superblock which references block %c @%llu (%s/%llu/%d) which is never written!\n",
 
 
2363			       btrfsic_get_block_type(state, l->block_ref_to),
 
2364			       l->block_ref_to->logical_bytenr,
2365			       l->block_ref_to->dev_state->name,
2366			       l->block_ref_to->dev_bytenr,
2367			       l->block_ref_to->mirror_num);
2368			ret = -1;
2369		} else if (!l->block_ref_to->is_iodone) {
2370			pr_info("btrfs: attempt to write superblock which references block %c @%llu (%s/%llu/%d) which is not yet iodone!\n",
 
 
2371			       btrfsic_get_block_type(state, l->block_ref_to),
 
2372			       l->block_ref_to->logical_bytenr,
2373			       l->block_ref_to->dev_state->name,
2374			       l->block_ref_to->dev_bytenr,
2375			       l->block_ref_to->mirror_num);
2376			ret = -1;
2377		} else if (l->block_ref_to->iodone_w_error) {
2378			pr_info("btrfs: attempt to write superblock which references block %c @%llu (%s/%llu/%d) which has write error!\n",
2379			       btrfsic_get_block_type(state, l->block_ref_to),
2380			       l->block_ref_to->logical_bytenr,
2381			       l->block_ref_to->dev_state->name,
2382			       l->block_ref_to->dev_bytenr,
2383			       l->block_ref_to->mirror_num);
2384			ret = -1;
2385		} else if (l->parent_generation !=
2386			   l->block_ref_to->generation &&
2387			   BTRFSIC_GENERATION_UNKNOWN !=
2388			   l->parent_generation &&
2389			   BTRFSIC_GENERATION_UNKNOWN !=
2390			   l->block_ref_to->generation) {
2391			pr_info("btrfs: attempt to write superblock which references block %c @%llu (%s/%llu/%d) with generation %llu != parent generation %llu!\n",
 
 
 
2392			       btrfsic_get_block_type(state, l->block_ref_to),
 
2393			       l->block_ref_to->logical_bytenr,
2394			       l->block_ref_to->dev_state->name,
2395			       l->block_ref_to->dev_bytenr,
2396			       l->block_ref_to->mirror_num,
2397			       l->block_ref_to->generation,
2398			       l->parent_generation);
2399			ret = -1;
2400		} else if (l->block_ref_to->flush_gen >
2401			   l->block_ref_to->dev_state->last_flush_gen) {
2402			pr_info("btrfs: attempt to write superblock which references block %c @%llu (%s/%llu/%d) which is not flushed out of disk's write cache (block flush_gen=%llu, dev->flush_gen=%llu)!\n",
 
 
 
 
2403			       btrfsic_get_block_type(state, l->block_ref_to),
 
2404			       l->block_ref_to->logical_bytenr,
2405			       l->block_ref_to->dev_state->name,
2406			       l->block_ref_to->dev_bytenr,
2407			       l->block_ref_to->mirror_num, block->flush_gen,
 
 
2408			       l->block_ref_to->dev_state->last_flush_gen);
2409			ret = -1;
2410		} else if (-1 == btrfsic_check_all_ref_blocks(state,
2411							      l->block_ref_to,
2412							      recursion_level +
2413							      1)) {
2414			ret = -1;
2415		}
2416	}
2417
2418	return ret;
2419}
2420
2421static int btrfsic_is_block_ref_by_superblock(
2422		const struct btrfsic_state *state,
2423		const struct btrfsic_block *block,
2424		int recursion_level)
2425{
2426	const struct btrfsic_block_link *l;
2427
2428	if (recursion_level >= 3 + BTRFS_MAX_LEVEL) {
2429		/* refer to comment at "abort cyclic linkage (case 1)" */
2430		if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
2431			pr_info("btrfsic: abort cyclic linkage (case 2).\n");
 
2432
2433		return 0;
2434	}
2435
2436	/*
2437	 * This algorithm is recursive because the amount of used stack space
2438	 * is very small and the max recursion depth is limited.
2439	 */
2440	list_for_each_entry(l, &block->ref_from_list, node_ref_from) {
 
 
 
 
2441		if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
2442			pr_info("rl=%d, %c @%llu (%s/%llu/%d) is ref %u* from %c @%llu (%s/%llu/%d)\n",
 
 
2443			       recursion_level,
2444			       btrfsic_get_block_type(state, block),
2445			       block->logical_bytenr, block->dev_state->name,
2446			       block->dev_bytenr, block->mirror_num,
 
 
2447			       l->ref_cnt,
2448			       btrfsic_get_block_type(state, l->block_ref_from),
 
2449			       l->block_ref_from->logical_bytenr,
2450			       l->block_ref_from->dev_state->name,
 
2451			       l->block_ref_from->dev_bytenr,
2452			       l->block_ref_from->mirror_num);
2453		if (l->block_ref_from->is_superblock &&
2454		    state->latest_superblock->dev_bytenr ==
2455		    l->block_ref_from->dev_bytenr &&
2456		    state->latest_superblock->dev_state->bdev ==
2457		    l->block_ref_from->dev_state->bdev)
2458			return 1;
2459		else if (btrfsic_is_block_ref_by_superblock(state,
2460							    l->block_ref_from,
2461							    recursion_level +
2462							    1))
2463			return 1;
2464	}
2465
2466	return 0;
2467}
2468
2469static void btrfsic_print_add_link(const struct btrfsic_state *state,
2470				   const struct btrfsic_block_link *l)
2471{
2472	pr_info("Add %u* link from %c @%llu (%s/%llu/%d) to %c @%llu (%s/%llu/%d).\n",
 
 
2473	       l->ref_cnt,
2474	       btrfsic_get_block_type(state, l->block_ref_from),
2475	       l->block_ref_from->logical_bytenr,
2476	       l->block_ref_from->dev_state->name,
2477	       l->block_ref_from->dev_bytenr, l->block_ref_from->mirror_num,
 
2478	       btrfsic_get_block_type(state, l->block_ref_to),
2479	       l->block_ref_to->logical_bytenr,
2480	       l->block_ref_to->dev_state->name, l->block_ref_to->dev_bytenr,
 
2481	       l->block_ref_to->mirror_num);
2482}
2483
2484static void btrfsic_print_rem_link(const struct btrfsic_state *state,
2485				   const struct btrfsic_block_link *l)
2486{
2487	pr_info("Rem %u* link from %c @%llu (%s/%llu/%d) to %c @%llu (%s/%llu/%d).\n",
 
 
2488	       l->ref_cnt,
2489	       btrfsic_get_block_type(state, l->block_ref_from),
2490	       l->block_ref_from->logical_bytenr,
2491	       l->block_ref_from->dev_state->name,
2492	       l->block_ref_from->dev_bytenr, l->block_ref_from->mirror_num,
 
2493	       btrfsic_get_block_type(state, l->block_ref_to),
2494	       l->block_ref_to->logical_bytenr,
2495	       l->block_ref_to->dev_state->name, l->block_ref_to->dev_bytenr,
 
2496	       l->block_ref_to->mirror_num);
2497}
2498
2499static char btrfsic_get_block_type(const struct btrfsic_state *state,
2500				   const struct btrfsic_block *block)
2501{
2502	if (block->is_superblock &&
2503	    state->latest_superblock->dev_bytenr == block->dev_bytenr &&
2504	    state->latest_superblock->dev_state->bdev == block->dev_state->bdev)
2505		return 'S';
2506	else if (block->is_superblock)
2507		return 's';
2508	else if (block->is_metadata)
2509		return 'M';
2510	else
2511		return 'D';
2512}
2513
2514static void btrfsic_dump_tree(const struct btrfsic_state *state)
2515{
2516	btrfsic_dump_tree_sub(state, state->latest_superblock, 0);
2517}
2518
2519static void btrfsic_dump_tree_sub(const struct btrfsic_state *state,
2520				  const struct btrfsic_block *block,
2521				  int indent_level)
2522{
2523	const struct btrfsic_block_link *l;
2524	int indent_add;
2525	static char buf[80];
2526	int cursor_position;
2527
2528	/*
2529	 * Should better fill an on-stack buffer with a complete line and
2530	 * dump it at once when it is time to print a newline character.
2531	 */
2532
2533	/*
2534	 * This algorithm is recursive because the amount of used stack space
2535	 * is very small and the max recursion depth is limited.
2536	 */
2537	indent_add = sprintf(buf, "%c-%llu(%s/%llu/%u)",
2538			     btrfsic_get_block_type(state, block),
2539			     block->logical_bytenr, block->dev_state->name,
2540			     block->dev_bytenr, block->mirror_num);
 
 
2541	if (indent_level + indent_add > BTRFSIC_TREE_DUMP_MAX_INDENT_LEVEL) {
2542		printk("[...]\n");
2543		return;
2544	}
2545	printk(buf);
2546	indent_level += indent_add;
2547	if (list_empty(&block->ref_to_list)) {
2548		printk("\n");
2549		return;
2550	}
2551	if (block->mirror_num > 1 &&
2552	    !(state->print_mask & BTRFSIC_PRINT_MASK_TREE_WITH_ALL_MIRRORS)) {
2553		printk(" [...]\n");
2554		return;
2555	}
2556
2557	cursor_position = indent_level;
2558	list_for_each_entry(l, &block->ref_to_list, node_ref_to) {
 
 
 
 
2559		while (cursor_position < indent_level) {
2560			printk(" ");
2561			cursor_position++;
2562		}
2563		if (l->ref_cnt > 1)
2564			indent_add = sprintf(buf, " %d*--> ", l->ref_cnt);
2565		else
2566			indent_add = sprintf(buf, " --> ");
2567		if (indent_level + indent_add >
2568		    BTRFSIC_TREE_DUMP_MAX_INDENT_LEVEL) {
2569			printk("[...]\n");
2570			cursor_position = 0;
2571			continue;
2572		}
2573
2574		printk(buf);
2575
2576		btrfsic_dump_tree_sub(state, l->block_ref_to,
2577				      indent_level + indent_add);
2578		cursor_position = 0;
2579	}
2580}
2581
2582static struct btrfsic_block_link *btrfsic_block_link_lookup_or_add(
2583		struct btrfsic_state *state,
2584		struct btrfsic_block_data_ctx *next_block_ctx,
2585		struct btrfsic_block *next_block,
2586		struct btrfsic_block *from_block,
2587		u64 parent_generation)
2588{
2589	struct btrfsic_block_link *l;
2590
2591	l = btrfsic_block_link_hashtable_lookup(next_block_ctx->dev->bdev,
2592						next_block_ctx->dev_bytenr,
2593						from_block->dev_state->bdev,
2594						from_block->dev_bytenr,
2595						&state->block_link_hashtable);
2596	if (NULL == l) {
2597		l = btrfsic_block_link_alloc();
2598		if (NULL == l) {
2599			pr_info("btrfsic: error, kmalloc failed!\n");
 
2600			return NULL;
2601		}
2602
2603		l->block_ref_to = next_block;
2604		l->block_ref_from = from_block;
2605		l->ref_cnt = 1;
2606		l->parent_generation = parent_generation;
2607
2608		if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
2609			btrfsic_print_add_link(state, l);
2610
2611		list_add(&l->node_ref_to, &from_block->ref_to_list);
2612		list_add(&l->node_ref_from, &next_block->ref_from_list);
2613
2614		btrfsic_block_link_hashtable_add(l,
2615						 &state->block_link_hashtable);
2616	} else {
2617		l->ref_cnt++;
2618		l->parent_generation = parent_generation;
2619		if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
2620			btrfsic_print_add_link(state, l);
2621	}
2622
2623	return l;
2624}
2625
2626static struct btrfsic_block *btrfsic_block_lookup_or_add(
2627		struct btrfsic_state *state,
2628		struct btrfsic_block_data_ctx *block_ctx,
2629		const char *additional_string,
2630		int is_metadata,
2631		int is_iodone,
2632		int never_written,
2633		int mirror_num,
2634		int *was_created)
2635{
2636	struct btrfsic_block *block;
2637
2638	block = btrfsic_block_hashtable_lookup(block_ctx->dev->bdev,
2639					       block_ctx->dev_bytenr,
2640					       &state->block_hashtable);
2641	if (NULL == block) {
2642		struct btrfsic_dev_state *dev_state;
2643
2644		block = btrfsic_block_alloc();
2645		if (NULL == block) {
2646			pr_info("btrfsic: error, kmalloc failed!\n");
2647			return NULL;
2648		}
2649		dev_state = btrfsic_dev_state_lookup(block_ctx->dev->bdev->bd_dev);
2650		if (NULL == dev_state) {
2651			pr_info("btrfsic: error, lookup dev_state failed!\n");
 
2652			btrfsic_block_free(block);
2653			return NULL;
2654		}
2655		block->dev_state = dev_state;
2656		block->dev_bytenr = block_ctx->dev_bytenr;
2657		block->logical_bytenr = block_ctx->start;
2658		block->is_metadata = is_metadata;
2659		block->is_iodone = is_iodone;
2660		block->never_written = never_written;
2661		block->mirror_num = mirror_num;
2662		if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
2663			pr_info("New %s%c-block @%llu (%s/%llu/%d)\n",
 
2664			       additional_string,
2665			       btrfsic_get_block_type(state, block),
2666			       block->logical_bytenr, dev_state->name,
2667			       block->dev_bytenr, mirror_num);
 
 
2668		list_add(&block->all_blocks_node, &state->all_blocks_list);
2669		btrfsic_block_hashtable_add(block, &state->block_hashtable);
2670		if (NULL != was_created)
2671			*was_created = 1;
2672	} else {
2673		if (NULL != was_created)
2674			*was_created = 0;
2675	}
2676
2677	return block;
2678}
2679
2680static void btrfsic_cmp_log_and_dev_bytenr(struct btrfsic_state *state,
2681					   u64 bytenr,
2682					   struct btrfsic_dev_state *dev_state,
2683					   u64 dev_bytenr)
2684{
2685	struct btrfs_fs_info *fs_info = state->fs_info;
2686	struct btrfsic_block_data_ctx block_ctx;
2687	int num_copies;
2688	int mirror_num;
2689	int match = 0;
2690	int ret;
 
 
2691
2692	num_copies = btrfs_num_copies(fs_info, bytenr, state->metablock_size);
 
2693
2694	for (mirror_num = 1; mirror_num <= num_copies; mirror_num++) {
2695		ret = btrfsic_map_block(state, bytenr, state->metablock_size,
2696					&block_ctx, mirror_num);
2697		if (ret) {
2698			pr_info("btrfsic: btrfsic_map_block(logical @%llu, mirror %d) failed!\n",
2699			       bytenr, mirror_num);
 
 
2700			continue;
2701		}
2702
2703		if (dev_state->bdev == block_ctx.dev->bdev &&
2704		    dev_bytenr == block_ctx.dev_bytenr) {
2705			match++;
2706			btrfsic_release_block_ctx(&block_ctx);
2707			break;
2708		}
2709		btrfsic_release_block_ctx(&block_ctx);
2710	}
2711
2712	if (WARN_ON(!match)) {
2713		pr_info("btrfs: attempt to write M-block which contains logical bytenr that doesn't map to dev+physical bytenr of submit_bio, buffer->log_bytenr=%llu, submit_bio(bdev=%s, phys_bytenr=%llu)!\n",
2714		       bytenr, dev_state->name, dev_bytenr);
 
 
 
2715		for (mirror_num = 1; mirror_num <= num_copies; mirror_num++) {
2716			ret = btrfsic_map_block(state, bytenr,
2717						state->metablock_size,
2718						&block_ctx, mirror_num);
2719			if (ret)
2720				continue;
2721
2722			pr_info("Read logical bytenr @%llu maps to (%s/%llu/%d)\n",
2723			       bytenr, block_ctx.dev->name,
2724			       block_ctx.dev_bytenr, mirror_num);
 
 
 
2725		}
 
2726	}
2727}
2728
2729static struct btrfsic_dev_state *btrfsic_dev_state_lookup(dev_t dev)
 
2730{
2731	return btrfsic_dev_state_hashtable_lookup(dev,
2732						  &btrfsic_dev_state_hashtable);
 
 
 
2733}
2734
2735int btrfsic_submit_bh(int op, int op_flags, struct buffer_head *bh)
2736{
2737	struct btrfsic_dev_state *dev_state;
2738
2739	if (!btrfsic_is_initialized)
2740		return submit_bh(op, op_flags, bh);
2741
2742	mutex_lock(&btrfsic_mutex);
2743	/* since btrfsic_submit_bh() might also be called before
2744	 * btrfsic_mount(), this might return NULL */
2745	dev_state = btrfsic_dev_state_lookup(bh->b_bdev->bd_dev);
2746
2747	/* Only called to write the superblock (incl. FLUSH/FUA) */
2748	if (NULL != dev_state &&
2749	    (op == REQ_OP_WRITE) && bh->b_size > 0) {
2750		u64 dev_bytenr;
2751
2752		dev_bytenr = BTRFS_BDEV_BLOCKSIZE * bh->b_blocknr;
2753		if (dev_state->state->print_mask &
2754		    BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH)
2755			pr_info("submit_bh(op=0x%x,0x%x, blocknr=%llu (bytenr %llu), size=%zu, data=%p, bdev=%p)\n",
2756			       op, op_flags, (unsigned long long)bh->b_blocknr,
2757			       dev_bytenr, bh->b_size, bh->b_data, bh->b_bdev);
 
 
 
 
2758		btrfsic_process_written_block(dev_state, dev_bytenr,
2759					      &bh->b_data, 1, NULL,
2760					      NULL, bh, op_flags);
2761	} else if (NULL != dev_state && (op_flags & REQ_PREFLUSH)) {
2762		if (dev_state->state->print_mask &
2763		    BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH)
2764			pr_info("submit_bh(op=0x%x,0x%x FLUSH, bdev=%p)\n",
2765			       op, op_flags, bh->b_bdev);
 
2766		if (!dev_state->dummy_block_for_bio_bh_flush.is_iodone) {
2767			if ((dev_state->state->print_mask &
2768			     (BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH |
2769			      BTRFSIC_PRINT_MASK_VERBOSE)))
2770				pr_info("btrfsic_submit_bh(%s) with FLUSH but dummy block already in use (ignored)!\n",
 
 
 
2771				       dev_state->name);
2772		} else {
2773			struct btrfsic_block *const block =
2774				&dev_state->dummy_block_for_bio_bh_flush;
2775
2776			block->is_iodone = 0;
2777			block->never_written = 0;
2778			block->iodone_w_error = 0;
2779			block->flush_gen = dev_state->last_flush_gen + 1;
2780			block->submit_bio_bh_rw = op_flags;
2781			block->orig_bio_bh_private = bh->b_private;
2782			block->orig_bio_bh_end_io.bh = bh->b_end_io;
2783			block->next_in_same_bio = NULL;
2784			bh->b_private = block;
2785			bh->b_end_io = btrfsic_bh_end_io;
2786		}
2787	}
2788	mutex_unlock(&btrfsic_mutex);
2789	return submit_bh(op, op_flags, bh);
2790}
2791
2792static void __btrfsic_submit_bio(struct bio *bio)
2793{
2794	struct btrfsic_dev_state *dev_state;
2795
2796	if (!btrfsic_is_initialized)
 
2797		return;
 
2798
2799	mutex_lock(&btrfsic_mutex);
2800	/* since btrfsic_submit_bio() is also called before
2801	 * btrfsic_mount(), this might return NULL */
2802	dev_state = btrfsic_dev_state_lookup(bio_dev(bio) + bio->bi_partno);
2803	if (NULL != dev_state &&
2804	    (bio_op(bio) == REQ_OP_WRITE) && bio_has_data(bio)) {
2805		unsigned int i = 0;
2806		u64 dev_bytenr;
2807		u64 cur_bytenr;
2808		struct bio_vec bvec;
2809		struct bvec_iter iter;
2810		int bio_is_patched;
2811		char **mapped_datav;
2812		unsigned int segs = bio_segments(bio);
2813
2814		dev_bytenr = 512 * bio->bi_iter.bi_sector;
2815		bio_is_patched = 0;
2816		if (dev_state->state->print_mask &
2817		    BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH)
2818			pr_info("submit_bio(rw=%d,0x%x, bi_vcnt=%u, bi_sector=%llu (bytenr %llu), bi_disk=%p)\n",
2819			       bio_op(bio), bio->bi_opf, segs,
2820			       (unsigned long long)bio->bi_iter.bi_sector,
2821			       dev_bytenr, bio->bi_disk);
 
 
2822
2823		mapped_datav = kmalloc_array(segs,
2824					     sizeof(*mapped_datav), GFP_NOFS);
2825		if (!mapped_datav)
2826			goto leave;
2827		cur_bytenr = dev_bytenr;
2828
2829		bio_for_each_segment(bvec, bio, iter) {
2830			BUG_ON(bvec.bv_len != PAGE_SIZE);
2831			mapped_datav[i] = kmap(bvec.bv_page);
2832			i++;
2833
2834			if (dev_state->state->print_mask &
2835			    BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH_VERBOSE)
2836				pr_info("#%u: bytenr=%llu, len=%u, offset=%u\n",
2837				       i, cur_bytenr, bvec.bv_len, bvec.bv_offset);
2838			cur_bytenr += bvec.bv_len;
 
 
 
 
 
 
 
 
 
2839		}
2840		btrfsic_process_written_block(dev_state, dev_bytenr,
2841					      mapped_datav, segs,
2842					      bio, &bio_is_patched,
2843					      NULL, bio->bi_opf);
2844		bio_for_each_segment(bvec, bio, iter)
2845			kunmap(bvec.bv_page);
 
 
2846		kfree(mapped_datav);
2847	} else if (NULL != dev_state && (bio->bi_opf & REQ_PREFLUSH)) {
2848		if (dev_state->state->print_mask &
2849		    BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH)
2850			pr_info("submit_bio(rw=%d,0x%x FLUSH, disk=%p)\n",
2851			       bio_op(bio), bio->bi_opf, bio->bi_disk);
 
2852		if (!dev_state->dummy_block_for_bio_bh_flush.is_iodone) {
2853			if ((dev_state->state->print_mask &
2854			     (BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH |
2855			      BTRFSIC_PRINT_MASK_VERBOSE)))
2856				pr_info("btrfsic_submit_bio(%s) with FLUSH but dummy block already in use (ignored)!\n",
 
 
 
2857				       dev_state->name);
2858		} else {
2859			struct btrfsic_block *const block =
2860				&dev_state->dummy_block_for_bio_bh_flush;
2861
2862			block->is_iodone = 0;
2863			block->never_written = 0;
2864			block->iodone_w_error = 0;
2865			block->flush_gen = dev_state->last_flush_gen + 1;
2866			block->submit_bio_bh_rw = bio->bi_opf;
2867			block->orig_bio_bh_private = bio->bi_private;
2868			block->orig_bio_bh_end_io.bio = bio->bi_end_io;
2869			block->next_in_same_bio = NULL;
2870			bio->bi_private = block;
2871			bio->bi_end_io = btrfsic_bio_end_io;
2872		}
2873	}
2874leave:
2875	mutex_unlock(&btrfsic_mutex);
2876}
2877
2878void btrfsic_submit_bio(struct bio *bio)
2879{
2880	__btrfsic_submit_bio(bio);
2881	submit_bio(bio);
2882}
2883
2884int btrfsic_submit_bio_wait(struct bio *bio)
2885{
2886	__btrfsic_submit_bio(bio);
2887	return submit_bio_wait(bio);
2888}
2889
2890int btrfsic_mount(struct btrfs_fs_info *fs_info,
2891		  struct btrfs_fs_devices *fs_devices,
2892		  int including_extent_data, u32 print_mask)
2893{
2894	int ret;
2895	struct btrfsic_state *state;
2896	struct list_head *dev_head = &fs_devices->devices;
2897	struct btrfs_device *device;
2898
2899	if (!PAGE_ALIGNED(fs_info->nodesize)) {
2900		pr_info("btrfsic: cannot handle nodesize %d not being a multiple of PAGE_SIZE %ld!\n",
2901		       fs_info->nodesize, PAGE_SIZE);
 
2902		return -1;
2903	}
2904	if (!PAGE_ALIGNED(fs_info->sectorsize)) {
2905		pr_info("btrfsic: cannot handle sectorsize %d not being a multiple of PAGE_SIZE %ld!\n",
2906		       fs_info->sectorsize, PAGE_SIZE);
 
2907		return -1;
2908	}
2909	state = kvzalloc(sizeof(*state), GFP_KERNEL);
2910	if (!state) {
2911		pr_info("btrfs check-integrity: allocation failed!\n");
2912		return -ENOMEM;
 
 
 
 
 
 
 
 
 
 
 
 
2913	}
2914
2915	if (!btrfsic_is_initialized) {
2916		mutex_init(&btrfsic_mutex);
2917		btrfsic_dev_state_hashtable_init(&btrfsic_dev_state_hashtable);
2918		btrfsic_is_initialized = 1;
2919	}
2920	mutex_lock(&btrfsic_mutex);
2921	state->fs_info = fs_info;
2922	state->print_mask = print_mask;
2923	state->include_extent_data = including_extent_data;
2924	state->csum_size = 0;
2925	state->metablock_size = fs_info->nodesize;
2926	state->datablock_size = fs_info->sectorsize;
2927	INIT_LIST_HEAD(&state->all_blocks_list);
2928	btrfsic_block_hashtable_init(&state->block_hashtable);
2929	btrfsic_block_link_hashtable_init(&state->block_link_hashtable);
2930	state->max_superblock_generation = 0;
2931	state->latest_superblock = NULL;
2932
2933	list_for_each_entry(device, dev_head, dev_list) {
2934		struct btrfsic_dev_state *ds;
2935		const char *p;
2936
2937		if (!device->bdev || !device->name)
2938			continue;
2939
2940		ds = btrfsic_dev_state_alloc();
2941		if (NULL == ds) {
2942			pr_info("btrfs check-integrity: kmalloc() failed!\n");
 
2943			mutex_unlock(&btrfsic_mutex);
2944			return -ENOMEM;
2945		}
2946		ds->bdev = device->bdev;
2947		ds->state = state;
2948		bdevname(ds->bdev, ds->name);
2949		ds->name[BDEVNAME_SIZE - 1] = '\0';
2950		p = kbasename(ds->name);
 
 
 
 
2951		strlcpy(ds->name, p, sizeof(ds->name));
2952		btrfsic_dev_state_hashtable_add(ds,
2953						&btrfsic_dev_state_hashtable);
2954	}
2955
2956	ret = btrfsic_process_superblock(state, fs_devices);
2957	if (0 != ret) {
2958		mutex_unlock(&btrfsic_mutex);
2959		btrfsic_unmount(fs_devices);
2960		return ret;
2961	}
2962
2963	if (state->print_mask & BTRFSIC_PRINT_MASK_INITIAL_DATABASE)
2964		btrfsic_dump_database(state);
2965	if (state->print_mask & BTRFSIC_PRINT_MASK_INITIAL_TREE)
2966		btrfsic_dump_tree(state);
2967
2968	mutex_unlock(&btrfsic_mutex);
2969	return 0;
2970}
2971
2972void btrfsic_unmount(struct btrfs_fs_devices *fs_devices)
 
2973{
2974	struct btrfsic_block *b_all, *tmp_all;
 
2975	struct btrfsic_state *state;
2976	struct list_head *dev_head = &fs_devices->devices;
2977	struct btrfs_device *device;
2978
2979	if (!btrfsic_is_initialized)
2980		return;
2981
2982	mutex_lock(&btrfsic_mutex);
2983
2984	state = NULL;
2985	list_for_each_entry(device, dev_head, dev_list) {
2986		struct btrfsic_dev_state *ds;
2987
2988		if (!device->bdev || !device->name)
2989			continue;
2990
2991		ds = btrfsic_dev_state_hashtable_lookup(
2992				device->bdev->bd_dev,
2993				&btrfsic_dev_state_hashtable);
2994		if (NULL != ds) {
2995			state = ds->state;
2996			btrfsic_dev_state_hashtable_remove(ds);
2997			btrfsic_dev_state_free(ds);
2998		}
2999	}
3000
3001	if (NULL == state) {
3002		pr_info("btrfsic: error, cannot find state information on umount!\n");
 
 
3003		mutex_unlock(&btrfsic_mutex);
3004		return;
3005	}
3006
3007	/*
3008	 * Don't care about keeping the lists' state up to date,
3009	 * just free all memory that was allocated dynamically.
3010	 * Free the blocks and the block_links.
3011	 */
3012	list_for_each_entry_safe(b_all, tmp_all, &state->all_blocks_list,
3013				 all_blocks_node) {
3014		struct btrfsic_block_link *l, *tmp;
 
 
 
 
 
 
 
 
 
 
3015
3016		list_for_each_entry_safe(l, tmp, &b_all->ref_to_list,
3017					 node_ref_to) {
3018			if (state->print_mask & BTRFSIC_PRINT_MASK_VERBOSE)
3019				btrfsic_print_rem_link(state, l);
3020
3021			l->ref_cnt--;
3022			if (0 == l->ref_cnt)
3023				btrfsic_block_link_free(l);
3024		}
3025
3026		if (b_all->is_iodone || b_all->never_written)
3027			btrfsic_block_free(b_all);
3028		else
3029			pr_info("btrfs: attempt to free %c-block @%llu (%s/%llu/%d) on umount which is not yet iodone!\n",
 
 
3030			       btrfsic_get_block_type(state, b_all),
3031			       b_all->logical_bytenr, b_all->dev_state->name,
3032			       b_all->dev_bytenr, b_all->mirror_num);
 
 
3033	}
3034
3035	mutex_unlock(&btrfsic_mutex);
3036
3037	kvfree(state);
3038}