1/*
   2 * Copyright (C) 2012 Red Hat, Inc.
   3 *
   4 * This file is released under the GPL.
   5 */
   6
   7#include "dm-cache-metadata.h"
   8
   9#include "persistent-data/dm-array.h"
  10#include "persistent-data/dm-bitset.h"
  11#include "persistent-data/dm-space-map.h"
  12#include "persistent-data/dm-space-map-disk.h"
  13#include "persistent-data/dm-transaction-manager.h"
  14
  15#include <linux/device-mapper.h>
  16#include <linux/refcount.h>
  17
  18/*----------------------------------------------------------------*/
  19
  20#define DM_MSG_PREFIX   "cache metadata"
  21
  22#define CACHE_SUPERBLOCK_MAGIC 06142003
  23#define CACHE_SUPERBLOCK_LOCATION 0
  24
  25/*
  26 * defines a range of metadata versions that this module can handle.
  27 */
  28#define MIN_CACHE_VERSION 1
  29#define MAX_CACHE_VERSION 2
  30
  31/*
  32 *  3 for btree insert +
  33 *  2 for btree lookup used within space map
  34 */
  35#define CACHE_MAX_CONCURRENT_LOCKS 5
  36#define SPACE_MAP_ROOT_SIZE 128
  37
  38enum superblock_flag_bits {
  39	/* for spotting crashes that would invalidate the dirty bitset */
  40	CLEAN_SHUTDOWN,
  41	/* metadata must be checked using the tools */
  42	NEEDS_CHECK,
  43};
  44
  45/*
  46 * Each mapping from cache block -> origin block carries a set of flags.
  47 */
  48enum mapping_bits {
  49	/*
  50	 * A valid mapping.  Because we're using an array we clear this
  51	 * flag for an non existant mapping.
  52	 */
  53	M_VALID = 1,
  54
  55	/*
  56	 * The data on the cache is different from that on the origin.
  57	 * This flag is only used by metadata format 1.
  58	 */
  59	M_DIRTY = 2
  60};
  61
  62struct cache_disk_superblock {
  63	__le32 csum;
  64	__le32 flags;
  65	__le64 blocknr;
  66
  67	__u8 uuid[16];
  68	__le64 magic;
  69	__le32 version;
  70
  71	__u8 policy_name[CACHE_POLICY_NAME_SIZE];
  72	__le32 policy_hint_size;
  73
  74	__u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
  75	__le64 mapping_root;
  76	__le64 hint_root;
  77
  78	__le64 discard_root;
  79	__le64 discard_block_size;
  80	__le64 discard_nr_blocks;
  81
  82	__le32 data_block_size;
  83	__le32 metadata_block_size;
  84	__le32 cache_blocks;
  85
  86	__le32 compat_flags;
  87	__le32 compat_ro_flags;
  88	__le32 incompat_flags;
  89
  90	__le32 read_hits;
  91	__le32 read_misses;
  92	__le32 write_hits;
  93	__le32 write_misses;
  94
  95	__le32 policy_version[CACHE_POLICY_VERSION_SIZE];
  96
  97	/*
  98	 * Metadata format 2 fields.
  99	 */
 100	__le64 dirty_root;
 101} __packed;
 102
 103struct dm_cache_metadata {
 104	refcount_t ref_count;
 105	struct list_head list;
 106
 107	unsigned version;
 108	struct block_device *bdev;
 109	struct dm_block_manager *bm;
 110	struct dm_space_map *metadata_sm;
 111	struct dm_transaction_manager *tm;
 112
 113	struct dm_array_info info;
 114	struct dm_array_info hint_info;
 115	struct dm_disk_bitset discard_info;
 116
 117	struct rw_semaphore root_lock;
 118	unsigned long flags;
 119	dm_block_t root;
 120	dm_block_t hint_root;
 121	dm_block_t discard_root;
 122
 123	sector_t discard_block_size;
 124	dm_dblock_t discard_nr_blocks;
 125
 126	sector_t data_block_size;
 127	dm_cblock_t cache_blocks;
 128	bool changed:1;
 129	bool clean_when_opened:1;
 130
 131	char policy_name[CACHE_POLICY_NAME_SIZE];
 132	unsigned policy_version[CACHE_POLICY_VERSION_SIZE];
 133	size_t policy_hint_size;
 134	struct dm_cache_statistics stats;
 135
 136	/*
 137	 * Reading the space map root can fail, so we read it into this
 138	 * buffer before the superblock is locked and updated.
 139	 */
 140	__u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
 141
 142	/*
 143	 * Set if a transaction has to be aborted but the attempt to roll
 144	 * back to the previous (good) transaction failed.  The only
 145	 * metadata operation permissible in this state is the closing of
 146	 * the device.
 147	 */
 148	bool fail_io:1;
 149
 150	/*
 151	 * Metadata format 2 fields.
 152	 */
 153	dm_block_t dirty_root;
 154	struct dm_disk_bitset dirty_info;
 155
 156	/*
 157	 * These structures are used when loading metadata.  They're too
 158	 * big to put on the stack.
 159	 */
 160	struct dm_array_cursor mapping_cursor;
 161	struct dm_array_cursor hint_cursor;
 162	struct dm_bitset_cursor dirty_cursor;
 163};
 164
 165/*-------------------------------------------------------------------
 166 * superblock validator
 167 *-----------------------------------------------------------------*/
 168
 169#define SUPERBLOCK_CSUM_XOR 9031977
 170
 171static void sb_prepare_for_write(struct dm_block_validator *v,
 172				 struct dm_block *b,
 173				 size_t sb_block_size)
 174{
 175	struct cache_disk_superblock *disk_super = dm_block_data(b);
 176
 177	disk_super->blocknr = cpu_to_le64(dm_block_location(b));
 178	disk_super->csum = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
 179						      sb_block_size - sizeof(__le32),
 180						      SUPERBLOCK_CSUM_XOR));
 181}
 182
 183static int check_metadata_version(struct cache_disk_superblock *disk_super)
 184{
 185	uint32_t metadata_version = le32_to_cpu(disk_super->version);
 186
 187	if (metadata_version < MIN_CACHE_VERSION || metadata_version > MAX_CACHE_VERSION) {
 188		DMERR("Cache metadata version %u found, but only versions between %u and %u supported.",
 189		      metadata_version, MIN_CACHE_VERSION, MAX_CACHE_VERSION);
 190		return -EINVAL;
 191	}
 192
 193	return 0;
 194}
 195
 196static int sb_check(struct dm_block_validator *v,
 197		    struct dm_block *b,
 198		    size_t sb_block_size)
 199{
 200	struct cache_disk_superblock *disk_super = dm_block_data(b);
 201	__le32 csum_le;
 202
 203	if (dm_block_location(b) != le64_to_cpu(disk_super->blocknr)) {
 204		DMERR("sb_check failed: blocknr %llu: wanted %llu",
 205		      le64_to_cpu(disk_super->blocknr),
 206		      (unsigned long long)dm_block_location(b));
 207		return -ENOTBLK;
 208	}
 209
 210	if (le64_to_cpu(disk_super->magic) != CACHE_SUPERBLOCK_MAGIC) {
 211		DMERR("sb_check failed: magic %llu: wanted %llu",
 212		      le64_to_cpu(disk_super->magic),
 213		      (unsigned long long)CACHE_SUPERBLOCK_MAGIC);
 214		return -EILSEQ;
 215	}
 216
 217	csum_le = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
 218					     sb_block_size - sizeof(__le32),
 219					     SUPERBLOCK_CSUM_XOR));
 220	if (csum_le != disk_super->csum) {
 221		DMERR("sb_check failed: csum %u: wanted %u",
 222		      le32_to_cpu(csum_le), le32_to_cpu(disk_super->csum));
 223		return -EILSEQ;
 224	}
 225
 226	return check_metadata_version(disk_super);
 227}
 228
 229static struct dm_block_validator sb_validator = {
 230	.name = "superblock",
 231	.prepare_for_write = sb_prepare_for_write,
 232	.check = sb_check
 233};
 234
 235/*----------------------------------------------------------------*/
 236
 237static int superblock_read_lock(struct dm_cache_metadata *cmd,
 238				struct dm_block **sblock)
 239{
 240	return dm_bm_read_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 241			       &sb_validator, sblock);
 242}
 243
 244static int superblock_lock_zero(struct dm_cache_metadata *cmd,
 245				struct dm_block **sblock)
 246{
 247	return dm_bm_write_lock_zero(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 248				     &sb_validator, sblock);
 249}
 250
 251static int superblock_lock(struct dm_cache_metadata *cmd,
 252			   struct dm_block **sblock)
 253{
 254	return dm_bm_write_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 255				&sb_validator, sblock);
 256}
 257
 258/*----------------------------------------------------------------*/
 259
 260static int __superblock_all_zeroes(struct dm_block_manager *bm, bool *result)
 261{
 262	int r;
 263	unsigned i;
 264	struct dm_block *b;
 265	__le64 *data_le, zero = cpu_to_le64(0);
 266	unsigned sb_block_size = dm_bm_block_size(bm) / sizeof(__le64);
 267
 268	/*
 269	 * We can't use a validator here - it may be all zeroes.
 270	 */
 271	r = dm_bm_read_lock(bm, CACHE_SUPERBLOCK_LOCATION, NULL, &b);
 272	if (r)
 273		return r;
 274
 275	data_le = dm_block_data(b);
 276	*result = true;
 277	for (i = 0; i < sb_block_size; i++) {
 278		if (data_le[i] != zero) {
 279			*result = false;
 280			break;
 281		}
 282	}
 283
 284	dm_bm_unlock(b);
 285
 286	return 0;
 287}
 288
 289static void __setup_mapping_info(struct dm_cache_metadata *cmd)
 290{
 291	struct dm_btree_value_type vt;
 292
 293	vt.context = NULL;
 294	vt.size = sizeof(__le64);
 295	vt.inc = NULL;
 296	vt.dec = NULL;
 297	vt.equal = NULL;
 298	dm_array_info_init(&cmd->info, cmd->tm, &vt);
 299
 300	if (cmd->policy_hint_size) {
 301		vt.size = sizeof(__le32);
 302		dm_array_info_init(&cmd->hint_info, cmd->tm, &vt);
 303	}
 304}
 305
 306static int __save_sm_root(struct dm_cache_metadata *cmd)
 307{
 308	int r;
 309	size_t metadata_len;
 310
 311	r = dm_sm_root_size(cmd->metadata_sm, &metadata_len);
 312	if (r < 0)
 313		return r;
 314
 315	return dm_sm_copy_root(cmd->metadata_sm, &cmd->metadata_space_map_root,
 316			       metadata_len);
 317}
 318
 319static void __copy_sm_root(struct dm_cache_metadata *cmd,
 320			   struct cache_disk_superblock *disk_super)
 321{
 322	memcpy(&disk_super->metadata_space_map_root,
 323	       &cmd->metadata_space_map_root,
 324	       sizeof(cmd->metadata_space_map_root));
 325}
 326
 327static bool separate_dirty_bits(struct dm_cache_metadata *cmd)
 328{
 329	return cmd->version >= 2;
 330}
 331
 332static int __write_initial_superblock(struct dm_cache_metadata *cmd)
 333{
 334	int r;
 335	struct dm_block *sblock;
 336	struct cache_disk_superblock *disk_super;
 337	sector_t bdev_size = bdev_nr_sectors(cmd->bdev);
 338
 339	/* FIXME: see if we can lose the max sectors limit */
 340	if (bdev_size > DM_CACHE_METADATA_MAX_SECTORS)
 341		bdev_size = DM_CACHE_METADATA_MAX_SECTORS;
 342
 343	r = dm_tm_pre_commit(cmd->tm);
 344	if (r < 0)
 345		return r;
 346
 347	/*
 348	 * dm_sm_copy_root() can fail.  So we need to do it before we start
 349	 * updating the superblock.
 350	 */
 351	r = __save_sm_root(cmd);
 352	if (r)
 353		return r;
 354
 355	r = superblock_lock_zero(cmd, &sblock);
 356	if (r)
 357		return r;
 358
 359	disk_super = dm_block_data(sblock);
 360	disk_super->flags = 0;
 361	memset(disk_super->uuid, 0, sizeof(disk_super->uuid));
 362	disk_super->magic = cpu_to_le64(CACHE_SUPERBLOCK_MAGIC);
 363	disk_super->version = cpu_to_le32(cmd->version);
 364	memset(disk_super->policy_name, 0, sizeof(disk_super->policy_name));
 365	memset(disk_super->policy_version, 0, sizeof(disk_super->policy_version));
 366	disk_super->policy_hint_size = cpu_to_le32(0);
 367
 368	__copy_sm_root(cmd, disk_super);
 369
 370	disk_super->mapping_root = cpu_to_le64(cmd->root);
 371	disk_super->hint_root = cpu_to_le64(cmd->hint_root);
 372	disk_super->discard_root = cpu_to_le64(cmd->discard_root);
 373	disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
 374	disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
 375	disk_super->metadata_block_size = cpu_to_le32(DM_CACHE_METADATA_BLOCK_SIZE);
 376	disk_super->data_block_size = cpu_to_le32(cmd->data_block_size);
 377	disk_super->cache_blocks = cpu_to_le32(0);
 378
 379	disk_super->read_hits = cpu_to_le32(0);
 380	disk_super->read_misses = cpu_to_le32(0);
 381	disk_super->write_hits = cpu_to_le32(0);
 382	disk_super->write_misses = cpu_to_le32(0);
 383
 384	if (separate_dirty_bits(cmd))
 385		disk_super->dirty_root = cpu_to_le64(cmd->dirty_root);
 386
 387	return dm_tm_commit(cmd->tm, sblock);
 388}
 389
 390static int __format_metadata(struct dm_cache_metadata *cmd)
 391{
 392	int r;
 393
 394	r = dm_tm_create_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 395				 &cmd->tm, &cmd->metadata_sm);
 396	if (r < 0) {
 397		DMERR("tm_create_with_sm failed");
 398		return r;
 399	}
 400
 401	__setup_mapping_info(cmd);
 402
 403	r = dm_array_empty(&cmd->info, &cmd->root);
 404	if (r < 0)
 405		goto bad;
 406
 407	if (separate_dirty_bits(cmd)) {
 408		dm_disk_bitset_init(cmd->tm, &cmd->dirty_info);
 409		r = dm_bitset_empty(&cmd->dirty_info, &cmd->dirty_root);
 410		if (r < 0)
 411			goto bad;
 412	}
 413
 414	dm_disk_bitset_init(cmd->tm, &cmd->discard_info);
 415	r = dm_bitset_empty(&cmd->discard_info, &cmd->discard_root);
 416	if (r < 0)
 417		goto bad;
 418
 419	cmd->discard_block_size = 0;
 420	cmd->discard_nr_blocks = 0;
 421
 422	r = __write_initial_superblock(cmd);
 423	if (r)
 424		goto bad;
 425
 426	cmd->clean_when_opened = true;
 427	return 0;
 428
 429bad:
 430	dm_tm_destroy(cmd->tm);
 431	dm_sm_destroy(cmd->metadata_sm);
 432
 433	return r;
 434}
 435
 436static int __check_incompat_features(struct cache_disk_superblock *disk_super,
 437				     struct dm_cache_metadata *cmd)
 438{
 439	uint32_t incompat_flags, features;
 440
 441	incompat_flags = le32_to_cpu(disk_super->incompat_flags);
 442	features = incompat_flags & ~DM_CACHE_FEATURE_INCOMPAT_SUPP;
 443	if (features) {
 444		DMERR("could not access metadata due to unsupported optional features (%lx).",
 445		      (unsigned long)features);
 446		return -EINVAL;
 447	}
 448
 449	/*
 450	 * Check for read-only metadata to skip the following RDWR checks.
 451	 */
 452	if (bdev_read_only(cmd->bdev))
 453		return 0;
 454
 455	features = le32_to_cpu(disk_super->compat_ro_flags) & ~DM_CACHE_FEATURE_COMPAT_RO_SUPP;
 456	if (features) {
 457		DMERR("could not access metadata RDWR due to unsupported optional features (%lx).",
 458		      (unsigned long)features);
 459		return -EINVAL;
 460	}
 461
 462	return 0;
 463}
 464
 465static int __open_metadata(struct dm_cache_metadata *cmd)
 466{
 467	int r;
 468	struct dm_block *sblock;
 469	struct cache_disk_superblock *disk_super;
 470	unsigned long sb_flags;
 471
 472	r = superblock_read_lock(cmd, &sblock);
 473	if (r < 0) {
 474		DMERR("couldn't read lock superblock");
 475		return r;
 476	}
 477
 478	disk_super = dm_block_data(sblock);
 479
 480	/* Verify the data block size hasn't changed */
 481	if (le32_to_cpu(disk_super->data_block_size) != cmd->data_block_size) {
 482		DMERR("changing the data block size (from %u to %llu) is not supported",
 483		      le32_to_cpu(disk_super->data_block_size),
 484		      (unsigned long long)cmd->data_block_size);
 485		r = -EINVAL;
 486		goto bad;
 487	}
 488
 489	r = __check_incompat_features(disk_super, cmd);
 490	if (r < 0)
 491		goto bad;
 492
 493	r = dm_tm_open_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 494			       disk_super->metadata_space_map_root,
 495			       sizeof(disk_super->metadata_space_map_root),
 496			       &cmd->tm, &cmd->metadata_sm);
 497	if (r < 0) {
 498		DMERR("tm_open_with_sm failed");
 499		goto bad;
 500	}
 501
 502	__setup_mapping_info(cmd);
 503	dm_disk_bitset_init(cmd->tm, &cmd->dirty_info);
 504	dm_disk_bitset_init(cmd->tm, &cmd->discard_info);
 505	sb_flags = le32_to_cpu(disk_super->flags);
 506	cmd->clean_when_opened = test_bit(CLEAN_SHUTDOWN, &sb_flags);
 507	dm_bm_unlock(sblock);
 508
 509	return 0;
 510
 511bad:
 512	dm_bm_unlock(sblock);
 513	return r;
 514}
 515
 516static int __open_or_format_metadata(struct dm_cache_metadata *cmd,
 517				     bool format_device)
 518{
 519	int r;
 520	bool unformatted = false;
 521
 522	r = __superblock_all_zeroes(cmd->bm, &unformatted);
 523	if (r)
 524		return r;
 525
 526	if (unformatted)
 527		return format_device ? __format_metadata(cmd) : -EPERM;
 528
 529	return __open_metadata(cmd);
 530}
 531
 532static int __create_persistent_data_objects(struct dm_cache_metadata *cmd,
 533					    bool may_format_device)
 534{
 535	int r;
 536	cmd->bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT,
 537					  CACHE_MAX_CONCURRENT_LOCKS);
 538	if (IS_ERR(cmd->bm)) {
 539		DMERR("could not create block manager");
 540		r = PTR_ERR(cmd->bm);
 541		cmd->bm = NULL;
 542		return r;
 543	}
 544
 545	r = __open_or_format_metadata(cmd, may_format_device);
 546	if (r) {
 547		dm_block_manager_destroy(cmd->bm);
 548		cmd->bm = NULL;
 549	}
 550
 551	return r;
 552}
 553
 554static void __destroy_persistent_data_objects(struct dm_cache_metadata *cmd,
 555					      bool destroy_bm)
 556{
 557	dm_sm_destroy(cmd->metadata_sm);
 558	dm_tm_destroy(cmd->tm);
 559	if (destroy_bm)
 560		dm_block_manager_destroy(cmd->bm);
 561}
 562
 563typedef unsigned long (*flags_mutator)(unsigned long);
 564
 565static void update_flags(struct cache_disk_superblock *disk_super,
 566			 flags_mutator mutator)
 567{
 568	uint32_t sb_flags = mutator(le32_to_cpu(disk_super->flags));
 569	disk_super->flags = cpu_to_le32(sb_flags);
 570}
 571
 572static unsigned long set_clean_shutdown(unsigned long flags)
 573{
 574	set_bit(CLEAN_SHUTDOWN, &flags);
 575	return flags;
 576}
 577
 578static unsigned long clear_clean_shutdown(unsigned long flags)
 579{
 580	clear_bit(CLEAN_SHUTDOWN, &flags);
 581	return flags;
 582}
 583
 584static void read_superblock_fields(struct dm_cache_metadata *cmd,
 585				   struct cache_disk_superblock *disk_super)
 586{
 587	cmd->version = le32_to_cpu(disk_super->version);
 588	cmd->flags = le32_to_cpu(disk_super->flags);
 589	cmd->root = le64_to_cpu(disk_super->mapping_root);
 590	cmd->hint_root = le64_to_cpu(disk_super->hint_root);
 591	cmd->discard_root = le64_to_cpu(disk_super->discard_root);
 592	cmd->discard_block_size = le64_to_cpu(disk_super->discard_block_size);
 593	cmd->discard_nr_blocks = to_dblock(le64_to_cpu(disk_super->discard_nr_blocks));
 594	cmd->data_block_size = le32_to_cpu(disk_super->data_block_size);
 595	cmd->cache_blocks = to_cblock(le32_to_cpu(disk_super->cache_blocks));
 596	strncpy(cmd->policy_name, disk_super->policy_name, sizeof(cmd->policy_name));
 597	cmd->policy_version[0] = le32_to_cpu(disk_super->policy_version[0]);
 598	cmd->policy_version[1] = le32_to_cpu(disk_super->policy_version[1]);
 599	cmd->policy_version[2] = le32_to_cpu(disk_super->policy_version[2]);
 600	cmd->policy_hint_size = le32_to_cpu(disk_super->policy_hint_size);
 601
 602	cmd->stats.read_hits = le32_to_cpu(disk_super->read_hits);
 603	cmd->stats.read_misses = le32_to_cpu(disk_super->read_misses);
 604	cmd->stats.write_hits = le32_to_cpu(disk_super->write_hits);
 605	cmd->stats.write_misses = le32_to_cpu(disk_super->write_misses);
 606
 607	if (separate_dirty_bits(cmd))
 608		cmd->dirty_root = le64_to_cpu(disk_super->dirty_root);
 609
 610	cmd->changed = false;
 611}
 612
 613/*
 614 * The mutator updates the superblock flags.
 615 */
 616static int __begin_transaction_flags(struct dm_cache_metadata *cmd,
 617				     flags_mutator mutator)
 618{
 619	int r;
 620	struct cache_disk_superblock *disk_super;
 621	struct dm_block *sblock;
 622
 623	r = superblock_lock(cmd, &sblock);
 624	if (r)
 625		return r;
 626
 627	disk_super = dm_block_data(sblock);
 628	update_flags(disk_super, mutator);
 629	read_superblock_fields(cmd, disk_super);
 630	dm_bm_unlock(sblock);
 631
 632	return dm_bm_flush(cmd->bm);
 633}
 634
 635static int __begin_transaction(struct dm_cache_metadata *cmd)
 636{
 637	int r;
 638	struct cache_disk_superblock *disk_super;
 639	struct dm_block *sblock;
 640
 641	/*
 642	 * We re-read the superblock every time.  Shouldn't need to do this
 643	 * really.
 644	 */
 645	r = superblock_read_lock(cmd, &sblock);
 646	if (r)
 647		return r;
 648
 649	disk_super = dm_block_data(sblock);
 650	read_superblock_fields(cmd, disk_super);
 651	dm_bm_unlock(sblock);
 652
 653	return 0;
 654}
 655
 656static int __commit_transaction(struct dm_cache_metadata *cmd,
 657				flags_mutator mutator)
 658{
 659	int r;
 660	struct cache_disk_superblock *disk_super;
 661	struct dm_block *sblock;
 662
 663	/*
 664	 * We need to know if the cache_disk_superblock exceeds a 512-byte sector.
 665	 */
 666	BUILD_BUG_ON(sizeof(struct cache_disk_superblock) > 512);
 667
 668	if (separate_dirty_bits(cmd)) {
 669		r = dm_bitset_flush(&cmd->dirty_info, cmd->dirty_root,
 670				    &cmd->dirty_root);
 671		if (r)
 672			return r;
 673	}
 674
 675	r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root,
 676			    &cmd->discard_root);
 677	if (r)
 678		return r;
 679
 680	r = dm_tm_pre_commit(cmd->tm);
 681	if (r < 0)
 682		return r;
 683
 684	r = __save_sm_root(cmd);
 685	if (r)
 686		return r;
 687
 688	r = superblock_lock(cmd, &sblock);
 689	if (r)
 690		return r;
 691
 692	disk_super = dm_block_data(sblock);
 693
 694	disk_super->flags = cpu_to_le32(cmd->flags);
 695	if (mutator)
 696		update_flags(disk_super, mutator);
 697
 698	disk_super->mapping_root = cpu_to_le64(cmd->root);
 699	if (separate_dirty_bits(cmd))
 700		disk_super->dirty_root = cpu_to_le64(cmd->dirty_root);
 701	disk_super->hint_root = cpu_to_le64(cmd->hint_root);
 702	disk_super->discard_root = cpu_to_le64(cmd->discard_root);
 703	disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
 704	disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
 705	disk_super->cache_blocks = cpu_to_le32(from_cblock(cmd->cache_blocks));
 706	strncpy(disk_super->policy_name, cmd->policy_name, sizeof(disk_super->policy_name));
 707	disk_super->policy_version[0] = cpu_to_le32(cmd->policy_version[0]);
 708	disk_super->policy_version[1] = cpu_to_le32(cmd->policy_version[1]);
 709	disk_super->policy_version[2] = cpu_to_le32(cmd->policy_version[2]);
 710	disk_super->policy_hint_size = cpu_to_le32(cmd->policy_hint_size);
 711
 712	disk_super->read_hits = cpu_to_le32(cmd->stats.read_hits);
 713	disk_super->read_misses = cpu_to_le32(cmd->stats.read_misses);
 714	disk_super->write_hits = cpu_to_le32(cmd->stats.write_hits);
 715	disk_super->write_misses = cpu_to_le32(cmd->stats.write_misses);
 716	__copy_sm_root(cmd, disk_super);
 717
 718	return dm_tm_commit(cmd->tm, sblock);
 719}
 720
 721/*----------------------------------------------------------------*/
 722
 723/*
 724 * The mappings are held in a dm-array that has 64-bit values stored in
 725 * little-endian format.  The index is the cblock, the high 48bits of the
 726 * value are the oblock and the low 16 bit the flags.
 727 */
 728#define FLAGS_MASK ((1 << 16) - 1)
 729
 730static __le64 pack_value(dm_oblock_t block, unsigned flags)
 731{
 732	uint64_t value = from_oblock(block);
 733	value <<= 16;
 734	value = value | (flags & FLAGS_MASK);
 735	return cpu_to_le64(value);
 736}
 737
 738static void unpack_value(__le64 value_le, dm_oblock_t *block, unsigned *flags)
 739{
 740	uint64_t value = le64_to_cpu(value_le);
 741	uint64_t b = value >> 16;
 742	*block = to_oblock(b);
 743	*flags = value & FLAGS_MASK;
 744}
 745
 746/*----------------------------------------------------------------*/
 747
 748static struct dm_cache_metadata *metadata_open(struct block_device *bdev,
 749					       sector_t data_block_size,
 750					       bool may_format_device,
 751					       size_t policy_hint_size,
 752					       unsigned metadata_version)
 753{
 754	int r;
 755	struct dm_cache_metadata *cmd;
 756
 757	cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
 758	if (!cmd) {
 759		DMERR("could not allocate metadata struct");
 760		return ERR_PTR(-ENOMEM);
 761	}
 762
 763	cmd->version = metadata_version;
 764	refcount_set(&cmd->ref_count, 1);
 765	init_rwsem(&cmd->root_lock);
 766	cmd->bdev = bdev;
 767	cmd->data_block_size = data_block_size;
 768	cmd->cache_blocks = 0;
 769	cmd->policy_hint_size = policy_hint_size;
 770	cmd->changed = true;
 771	cmd->fail_io = false;
 772
 773	r = __create_persistent_data_objects(cmd, may_format_device);
 774	if (r) {
 775		kfree(cmd);
 776		return ERR_PTR(r);
 777	}
 778
 779	r = __begin_transaction_flags(cmd, clear_clean_shutdown);
 780	if (r < 0) {
 781		dm_cache_metadata_close(cmd);
 782		return ERR_PTR(r);
 783	}
 784
 785	return cmd;
 786}
 787
 788/*
 789 * We keep a little list of ref counted metadata objects to prevent two
 790 * different target instances creating separate bufio instances.  This is
 791 * an issue if a table is reloaded before the suspend.
 792 */
 793static DEFINE_MUTEX(table_lock);
 794static LIST_HEAD(table);
 795
 796static struct dm_cache_metadata *lookup(struct block_device *bdev)
 797{
 798	struct dm_cache_metadata *cmd;
 799
 800	list_for_each_entry(cmd, &table, list)
 801		if (cmd->bdev == bdev) {
 802			refcount_inc(&cmd->ref_count);
 803			return cmd;
 804		}
 805
 806	return NULL;
 807}
 808
 809static struct dm_cache_metadata *lookup_or_open(struct block_device *bdev,
 810						sector_t data_block_size,
 811						bool may_format_device,
 812						size_t policy_hint_size,
 813						unsigned metadata_version)
 814{
 815	struct dm_cache_metadata *cmd, *cmd2;
 816
 817	mutex_lock(&table_lock);
 818	cmd = lookup(bdev);
 819	mutex_unlock(&table_lock);
 820
 821	if (cmd)
 822		return cmd;
 823
 824	cmd = metadata_open(bdev, data_block_size, may_format_device,
 825			    policy_hint_size, metadata_version);
 826	if (!IS_ERR(cmd)) {
 827		mutex_lock(&table_lock);
 828		cmd2 = lookup(bdev);
 829		if (cmd2) {
 830			mutex_unlock(&table_lock);
 831			__destroy_persistent_data_objects(cmd, true);
 832			kfree(cmd);
 833			return cmd2;
 834		}
 835		list_add(&cmd->list, &table);
 836		mutex_unlock(&table_lock);
 837	}
 838
 839	return cmd;
 840}
 841
 842static bool same_params(struct dm_cache_metadata *cmd, sector_t data_block_size)
 843{
 844	if (cmd->data_block_size != data_block_size) {
 845		DMERR("data_block_size (%llu) different from that in metadata (%llu)",
 846		      (unsigned long long) data_block_size,
 847		      (unsigned long long) cmd->data_block_size);
 848		return false;
 849	}
 850
 851	return true;
 852}
 853
 854struct dm_cache_metadata *dm_cache_metadata_open(struct block_device *bdev,
 855						 sector_t data_block_size,
 856						 bool may_format_device,
 857						 size_t policy_hint_size,
 858						 unsigned metadata_version)
 859{
 860	struct dm_cache_metadata *cmd = lookup_or_open(bdev, data_block_size, may_format_device,
 861						       policy_hint_size, metadata_version);
 862
 863	if (!IS_ERR(cmd) && !same_params(cmd, data_block_size)) {
 864		dm_cache_metadata_close(cmd);
 865		return ERR_PTR(-EINVAL);
 866	}
 867
 868	return cmd;
 869}
 870
 871void dm_cache_metadata_close(struct dm_cache_metadata *cmd)
 872{
 873	if (refcount_dec_and_test(&cmd->ref_count)) {
 874		mutex_lock(&table_lock);
 875		list_del(&cmd->list);
 876		mutex_unlock(&table_lock);
 877
 878		if (!cmd->fail_io)
 879			__destroy_persistent_data_objects(cmd, true);
 880		kfree(cmd);
 881	}
 882}
 883
 884/*
 885 * Checks that the given cache block is either unmapped or clean.
 886 */
 887static int block_clean_combined_dirty(struct dm_cache_metadata *cmd, dm_cblock_t b,
 888				      bool *result)
 889{
 890	int r;
 891	__le64 value;
 892	dm_oblock_t ob;
 893	unsigned flags;
 894
 895	r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(b), &value);
 896	if (r)
 897		return r;
 898
 899	unpack_value(value, &ob, &flags);
 900	*result = !((flags & M_VALID) && (flags & M_DIRTY));
 901
 902	return 0;
 903}
 904
 905static int blocks_are_clean_combined_dirty(struct dm_cache_metadata *cmd,
 906					   dm_cblock_t begin, dm_cblock_t end,
 907					   bool *result)
 908{
 909	int r;
 910	*result = true;
 911
 912	while (begin != end) {
 913		r = block_clean_combined_dirty(cmd, begin, result);
 914		if (r) {
 915			DMERR("block_clean_combined_dirty failed");
 916			return r;
 917		}
 918
 919		if (!*result) {
 920			DMERR("cache block %llu is dirty",
 921			      (unsigned long long) from_cblock(begin));
 922			return 0;
 923		}
 924
 925		begin = to_cblock(from_cblock(begin) + 1);
 926	}
 927
 928	return 0;
 929}
 930
 931static int blocks_are_clean_separate_dirty(struct dm_cache_metadata *cmd,
 932					   dm_cblock_t begin, dm_cblock_t end,
 933					   bool *result)
 934{
 935	int r;
 936	bool dirty_flag;
 937	*result = true;
 938
 939	if (from_cblock(cmd->cache_blocks) == 0)
 940		/* Nothing to do */
 941		return 0;
 942
 943	r = dm_bitset_cursor_begin(&cmd->dirty_info, cmd->dirty_root,
 944				   from_cblock(cmd->cache_blocks), &cmd->dirty_cursor);
 945	if (r) {
 946		DMERR("%s: dm_bitset_cursor_begin for dirty failed", __func__);
 947		return r;
 948	}
 949
 950	r = dm_bitset_cursor_skip(&cmd->dirty_cursor, from_cblock(begin));
 951	if (r) {
 952		DMERR("%s: dm_bitset_cursor_skip for dirty failed", __func__);
 953		dm_bitset_cursor_end(&cmd->dirty_cursor);
 954		return r;
 955	}
 956
 957	while (begin != end) {
 958		/*
 959		 * We assume that unmapped blocks have their dirty bit
 960		 * cleared.
 961		 */
 962		dirty_flag = dm_bitset_cursor_get_value(&cmd->dirty_cursor);
 963		if (dirty_flag) {
 964			DMERR("%s: cache block %llu is dirty", __func__,
 965			      (unsigned long long) from_cblock(begin));
 966			dm_bitset_cursor_end(&cmd->dirty_cursor);
 967			*result = false;
 968			return 0;
 969		}
 970
 971		begin = to_cblock(from_cblock(begin) + 1);
 972		if (begin == end)
 973			break;
 974
 975		r = dm_bitset_cursor_next(&cmd->dirty_cursor);
 976		if (r) {
 977			DMERR("%s: dm_bitset_cursor_next for dirty failed", __func__);
 978			dm_bitset_cursor_end(&cmd->dirty_cursor);
 979			return r;
 980		}
 981	}
 982
 983	dm_bitset_cursor_end(&cmd->dirty_cursor);
 984
 985	return 0;
 986}
 987
 988static int blocks_are_unmapped_or_clean(struct dm_cache_metadata *cmd,
 989					dm_cblock_t begin, dm_cblock_t end,
 990					bool *result)
 991{
 992	if (separate_dirty_bits(cmd))
 993		return blocks_are_clean_separate_dirty(cmd, begin, end, result);
 994	else
 995		return blocks_are_clean_combined_dirty(cmd, begin, end, result);
 996}
 997
 998static bool cmd_write_lock(struct dm_cache_metadata *cmd)
 999{
1000	down_write(&cmd->root_lock);
1001	if (cmd->fail_io || dm_bm_is_read_only(cmd->bm)) {
1002		up_write(&cmd->root_lock);
1003		return false;
1004	}
1005	return true;
1006}
1007
1008#define WRITE_LOCK(cmd)				\
1009	do {					\
1010		if (!cmd_write_lock((cmd)))	\
1011			return -EINVAL;		\
1012	} while(0)
1013
1014#define WRITE_LOCK_VOID(cmd)			\
1015	do {					\
1016		if (!cmd_write_lock((cmd)))	\
1017			return;			\
1018	} while(0)
1019
1020#define WRITE_UNLOCK(cmd) \
1021	up_write(&(cmd)->root_lock)
1022
1023static bool cmd_read_lock(struct dm_cache_metadata *cmd)
1024{
1025	down_read(&cmd->root_lock);
1026	if (cmd->fail_io) {
1027		up_read(&cmd->root_lock);
1028		return false;
1029	}
1030	return true;
1031}
1032
1033#define READ_LOCK(cmd)				\
1034	do {					\
1035		if (!cmd_read_lock((cmd)))	\
1036			return -EINVAL;		\
1037	} while(0)
1038
1039#define READ_LOCK_VOID(cmd)			\
1040	do {					\
1041		if (!cmd_read_lock((cmd)))	\
1042			return;			\
1043	} while(0)
1044
1045#define READ_UNLOCK(cmd) \
1046	up_read(&(cmd)->root_lock)
1047
1048int dm_cache_resize(struct dm_cache_metadata *cmd, dm_cblock_t new_cache_size)
1049{
1050	int r;
1051	bool clean;
1052	__le64 null_mapping = pack_value(0, 0);
1053
1054	WRITE_LOCK(cmd);
1055	__dm_bless_for_disk(&null_mapping);
1056
1057	if (from_cblock(new_cache_size) < from_cblock(cmd->cache_blocks)) {
1058		r = blocks_are_unmapped_or_clean(cmd, new_cache_size, cmd->cache_blocks, &clean);
1059		if (r) {
1060			__dm_unbless_for_disk(&null_mapping);
1061			goto out;
1062		}
1063
1064		if (!clean) {
1065			DMERR("unable to shrink cache due to dirty blocks");
1066			r = -EINVAL;
1067			__dm_unbless_for_disk(&null_mapping);
1068			goto out;
1069		}
1070	}
1071
1072	r = dm_array_resize(&cmd->info, cmd->root, from_cblock(cmd->cache_blocks),
1073			    from_cblock(new_cache_size),
1074			    &null_mapping, &cmd->root);
1075	if (r)
1076		goto out;
1077
1078	if (separate_dirty_bits(cmd)) {
1079		r = dm_bitset_resize(&cmd->dirty_info, cmd->dirty_root,
1080				     from_cblock(cmd->cache_blocks), from_cblock(new_cache_size),
1081				     false, &cmd->dirty_root);
1082		if (r)
1083			goto out;
1084	}
1085
1086	cmd->cache_blocks = new_cache_size;
1087	cmd->changed = true;
1088
1089out:
1090	WRITE_UNLOCK(cmd);
1091
1092	return r;
1093}
1094
1095int dm_cache_discard_bitset_resize(struct dm_cache_metadata *cmd,
1096				   sector_t discard_block_size,
1097				   dm_dblock_t new_nr_entries)
1098{
1099	int r;
1100
1101	WRITE_LOCK(cmd);
1102	r = dm_bitset_resize(&cmd->discard_info,
1103			     cmd->discard_root,
1104			     from_dblock(cmd->discard_nr_blocks),
1105			     from_dblock(new_nr_entries),
1106			     false, &cmd->discard_root);
1107	if (!r) {
1108		cmd->discard_block_size = discard_block_size;
1109		cmd->discard_nr_blocks = new_nr_entries;
1110	}
1111
1112	cmd->changed = true;
1113	WRITE_UNLOCK(cmd);
1114
1115	return r;
1116}
1117
1118static int __set_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
1119{
1120	return dm_bitset_set_bit(&cmd->discard_info, cmd->discard_root,
1121				 from_dblock(b), &cmd->discard_root);
1122}
1123
1124static int __clear_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
1125{
1126	return dm_bitset_clear_bit(&cmd->discard_info, cmd->discard_root,
1127				   from_dblock(b), &cmd->discard_root);
1128}
1129
1130static int __discard(struct dm_cache_metadata *cmd,
1131		     dm_dblock_t dblock, bool discard)
1132{
1133	int r;
1134
1135	r = (discard ? __set_discard : __clear_discard)(cmd, dblock);
1136	if (r)
1137		return r;
1138
1139	cmd->changed = true;
1140	return 0;
1141}
1142
1143int dm_cache_set_discard(struct dm_cache_metadata *cmd,
1144			 dm_dblock_t dblock, bool discard)
1145{
1146	int r;
1147
1148	WRITE_LOCK(cmd);
1149	r = __discard(cmd, dblock, discard);
1150	WRITE_UNLOCK(cmd);
1151
1152	return r;
1153}
1154
1155static int __load_discards(struct dm_cache_metadata *cmd,
1156			   load_discard_fn fn, void *context)
1157{
1158	int r = 0;
1159	uint32_t b;
1160	struct dm_bitset_cursor c;
1161
1162	if (from_dblock(cmd->discard_nr_blocks) == 0)
1163		/* nothing to do */
1164		return 0;
1165
1166	if (cmd->clean_when_opened) {
1167		r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root, &cmd->discard_root);
1168		if (r)
1169			return r;
1170
1171		r = dm_bitset_cursor_begin(&cmd->discard_info, cmd->discard_root,
1172					   from_dblock(cmd->discard_nr_blocks), &c);
1173		if (r)
1174			return r;
1175
1176		for (b = 0; ; b++) {
1177			r = fn(context, cmd->discard_block_size, to_dblock(b),
1178			       dm_bitset_cursor_get_value(&c));
1179			if (r)
1180				break;
1181
1182			if (b >= (from_dblock(cmd->discard_nr_blocks) - 1))
1183				break;
1184
1185			r = dm_bitset_cursor_next(&c);
1186			if (r)
1187				break;
1188		}
1189
1190		dm_bitset_cursor_end(&c);
1191
1192	} else {
1193		for (b = 0; b < from_dblock(cmd->discard_nr_blocks); b++) {
1194			r = fn(context, cmd->discard_block_size, to_dblock(b), false);
1195			if (r)
1196				return r;
1197		}
1198	}
1199
1200	return r;
1201}
1202
1203int dm_cache_load_discards(struct dm_cache_metadata *cmd,
1204			   load_discard_fn fn, void *context)
1205{
1206	int r;
1207
1208	READ_LOCK(cmd);
1209	r = __load_discards(cmd, fn, context);
1210	READ_UNLOCK(cmd);
1211
1212	return r;
1213}
1214
1215int dm_cache_size(struct dm_cache_metadata *cmd, dm_cblock_t *result)
1216{
1217	READ_LOCK(cmd);
1218	*result = cmd->cache_blocks;
1219	READ_UNLOCK(cmd);
1220
1221	return 0;
1222}
1223
1224static int __remove(struct dm_cache_metadata *cmd, dm_cblock_t cblock)
1225{
1226	int r;
1227	__le64 value = pack_value(0, 0);
1228
1229	__dm_bless_for_disk(&value);
1230	r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
1231			       &value, &cmd->root);
1232	if (r)
1233		return r;
1234
1235	cmd->changed = true;
1236	return 0;
1237}
1238
1239int dm_cache_remove_mapping(struct dm_cache_metadata *cmd, dm_cblock_t cblock)
1240{
1241	int r;
1242
1243	WRITE_LOCK(cmd);
1244	r = __remove(cmd, cblock);
1245	WRITE_UNLOCK(cmd);
1246
1247	return r;
1248}
1249
1250static int __insert(struct dm_cache_metadata *cmd,
1251		    dm_cblock_t cblock, dm_oblock_t oblock)
1252{
1253	int r;
1254	__le64 value = pack_value(oblock, M_VALID);
1255	__dm_bless_for_disk(&value);
1256
1257	r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
1258			       &value, &cmd->root);
1259	if (r)
1260		return r;
1261
1262	cmd->changed = true;
1263	return 0;
1264}
1265
1266int dm_cache_insert_mapping(struct dm_cache_metadata *cmd,
1267			    dm_cblock_t cblock, dm_oblock_t oblock)
1268{
1269	int r;
1270
1271	WRITE_LOCK(cmd);
1272	r = __insert(cmd, cblock, oblock);
1273	WRITE_UNLOCK(cmd);
1274
1275	return r;
1276}
1277
1278struct thunk {
1279	load_mapping_fn fn;
1280	void *context;
1281
1282	struct dm_cache_metadata *cmd;
1283	bool respect_dirty_flags;
1284	bool hints_valid;
1285};
1286
1287static bool policy_unchanged(struct dm_cache_metadata *cmd,
1288			     struct dm_cache_policy *policy)
1289{
1290	const char *policy_name = dm_cache_policy_get_name(policy);
1291	const unsigned *policy_version = dm_cache_policy_get_version(policy);
1292	size_t policy_hint_size = dm_cache_policy_get_hint_size(policy);
1293
1294	/*
1295	 * Ensure policy names match.
1296	 */
1297	if (strncmp(cmd->policy_name, policy_name, sizeof(cmd->policy_name)))
1298		return false;
1299
1300	/*
1301	 * Ensure policy major versions match.
1302	 */
1303	if (cmd->policy_version[0] != policy_version[0])
1304		return false;
1305
1306	/*
1307	 * Ensure policy hint sizes match.
1308	 */
1309	if (cmd->policy_hint_size != policy_hint_size)
1310		return false;
1311
1312	return true;
1313}
1314
1315static bool hints_array_initialized(struct dm_cache_metadata *cmd)
1316{
1317	return cmd->hint_root && cmd->policy_hint_size;
1318}
1319
1320static bool hints_array_available(struct dm_cache_metadata *cmd,
1321				  struct dm_cache_policy *policy)
1322{
1323	return cmd->clean_when_opened && policy_unchanged(cmd, policy) &&
1324		hints_array_initialized(cmd);
1325}
1326
1327static int __load_mapping_v1(struct dm_cache_metadata *cmd,
1328			     uint64_t cb, bool hints_valid,
1329			     struct dm_array_cursor *mapping_cursor,
1330			     struct dm_array_cursor *hint_cursor,
1331			     load_mapping_fn fn, void *context)
1332{
1333	int r = 0;
1334
1335	__le64 mapping;
1336	__le32 hint = 0;
1337
1338	__le64 *mapping_value_le;
1339	__le32 *hint_value_le;
1340
1341	dm_oblock_t oblock;
1342	unsigned flags;
1343	bool dirty = true;
1344
1345	dm_array_cursor_get_value(mapping_cursor, (void **) &mapping_value_le);
1346	memcpy(&mapping, mapping_value_le, sizeof(mapping));
1347	unpack_value(mapping, &oblock, &flags);
1348
1349	if (flags & M_VALID) {
1350		if (hints_valid) {
1351			dm_array_cursor_get_value(hint_cursor, (void **) &hint_value_le);
1352			memcpy(&hint, hint_value_le, sizeof(hint));
1353		}
1354		if (cmd->clean_when_opened)
1355			dirty = flags & M_DIRTY;
1356
1357		r = fn(context, oblock, to_cblock(cb), dirty,
1358		       le32_to_cpu(hint), hints_valid);
1359		if (r) {
1360			DMERR("policy couldn't load cache block %llu",
1361			      (unsigned long long) from_cblock(to_cblock(cb)));
1362		}
1363	}
1364
1365	return r;
1366}
1367
1368static int __load_mapping_v2(struct dm_cache_metadata *cmd,
1369			     uint64_t cb, bool hints_valid,
1370			     struct dm_array_cursor *mapping_cursor,
1371			     struct dm_array_cursor *hint_cursor,
1372			     struct dm_bitset_cursor *dirty_cursor,
1373			     load_mapping_fn fn, void *context)
1374{
1375	int r = 0;
1376
1377	__le64 mapping;
1378	__le32 hint = 0;
1379
1380	__le64 *mapping_value_le;
1381	__le32 *hint_value_le;
1382
1383	dm_oblock_t oblock;
1384	unsigned flags;
1385	bool dirty = true;
1386
1387	dm_array_cursor_get_value(mapping_cursor, (void **) &mapping_value_le);
1388	memcpy(&mapping, mapping_value_le, sizeof(mapping));
1389	unpack_value(mapping, &oblock, &flags);
1390
1391	if (flags & M_VALID) {
1392		if (hints_valid) {
1393			dm_array_cursor_get_value(hint_cursor, (void **) &hint_value_le);
1394			memcpy(&hint, hint_value_le, sizeof(hint));
1395		}
1396		if (cmd->clean_when_opened)
1397			dirty = dm_bitset_cursor_get_value(dirty_cursor);
1398
1399		r = fn(context, oblock, to_cblock(cb), dirty,
1400		       le32_to_cpu(hint), hints_valid);
1401		if (r) {
1402			DMERR("policy couldn't load cache block %llu",
1403			      (unsigned long long) from_cblock(to_cblock(cb)));
1404		}
1405	}
1406
1407	return r;
1408}
1409
1410static int __load_mappings(struct dm_cache_metadata *cmd,
1411			   struct dm_cache_policy *policy,
1412			   load_mapping_fn fn, void *context)
1413{
1414	int r;
1415	uint64_t cb;
1416
1417	bool hints_valid = hints_array_available(cmd, policy);
1418
1419	if (from_cblock(cmd->cache_blocks) == 0)
1420		/* Nothing to do */
1421		return 0;
1422
1423	r = dm_array_cursor_begin(&cmd->info, cmd->root, &cmd->mapping_cursor);
1424	if (r)
1425		return r;
1426
1427	if (hints_valid) {
1428		r = dm_array_cursor_begin(&cmd->hint_info, cmd->hint_root, &cmd->hint_cursor);
1429		if (r) {
1430			dm_array_cursor_end(&cmd->mapping_cursor);
1431			return r;
1432		}
1433	}
1434
1435	if (separate_dirty_bits(cmd)) {
1436		r = dm_bitset_cursor_begin(&cmd->dirty_info, cmd->dirty_root,
1437					   from_cblock(cmd->cache_blocks),
1438					   &cmd->dirty_cursor);
1439		if (r) {
1440			dm_array_cursor_end(&cmd->hint_cursor);
1441			dm_array_cursor_end(&cmd->mapping_cursor);
1442			return r;
1443		}
1444	}
1445
1446	for (cb = 0; ; cb++) {
1447		if (separate_dirty_bits(cmd))
1448			r = __load_mapping_v2(cmd, cb, hints_valid,
1449					      &cmd->mapping_cursor,
1450					      &cmd->hint_cursor,
1451					      &cmd->dirty_cursor,
1452					      fn, context);
1453		else
1454			r = __load_mapping_v1(cmd, cb, hints_valid,
1455					      &cmd->mapping_cursor, &cmd->hint_cursor,
1456					      fn, context);
1457		if (r)
1458			goto out;
1459
1460		/*
1461		 * We need to break out before we move the cursors.
1462		 */
1463		if (cb >= (from_cblock(cmd->cache_blocks) - 1))
1464			break;
1465
1466		r = dm_array_cursor_next(&cmd->mapping_cursor);
1467		if (r) {
1468			DMERR("dm_array_cursor_next for mapping failed");
1469			goto out;
1470		}
1471
1472		if (hints_valid) {
1473			r = dm_array_cursor_next(&cmd->hint_cursor);
1474			if (r) {
1475				dm_array_cursor_end(&cmd->hint_cursor);
1476				hints_valid = false;
1477			}
1478		}
1479
1480		if (separate_dirty_bits(cmd)) {
1481			r = dm_bitset_cursor_next(&cmd->dirty_cursor);
1482			if (r) {
1483				DMERR("dm_bitset_cursor_next for dirty failed");
1484				goto out;
1485			}
1486		}
1487	}
1488out:
1489	dm_array_cursor_end(&cmd->mapping_cursor);
1490	if (hints_valid)
1491		dm_array_cursor_end(&cmd->hint_cursor);
1492
1493	if (separate_dirty_bits(cmd))
1494		dm_bitset_cursor_end(&cmd->dirty_cursor);
1495
1496	return r;
1497}
1498
1499int dm_cache_load_mappings(struct dm_cache_metadata *cmd,
1500			   struct dm_cache_policy *policy,
1501			   load_mapping_fn fn, void *context)
1502{
1503	int r;
1504
1505	READ_LOCK(cmd);
1506	r = __load_mappings(cmd, policy, fn, context);
1507	READ_UNLOCK(cmd);
1508
1509	return r;
1510}
1511
1512static int __dump_mapping(void *context, uint64_t cblock, void *leaf)
1513{
 
1514	__le64 value;
1515	dm_oblock_t oblock;
1516	unsigned flags;
1517
1518	memcpy(&value, leaf, sizeof(value));
1519	unpack_value(value, &oblock, &flags);
1520
1521	return 0;
1522}
1523
1524static int __dump_mappings(struct dm_cache_metadata *cmd)
1525{
1526	return dm_array_walk(&cmd->info, cmd->root, __dump_mapping, NULL);
1527}
1528
1529void dm_cache_dump(struct dm_cache_metadata *cmd)
1530{
1531	READ_LOCK_VOID(cmd);
1532	__dump_mappings(cmd);
1533	READ_UNLOCK(cmd);
1534}
1535
1536int dm_cache_changed_this_transaction(struct dm_cache_metadata *cmd)
1537{
1538	int r;
1539
1540	READ_LOCK(cmd);
1541	r = cmd->changed;
1542	READ_UNLOCK(cmd);
1543
1544	return r;
1545}
1546
1547static int __dirty(struct dm_cache_metadata *cmd, dm_cblock_t cblock, bool dirty)
1548{
1549	int r;
1550	unsigned flags;
1551	dm_oblock_t oblock;
1552	__le64 value;
1553
1554	r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(cblock), &value);
1555	if (r)
1556		return r;
1557
1558	unpack_value(value, &oblock, &flags);
1559
1560	if (((flags & M_DIRTY) && dirty) || (!(flags & M_DIRTY) && !dirty))
1561		/* nothing to be done */
1562		return 0;
1563
1564	value = pack_value(oblock, (flags & ~M_DIRTY) | (dirty ? M_DIRTY : 0));
1565	__dm_bless_for_disk(&value);
1566
1567	r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
1568			       &value, &cmd->root);
1569	if (r)
1570		return r;
1571
1572	cmd->changed = true;
1573	return 0;
1574
1575}
1576
1577static int __set_dirty_bits_v1(struct dm_cache_metadata *cmd, unsigned nr_bits, unsigned long *bits)
1578{
1579	int r;
1580	unsigned i;
1581	for (i = 0; i < nr_bits; i++) {
1582		r = __dirty(cmd, to_cblock(i), test_bit(i, bits));
1583		if (r)
1584			return r;
1585	}
1586
1587	return 0;
1588}
1589
1590static int is_dirty_callback(uint32_t index, bool *value, void *context)
1591{
1592	unsigned long *bits = context;
1593	*value = test_bit(index, bits);
1594	return 0;
1595}
1596
1597static int __set_dirty_bits_v2(struct dm_cache_metadata *cmd, unsigned nr_bits, unsigned long *bits)
1598{
1599	int r = 0;
1600
1601	/* nr_bits is really just a sanity check */
1602	if (nr_bits != from_cblock(cmd->cache_blocks)) {
1603		DMERR("dirty bitset is wrong size");
1604		return -EINVAL;
1605	}
1606
1607	r = dm_bitset_del(&cmd->dirty_info, cmd->dirty_root);
1608	if (r)
1609		return r;
1610
1611	cmd->changed = true;
1612	return dm_bitset_new(&cmd->dirty_info, &cmd->dirty_root, nr_bits, is_dirty_callback, bits);
1613}
1614
1615int dm_cache_set_dirty_bits(struct dm_cache_metadata *cmd,
1616			    unsigned nr_bits,
1617			    unsigned long *bits)
1618{
1619	int r;
1620
1621	WRITE_LOCK(cmd);
1622	if (separate_dirty_bits(cmd))
1623		r = __set_dirty_bits_v2(cmd, nr_bits, bits);
1624	else
1625		r = __set_dirty_bits_v1(cmd, nr_bits, bits);
1626	WRITE_UNLOCK(cmd);
1627
1628	return r;
1629}
1630
1631void dm_cache_metadata_get_stats(struct dm_cache_metadata *cmd,
1632				 struct dm_cache_statistics *stats)
1633{
1634	READ_LOCK_VOID(cmd);
1635	*stats = cmd->stats;
1636	READ_UNLOCK(cmd);
1637}
1638
1639void dm_cache_metadata_set_stats(struct dm_cache_metadata *cmd,
1640				 struct dm_cache_statistics *stats)
1641{
1642	WRITE_LOCK_VOID(cmd);
1643	cmd->stats = *stats;
1644	WRITE_UNLOCK(cmd);
1645}
1646
1647int dm_cache_commit(struct dm_cache_metadata *cmd, bool clean_shutdown)
1648{
1649	int r = -EINVAL;
1650	flags_mutator mutator = (clean_shutdown ? set_clean_shutdown :
1651				 clear_clean_shutdown);
1652
1653	WRITE_LOCK(cmd);
1654	if (cmd->fail_io)
1655		goto out;
1656
1657	r = __commit_transaction(cmd, mutator);
1658	if (r)
1659		goto out;
1660
1661	r = __begin_transaction(cmd);
1662out:
1663	WRITE_UNLOCK(cmd);
1664	return r;
1665}
1666
1667int dm_cache_get_free_metadata_block_count(struct dm_cache_metadata *cmd,
1668					   dm_block_t *result)
1669{
1670	int r = -EINVAL;
1671
1672	READ_LOCK(cmd);
1673	if (!cmd->fail_io)
1674		r = dm_sm_get_nr_free(cmd->metadata_sm, result);
1675	READ_UNLOCK(cmd);
1676
1677	return r;
1678}
1679
1680int dm_cache_get_metadata_dev_size(struct dm_cache_metadata *cmd,
1681				   dm_block_t *result)
1682{
1683	int r = -EINVAL;
1684
1685	READ_LOCK(cmd);
1686	if (!cmd->fail_io)
1687		r = dm_sm_get_nr_blocks(cmd->metadata_sm, result);
1688	READ_UNLOCK(cmd);
1689
1690	return r;
1691}
1692
1693/*----------------------------------------------------------------*/
1694
1695static int get_hint(uint32_t index, void *value_le, void *context)
1696{
1697	uint32_t value;
1698	struct dm_cache_policy *policy = context;
1699
1700	value = policy_get_hint(policy, to_cblock(index));
1701	*((__le32 *) value_le) = cpu_to_le32(value);
1702
1703	return 0;
1704}
1705
1706/*
1707 * It's quicker to always delete the hint array, and recreate with
1708 * dm_array_new().
1709 */
1710static int write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
1711{
1712	int r;
1713	size_t hint_size;
1714	const char *policy_name = dm_cache_policy_get_name(policy);
1715	const unsigned *policy_version = dm_cache_policy_get_version(policy);
1716
1717	if (!policy_name[0] ||
1718	    (strlen(policy_name) > sizeof(cmd->policy_name) - 1))
1719		return -EINVAL;
1720
1721	strncpy(cmd->policy_name, policy_name, sizeof(cmd->policy_name));
1722	memcpy(cmd->policy_version, policy_version, sizeof(cmd->policy_version));
1723
1724	hint_size = dm_cache_policy_get_hint_size(policy);
1725	if (!hint_size)
1726		return 0; /* short-circuit hints initialization */
1727	cmd->policy_hint_size = hint_size;
1728
1729	if (cmd->hint_root) {
1730		r = dm_array_del(&cmd->hint_info, cmd->hint_root);
1731		if (r)
1732			return r;
1733	}
1734
1735	return dm_array_new(&cmd->hint_info, &cmd->hint_root,
1736			    from_cblock(cmd->cache_blocks),
1737			    get_hint, policy);
1738}
1739
1740int dm_cache_write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
1741{
1742	int r;
1743
1744	WRITE_LOCK(cmd);
1745	r = write_hints(cmd, policy);
1746	WRITE_UNLOCK(cmd);
1747
1748	return r;
1749}
1750
1751int dm_cache_metadata_all_clean(struct dm_cache_metadata *cmd, bool *result)
1752{
1753	int r;
1754
1755	READ_LOCK(cmd);
1756	r = blocks_are_unmapped_or_clean(cmd, 0, cmd->cache_blocks, result);
1757	READ_UNLOCK(cmd);
1758
1759	return r;
1760}
1761
1762void dm_cache_metadata_set_read_only(struct dm_cache_metadata *cmd)
1763{
1764	WRITE_LOCK_VOID(cmd);
1765	dm_bm_set_read_only(cmd->bm);
1766	WRITE_UNLOCK(cmd);
1767}
1768
1769void dm_cache_metadata_set_read_write(struct dm_cache_metadata *cmd)
1770{
1771	WRITE_LOCK_VOID(cmd);
1772	dm_bm_set_read_write(cmd->bm);
1773	WRITE_UNLOCK(cmd);
1774}
1775
1776int dm_cache_metadata_set_needs_check(struct dm_cache_metadata *cmd)
1777{
1778	int r;
1779	struct dm_block *sblock;
1780	struct cache_disk_superblock *disk_super;
1781
1782	WRITE_LOCK(cmd);
1783	set_bit(NEEDS_CHECK, &cmd->flags);
1784
1785	r = superblock_lock(cmd, &sblock);
1786	if (r) {
1787		DMERR("couldn't read superblock");
1788		goto out;
1789	}
1790
1791	disk_super = dm_block_data(sblock);
1792	disk_super->flags = cpu_to_le32(cmd->flags);
1793
1794	dm_bm_unlock(sblock);
1795
1796out:
1797	WRITE_UNLOCK(cmd);
1798	return r;
1799}
1800
1801int dm_cache_metadata_needs_check(struct dm_cache_metadata *cmd, bool *result)
1802{
1803	READ_LOCK(cmd);
1804	*result = !!test_bit(NEEDS_CHECK, &cmd->flags);
1805	READ_UNLOCK(cmd);
1806
1807	return 0;
1808}
1809
1810int dm_cache_metadata_abort(struct dm_cache_metadata *cmd)
1811{
1812	int r = -EINVAL;
1813	struct dm_block_manager *old_bm = NULL, *new_bm = NULL;
1814
1815	/* fail_io is double-checked with cmd->root_lock held below */
1816	if (unlikely(cmd->fail_io))
1817		return r;
1818
1819	/*
1820	 * Replacement block manager (new_bm) is created and old_bm destroyed outside of
1821	 * cmd root_lock to avoid ABBA deadlock that would result (due to life-cycle of
1822	 * shrinker associated with the block manager's bufio client vs cmd root_lock).
1823	 * - must take shrinker_rwsem without holding cmd->root_lock
1824	 */
1825	new_bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT,
1826					 CACHE_MAX_CONCURRENT_LOCKS);
1827
1828	WRITE_LOCK(cmd);
1829	if (cmd->fail_io) {
1830		WRITE_UNLOCK(cmd);
1831		goto out;
1832	}
1833
1834	__destroy_persistent_data_objects(cmd, false);
1835	old_bm = cmd->bm;
1836	if (IS_ERR(new_bm)) {
1837		DMERR("could not create block manager during abort");
1838		cmd->bm = NULL;
1839		r = PTR_ERR(new_bm);
1840		goto out_unlock;
1841	}
1842
1843	cmd->bm = new_bm;
1844	r = __open_or_format_metadata(cmd, false);
1845	if (r) {
1846		cmd->bm = NULL;
1847		goto out_unlock;
1848	}
1849	new_bm = NULL;
1850out_unlock:
1851	if (r)
1852		cmd->fail_io = true;
1853	WRITE_UNLOCK(cmd);
1854	dm_block_manager_destroy(old_bm);
1855out:
1856	if (new_bm && !IS_ERR(new_bm))
1857		dm_block_manager_destroy(new_bm);
1858
1859	return r;
1860}