1/*
   2 * dm-snapshot.c
   3 *
   4 * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
   5 *
   6 * This file is released under the GPL.
   7 */
   8
   9#include <linux/blkdev.h>
  10#include <linux/device-mapper.h>
  11#include <linux/delay.h>
  12#include <linux/fs.h>
  13#include <linux/init.h>
  14#include <linux/kdev_t.h>
  15#include <linux/list.h>
  16#include <linux/mempool.h>
  17#include <linux/module.h>
  18#include <linux/slab.h>
  19#include <linux/vmalloc.h>
  20#include <linux/log2.h>
  21#include <linux/dm-kcopyd.h>
  22
 
 
  23#include "dm-exception-store.h"
  24
  25#define DM_MSG_PREFIX "snapshots"
  26
  27static const char dm_snapshot_merge_target_name[] = "snapshot-merge";
  28
  29#define dm_target_is_snapshot_merge(ti) \
  30	((ti)->type->name == dm_snapshot_merge_target_name)
  31
  32/*
  33 * The size of the mempool used to track chunks in use.
  34 */
  35#define MIN_IOS 256
  36
  37#define DM_TRACKED_CHUNK_HASH_SIZE	16
  38#define DM_TRACKED_CHUNK_HASH(x)	((unsigned long)(x) & \
  39					 (DM_TRACKED_CHUNK_HASH_SIZE - 1))
  40
  41struct dm_exception_table {
  42	uint32_t hash_mask;
  43	unsigned hash_shift;
  44	struct list_head *table;
  45};
  46
  47struct dm_snapshot {
  48	struct rw_semaphore lock;
  49
  50	struct dm_dev *origin;
  51	struct dm_dev *cow;
  52
  53	struct dm_target *ti;
  54
  55	/* List of snapshots per Origin */
  56	struct list_head list;
  57
  58	/*
  59	 * You can't use a snapshot if this is 0 (e.g. if full).
  60	 * A snapshot-merge target never clears this.
  61	 */
  62	int valid;
  63
 
 
 
 
 
 
 
  64	/* Origin writes don't trigger exceptions until this is set */
  65	int active;
  66
  67	atomic_t pending_exceptions_count;
  68
  69	/* Protected by "lock" */
  70	sector_t exception_start_sequence;
  71
  72	/* Protected by kcopyd single-threaded callback */
  73	sector_t exception_complete_sequence;
  74
  75	/*
  76	 * A list of pending exceptions that completed out of order.
  77	 * Protected by kcopyd single-threaded callback.
  78	 */
  79	struct list_head out_of_order_list;
  80
  81	mempool_t *pending_pool;
  82
  83	struct dm_exception_table pending;
  84	struct dm_exception_table complete;
  85
  86	/*
  87	 * pe_lock protects all pending_exception operations and access
  88	 * as well as the snapshot_bios list.
  89	 */
  90	spinlock_t pe_lock;
  91
  92	/* Chunks with outstanding reads */
  93	spinlock_t tracked_chunk_lock;
  94	struct hlist_head tracked_chunk_hash[DM_TRACKED_CHUNK_HASH_SIZE];
  95
  96	/* The on disk metadata handler */
  97	struct dm_exception_store *store;
  98
  99	struct dm_kcopyd_client *kcopyd_client;
 100
 101	/* Wait for events based on state_bits */
 102	unsigned long state_bits;
 103
 104	/* Range of chunks currently being merged. */
 105	chunk_t first_merging_chunk;
 106	int num_merging_chunks;
 107
 108	/*
 109	 * The merge operation failed if this flag is set.
 110	 * Failure modes are handled as follows:
 111	 * - I/O error reading the header
 112	 *   	=> don't load the target; abort.
 113	 * - Header does not have "valid" flag set
 114	 *   	=> use the origin; forget about the snapshot.
 115	 * - I/O error when reading exceptions
 116	 *   	=> don't load the target; abort.
 117	 *         (We can't use the intermediate origin state.)
 118	 * - I/O error while merging
 119	 *	=> stop merging; set merge_failed; process I/O normally.
 120	 */
 121	int merge_failed;
 122
 123	/*
 124	 * Incoming bios that overlap with chunks being merged must wait
 125	 * for them to be committed.
 126	 */
 127	struct bio_list bios_queued_during_merge;
 128};
 129
 130/*
 131 * state_bits:
 132 *   RUNNING_MERGE  - Merge operation is in progress.
 133 *   SHUTDOWN_MERGE - Set to signal that merge needs to be stopped;
 134 *                    cleared afterwards.
 135 */
 136#define RUNNING_MERGE          0
 137#define SHUTDOWN_MERGE         1
 138
 139DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,
 140		"A percentage of time allocated for copy on write");
 141
 142struct dm_dev *dm_snap_origin(struct dm_snapshot *s)
 143{
 144	return s->origin;
 145}
 146EXPORT_SYMBOL(dm_snap_origin);
 147
 148struct dm_dev *dm_snap_cow(struct dm_snapshot *s)
 149{
 150	return s->cow;
 151}
 152EXPORT_SYMBOL(dm_snap_cow);
 153
 154static sector_t chunk_to_sector(struct dm_exception_store *store,
 155				chunk_t chunk)
 156{
 157	return chunk << store->chunk_shift;
 158}
 159
 160static int bdev_equal(struct block_device *lhs, struct block_device *rhs)
 161{
 162	/*
 163	 * There is only ever one instance of a particular block
 164	 * device so we can compare pointers safely.
 165	 */
 166	return lhs == rhs;
 167}
 168
 169struct dm_snap_pending_exception {
 170	struct dm_exception e;
 171
 172	/*
 173	 * Origin buffers waiting for this to complete are held
 174	 * in a bio list
 175	 */
 176	struct bio_list origin_bios;
 177	struct bio_list snapshot_bios;
 178
 179	/* Pointer back to snapshot context */
 180	struct dm_snapshot *snap;
 181
 182	/*
 183	 * 1 indicates the exception has already been sent to
 184	 * kcopyd.
 185	 */
 186	int started;
 187
 188	/* There was copying error. */
 189	int copy_error;
 190
 191	/* A sequence number, it is used for in-order completion. */
 192	sector_t exception_sequence;
 193
 194	struct list_head out_of_order_entry;
 195
 196	/*
 197	 * For writing a complete chunk, bypassing the copy.
 198	 */
 199	struct bio *full_bio;
 200	bio_end_io_t *full_bio_end_io;
 201	void *full_bio_private;
 202};
 203
 204/*
 205 * Hash table mapping origin volumes to lists of snapshots and
 206 * a lock to protect it
 207 */
 208static struct kmem_cache *exception_cache;
 209static struct kmem_cache *pending_cache;
 210
 211struct dm_snap_tracked_chunk {
 212	struct hlist_node node;
 213	chunk_t chunk;
 214};
 215
 216static void init_tracked_chunk(struct bio *bio)
 217{
 218	struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
 219	INIT_HLIST_NODE(&c->node);
 220}
 221
 222static bool is_bio_tracked(struct bio *bio)
 223{
 224	struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
 225	return !hlist_unhashed(&c->node);
 226}
 227
 228static void track_chunk(struct dm_snapshot *s, struct bio *bio, chunk_t chunk)
 229{
 230	struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
 231
 232	c->chunk = chunk;
 233
 234	spin_lock_irq(&s->tracked_chunk_lock);
 235	hlist_add_head(&c->node,
 236		       &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]);
 237	spin_unlock_irq(&s->tracked_chunk_lock);
 238}
 239
 240static void stop_tracking_chunk(struct dm_snapshot *s, struct bio *bio)
 241{
 242	struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
 243	unsigned long flags;
 244
 245	spin_lock_irqsave(&s->tracked_chunk_lock, flags);
 246	hlist_del(&c->node);
 247	spin_unlock_irqrestore(&s->tracked_chunk_lock, flags);
 248}
 249
 250static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk)
 251{
 252	struct dm_snap_tracked_chunk *c;
 253	int found = 0;
 254
 255	spin_lock_irq(&s->tracked_chunk_lock);
 256
 257	hlist_for_each_entry(c,
 258	    &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)], node) {
 259		if (c->chunk == chunk) {
 260			found = 1;
 261			break;
 262		}
 263	}
 264
 265	spin_unlock_irq(&s->tracked_chunk_lock);
 266
 267	return found;
 268}
 269
 270/*
 271 * This conflicting I/O is extremely improbable in the caller,
 272 * so msleep(1) is sufficient and there is no need for a wait queue.
 273 */
 274static void __check_for_conflicting_io(struct dm_snapshot *s, chunk_t chunk)
 275{
 276	while (__chunk_is_tracked(s, chunk))
 277		msleep(1);
 278}
 279
 280/*
 281 * One of these per registered origin, held in the snapshot_origins hash
 282 */
 283struct origin {
 284	/* The origin device */
 285	struct block_device *bdev;
 286
 287	struct list_head hash_list;
 288
 289	/* List of snapshots for this origin */
 290	struct list_head snapshots;
 291};
 292
 293/*
 
 
 
 
 
 
 
 
 
 
 294 * Size of the hash table for origin volumes. If we make this
 295 * the size of the minors list then it should be nearly perfect
 296 */
 297#define ORIGIN_HASH_SIZE 256
 298#define ORIGIN_MASK      0xFF
 299static struct list_head *_origins;
 
 300static struct rw_semaphore _origins_lock;
 301
 302static DECLARE_WAIT_QUEUE_HEAD(_pending_exceptions_done);
 303static DEFINE_SPINLOCK(_pending_exceptions_done_spinlock);
 304static uint64_t _pending_exceptions_done_count;
 305
 306static int init_origin_hash(void)
 307{
 308	int i;
 309
 310	_origins = kmalloc(ORIGIN_HASH_SIZE * sizeof(struct list_head),
 311			   GFP_KERNEL);
 312	if (!_origins) {
 313		DMERR("unable to allocate memory");
 314		return -ENOMEM;
 315	}
 316
 317	for (i = 0; i < ORIGIN_HASH_SIZE; i++)
 318		INIT_LIST_HEAD(_origins + i);
 
 
 
 
 
 
 
 
 
 
 
 319	init_rwsem(&_origins_lock);
 320
 321	return 0;
 322}
 323
 324static void exit_origin_hash(void)
 325{
 326	kfree(_origins);
 
 327}
 328
 329static unsigned origin_hash(struct block_device *bdev)
 330{
 331	return bdev->bd_dev & ORIGIN_MASK;
 332}
 333
 334static struct origin *__lookup_origin(struct block_device *origin)
 335{
 336	struct list_head *ol;
 337	struct origin *o;
 338
 339	ol = &_origins[origin_hash(origin)];
 340	list_for_each_entry (o, ol, hash_list)
 341		if (bdev_equal(o->bdev, origin))
 342			return o;
 343
 344	return NULL;
 345}
 346
 347static void __insert_origin(struct origin *o)
 348{
 349	struct list_head *sl = &_origins[origin_hash(o->bdev)];
 350	list_add_tail(&o->hash_list, sl);
 351}
 352
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 353/*
 354 * _origins_lock must be held when calling this function.
 355 * Returns number of snapshots registered using the supplied cow device, plus:
 356 * snap_src - a snapshot suitable for use as a source of exception handover
 357 * snap_dest - a snapshot capable of receiving exception handover.
 358 * snap_merge - an existing snapshot-merge target linked to the same origin.
 359 *   There can be at most one snapshot-merge target. The parameter is optional.
 360 *
 361 * Possible return values and states of snap_src and snap_dest.
 362 *   0: NULL, NULL  - first new snapshot
 363 *   1: snap_src, NULL - normal snapshot
 364 *   2: snap_src, snap_dest  - waiting for handover
 365 *   2: snap_src, NULL - handed over, waiting for old to be deleted
 366 *   1: NULL, snap_dest - source got destroyed without handover
 367 */
 368static int __find_snapshots_sharing_cow(struct dm_snapshot *snap,
 369					struct dm_snapshot **snap_src,
 370					struct dm_snapshot **snap_dest,
 371					struct dm_snapshot **snap_merge)
 372{
 373	struct dm_snapshot *s;
 374	struct origin *o;
 375	int count = 0;
 376	int active;
 377
 378	o = __lookup_origin(snap->origin->bdev);
 379	if (!o)
 380		goto out;
 381
 382	list_for_each_entry(s, &o->snapshots, list) {
 383		if (dm_target_is_snapshot_merge(s->ti) && snap_merge)
 384			*snap_merge = s;
 385		if (!bdev_equal(s->cow->bdev, snap->cow->bdev))
 386			continue;
 387
 388		down_read(&s->lock);
 389		active = s->active;
 390		up_read(&s->lock);
 391
 392		if (active) {
 393			if (snap_src)
 394				*snap_src = s;
 395		} else if (snap_dest)
 396			*snap_dest = s;
 397
 398		count++;
 399	}
 400
 401out:
 402	return count;
 403}
 404
 405/*
 406 * On success, returns 1 if this snapshot is a handover destination,
 407 * otherwise returns 0.
 408 */
 409static int __validate_exception_handover(struct dm_snapshot *snap)
 410{
 411	struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
 412	struct dm_snapshot *snap_merge = NULL;
 413
 414	/* Does snapshot need exceptions handed over to it? */
 415	if ((__find_snapshots_sharing_cow(snap, &snap_src, &snap_dest,
 416					  &snap_merge) == 2) ||
 417	    snap_dest) {
 418		snap->ti->error = "Snapshot cow pairing for exception "
 419				  "table handover failed";
 420		return -EINVAL;
 421	}
 422
 423	/*
 424	 * If no snap_src was found, snap cannot become a handover
 425	 * destination.
 426	 */
 427	if (!snap_src)
 428		return 0;
 429
 430	/*
 431	 * Non-snapshot-merge handover?
 432	 */
 433	if (!dm_target_is_snapshot_merge(snap->ti))
 434		return 1;
 435
 436	/*
 437	 * Do not allow more than one merging snapshot.
 438	 */
 439	if (snap_merge) {
 440		snap->ti->error = "A snapshot is already merging.";
 441		return -EINVAL;
 442	}
 443
 444	if (!snap_src->store->type->prepare_merge ||
 445	    !snap_src->store->type->commit_merge) {
 446		snap->ti->error = "Snapshot exception store does not "
 447				  "support snapshot-merge.";
 448		return -EINVAL;
 449	}
 450
 451	return 1;
 452}
 453
 454static void __insert_snapshot(struct origin *o, struct dm_snapshot *s)
 455{
 456	struct dm_snapshot *l;
 457
 458	/* Sort the list according to chunk size, largest-first smallest-last */
 459	list_for_each_entry(l, &o->snapshots, list)
 460		if (l->store->chunk_size < s->store->chunk_size)
 461			break;
 462	list_add_tail(&s->list, &l->list);
 463}
 464
 465/*
 466 * Make a note of the snapshot and its origin so we can look it
 467 * up when the origin has a write on it.
 468 *
 469 * Also validate snapshot exception store handovers.
 470 * On success, returns 1 if this registration is a handover destination,
 471 * otherwise returns 0.
 472 */
 473static int register_snapshot(struct dm_snapshot *snap)
 474{
 475	struct origin *o, *new_o = NULL;
 476	struct block_device *bdev = snap->origin->bdev;
 477	int r = 0;
 478
 479	new_o = kmalloc(sizeof(*new_o), GFP_KERNEL);
 480	if (!new_o)
 481		return -ENOMEM;
 482
 483	down_write(&_origins_lock);
 484
 485	r = __validate_exception_handover(snap);
 486	if (r < 0) {
 487		kfree(new_o);
 488		goto out;
 489	}
 490
 491	o = __lookup_origin(bdev);
 492	if (o)
 493		kfree(new_o);
 494	else {
 495		/* New origin */
 496		o = new_o;
 497
 498		/* Initialise the struct */
 499		INIT_LIST_HEAD(&o->snapshots);
 500		o->bdev = bdev;
 501
 502		__insert_origin(o);
 503	}
 504
 505	__insert_snapshot(o, snap);
 506
 507out:
 508	up_write(&_origins_lock);
 509
 510	return r;
 511}
 512
 513/*
 514 * Move snapshot to correct place in list according to chunk size.
 515 */
 516static void reregister_snapshot(struct dm_snapshot *s)
 517{
 518	struct block_device *bdev = s->origin->bdev;
 519
 520	down_write(&_origins_lock);
 521
 522	list_del(&s->list);
 523	__insert_snapshot(__lookup_origin(bdev), s);
 524
 525	up_write(&_origins_lock);
 526}
 527
 528static void unregister_snapshot(struct dm_snapshot *s)
 529{
 530	struct origin *o;
 531
 532	down_write(&_origins_lock);
 533	o = __lookup_origin(s->origin->bdev);
 534
 535	list_del(&s->list);
 536	if (o && list_empty(&o->snapshots)) {
 537		list_del(&o->hash_list);
 538		kfree(o);
 539	}
 540
 541	up_write(&_origins_lock);
 542}
 543
 544/*
 545 * Implementation of the exception hash tables.
 546 * The lowest hash_shift bits of the chunk number are ignored, allowing
 547 * some consecutive chunks to be grouped together.
 548 */
 549static int dm_exception_table_init(struct dm_exception_table *et,
 550				   uint32_t size, unsigned hash_shift)
 551{
 552	unsigned int i;
 553
 554	et->hash_shift = hash_shift;
 555	et->hash_mask = size - 1;
 556	et->table = dm_vcalloc(size, sizeof(struct list_head));
 557	if (!et->table)
 558		return -ENOMEM;
 559
 560	for (i = 0; i < size; i++)
 561		INIT_LIST_HEAD(et->table + i);
 562
 563	return 0;
 564}
 565
 566static void dm_exception_table_exit(struct dm_exception_table *et,
 567				    struct kmem_cache *mem)
 568{
 569	struct list_head *slot;
 570	struct dm_exception *ex, *next;
 571	int i, size;
 572
 573	size = et->hash_mask + 1;
 574	for (i = 0; i < size; i++) {
 575		slot = et->table + i;
 576
 577		list_for_each_entry_safe (ex, next, slot, hash_list)
 578			kmem_cache_free(mem, ex);
 579	}
 580
 581	vfree(et->table);
 582}
 583
 584static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk)
 585{
 586	return (chunk >> et->hash_shift) & et->hash_mask;
 587}
 588
 589static void dm_remove_exception(struct dm_exception *e)
 590{
 591	list_del(&e->hash_list);
 592}
 593
 594/*
 595 * Return the exception data for a sector, or NULL if not
 596 * remapped.
 597 */
 598static struct dm_exception *dm_lookup_exception(struct dm_exception_table *et,
 599						chunk_t chunk)
 600{
 601	struct list_head *slot;
 602	struct dm_exception *e;
 603
 604	slot = &et->table[exception_hash(et, chunk)];
 605	list_for_each_entry (e, slot, hash_list)
 606		if (chunk >= e->old_chunk &&
 607		    chunk <= e->old_chunk + dm_consecutive_chunk_count(e))
 608			return e;
 609
 610	return NULL;
 611}
 612
 613static struct dm_exception *alloc_completed_exception(gfp_t gfp)
 614{
 615	struct dm_exception *e;
 616
 617	e = kmem_cache_alloc(exception_cache, gfp);
 618	if (!e && gfp == GFP_NOIO)
 619		e = kmem_cache_alloc(exception_cache, GFP_ATOMIC);
 620
 621	return e;
 622}
 623
 624static void free_completed_exception(struct dm_exception *e)
 625{
 626	kmem_cache_free(exception_cache, e);
 627}
 628
 629static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s)
 630{
 631	struct dm_snap_pending_exception *pe = mempool_alloc(s->pending_pool,
 632							     GFP_NOIO);
 633
 634	atomic_inc(&s->pending_exceptions_count);
 635	pe->snap = s;
 636
 637	return pe;
 638}
 639
 640static void free_pending_exception(struct dm_snap_pending_exception *pe)
 641{
 642	struct dm_snapshot *s = pe->snap;
 643
 644	mempool_free(pe, s->pending_pool);
 645	smp_mb__before_atomic_dec();
 646	atomic_dec(&s->pending_exceptions_count);
 647}
 648
 649static void dm_insert_exception(struct dm_exception_table *eh,
 650				struct dm_exception *new_e)
 651{
 652	struct list_head *l;
 653	struct dm_exception *e = NULL;
 654
 655	l = &eh->table[exception_hash(eh, new_e->old_chunk)];
 656
 657	/* Add immediately if this table doesn't support consecutive chunks */
 658	if (!eh->hash_shift)
 659		goto out;
 660
 661	/* List is ordered by old_chunk */
 662	list_for_each_entry_reverse(e, l, hash_list) {
 663		/* Insert after an existing chunk? */
 664		if (new_e->old_chunk == (e->old_chunk +
 665					 dm_consecutive_chunk_count(e) + 1) &&
 666		    new_e->new_chunk == (dm_chunk_number(e->new_chunk) +
 667					 dm_consecutive_chunk_count(e) + 1)) {
 668			dm_consecutive_chunk_count_inc(e);
 669			free_completed_exception(new_e);
 670			return;
 671		}
 672
 673		/* Insert before an existing chunk? */
 674		if (new_e->old_chunk == (e->old_chunk - 1) &&
 675		    new_e->new_chunk == (dm_chunk_number(e->new_chunk) - 1)) {
 676			dm_consecutive_chunk_count_inc(e);
 677			e->old_chunk--;
 678			e->new_chunk--;
 679			free_completed_exception(new_e);
 680			return;
 681		}
 682
 683		if (new_e->old_chunk > e->old_chunk)
 684			break;
 685	}
 686
 687out:
 688	list_add(&new_e->hash_list, e ? &e->hash_list : l);
 689}
 690
 691/*
 692 * Callback used by the exception stores to load exceptions when
 693 * initialising.
 694 */
 695static int dm_add_exception(void *context, chunk_t old, chunk_t new)
 696{
 697	struct dm_snapshot *s = context;
 698	struct dm_exception *e;
 699
 700	e = alloc_completed_exception(GFP_KERNEL);
 701	if (!e)
 702		return -ENOMEM;
 703
 704	e->old_chunk = old;
 705
 706	/* Consecutive_count is implicitly initialised to zero */
 707	e->new_chunk = new;
 708
 709	dm_insert_exception(&s->complete, e);
 710
 711	return 0;
 712}
 713
 714/*
 715 * Return a minimum chunk size of all snapshots that have the specified origin.
 716 * Return zero if the origin has no snapshots.
 717 */
 718static uint32_t __minimum_chunk_size(struct origin *o)
 719{
 720	struct dm_snapshot *snap;
 721	unsigned chunk_size = 0;
 722
 723	if (o)
 724		list_for_each_entry(snap, &o->snapshots, list)
 725			chunk_size = min_not_zero(chunk_size,
 726						  snap->store->chunk_size);
 727
 728	return (uint32_t) chunk_size;
 729}
 730
 731/*
 732 * Hard coded magic.
 733 */
 734static int calc_max_buckets(void)
 735{
 736	/* use a fixed size of 2MB */
 737	unsigned long mem = 2 * 1024 * 1024;
 738	mem /= sizeof(struct list_head);
 739
 740	return mem;
 741}
 742
 743/*
 744 * Allocate room for a suitable hash table.
 745 */
 746static int init_hash_tables(struct dm_snapshot *s)
 747{
 748	sector_t hash_size, cow_dev_size, max_buckets;
 749
 750	/*
 751	 * Calculate based on the size of the original volume or
 752	 * the COW volume...
 753	 */
 754	cow_dev_size = get_dev_size(s->cow->bdev);
 755	max_buckets = calc_max_buckets();
 756
 757	hash_size = cow_dev_size >> s->store->chunk_shift;
 758	hash_size = min(hash_size, max_buckets);
 759
 760	if (hash_size < 64)
 761		hash_size = 64;
 762	hash_size = rounddown_pow_of_two(hash_size);
 763	if (dm_exception_table_init(&s->complete, hash_size,
 764				    DM_CHUNK_CONSECUTIVE_BITS))
 765		return -ENOMEM;
 766
 767	/*
 768	 * Allocate hash table for in-flight exceptions
 769	 * Make this smaller than the real hash table
 770	 */
 771	hash_size >>= 3;
 772	if (hash_size < 64)
 773		hash_size = 64;
 774
 775	if (dm_exception_table_init(&s->pending, hash_size, 0)) {
 776		dm_exception_table_exit(&s->complete, exception_cache);
 777		return -ENOMEM;
 778	}
 779
 780	return 0;
 781}
 782
 783static void merge_shutdown(struct dm_snapshot *s)
 784{
 785	clear_bit_unlock(RUNNING_MERGE, &s->state_bits);
 786	smp_mb__after_clear_bit();
 787	wake_up_bit(&s->state_bits, RUNNING_MERGE);
 788}
 789
 790static struct bio *__release_queued_bios_after_merge(struct dm_snapshot *s)
 791{
 792	s->first_merging_chunk = 0;
 793	s->num_merging_chunks = 0;
 794
 795	return bio_list_get(&s->bios_queued_during_merge);
 796}
 797
 798/*
 799 * Remove one chunk from the index of completed exceptions.
 800 */
 801static int __remove_single_exception_chunk(struct dm_snapshot *s,
 802					   chunk_t old_chunk)
 803{
 804	struct dm_exception *e;
 805
 806	e = dm_lookup_exception(&s->complete, old_chunk);
 807	if (!e) {
 808		DMERR("Corruption detected: exception for block %llu is "
 809		      "on disk but not in memory",
 810		      (unsigned long long)old_chunk);
 811		return -EINVAL;
 812	}
 813
 814	/*
 815	 * If this is the only chunk using this exception, remove exception.
 816	 */
 817	if (!dm_consecutive_chunk_count(e)) {
 818		dm_remove_exception(e);
 819		free_completed_exception(e);
 820		return 0;
 821	}
 822
 823	/*
 824	 * The chunk may be either at the beginning or the end of a
 825	 * group of consecutive chunks - never in the middle.  We are
 826	 * removing chunks in the opposite order to that in which they
 827	 * were added, so this should always be true.
 828	 * Decrement the consecutive chunk counter and adjust the
 829	 * starting point if necessary.
 830	 */
 831	if (old_chunk == e->old_chunk) {
 832		e->old_chunk++;
 833		e->new_chunk++;
 834	} else if (old_chunk != e->old_chunk +
 835		   dm_consecutive_chunk_count(e)) {
 836		DMERR("Attempt to merge block %llu from the "
 837		      "middle of a chunk range [%llu - %llu]",
 838		      (unsigned long long)old_chunk,
 839		      (unsigned long long)e->old_chunk,
 840		      (unsigned long long)
 841		      e->old_chunk + dm_consecutive_chunk_count(e));
 842		return -EINVAL;
 843	}
 844
 845	dm_consecutive_chunk_count_dec(e);
 846
 847	return 0;
 848}
 849
 850static void flush_bios(struct bio *bio);
 851
 852static int remove_single_exception_chunk(struct dm_snapshot *s)
 853{
 854	struct bio *b = NULL;
 855	int r;
 856	chunk_t old_chunk = s->first_merging_chunk + s->num_merging_chunks - 1;
 857
 858	down_write(&s->lock);
 859
 860	/*
 861	 * Process chunks (and associated exceptions) in reverse order
 862	 * so that dm_consecutive_chunk_count_dec() accounting works.
 863	 */
 864	do {
 865		r = __remove_single_exception_chunk(s, old_chunk);
 866		if (r)
 867			goto out;
 868	} while (old_chunk-- > s->first_merging_chunk);
 869
 870	b = __release_queued_bios_after_merge(s);
 871
 872out:
 873	up_write(&s->lock);
 874	if (b)
 875		flush_bios(b);
 876
 877	return r;
 878}
 879
 880static int origin_write_extent(struct dm_snapshot *merging_snap,
 881			       sector_t sector, unsigned chunk_size);
 882
 883static void merge_callback(int read_err, unsigned long write_err,
 884			   void *context);
 885
 886static uint64_t read_pending_exceptions_done_count(void)
 887{
 888	uint64_t pending_exceptions_done;
 889
 890	spin_lock(&_pending_exceptions_done_spinlock);
 891	pending_exceptions_done = _pending_exceptions_done_count;
 892	spin_unlock(&_pending_exceptions_done_spinlock);
 893
 894	return pending_exceptions_done;
 895}
 896
 897static void increment_pending_exceptions_done_count(void)
 898{
 899	spin_lock(&_pending_exceptions_done_spinlock);
 900	_pending_exceptions_done_count++;
 901	spin_unlock(&_pending_exceptions_done_spinlock);
 902
 903	wake_up_all(&_pending_exceptions_done);
 904}
 905
 906static void snapshot_merge_next_chunks(struct dm_snapshot *s)
 907{
 908	int i, linear_chunks;
 909	chunk_t old_chunk, new_chunk;
 910	struct dm_io_region src, dest;
 911	sector_t io_size;
 912	uint64_t previous_count;
 913
 914	BUG_ON(!test_bit(RUNNING_MERGE, &s->state_bits));
 915	if (unlikely(test_bit(SHUTDOWN_MERGE, &s->state_bits)))
 916		goto shut;
 917
 918	/*
 919	 * valid flag never changes during merge, so no lock required.
 920	 */
 921	if (!s->valid) {
 922		DMERR("Snapshot is invalid: can't merge");
 923		goto shut;
 924	}
 925
 926	linear_chunks = s->store->type->prepare_merge(s->store, &old_chunk,
 927						      &new_chunk);
 928	if (linear_chunks <= 0) {
 929		if (linear_chunks < 0) {
 930			DMERR("Read error in exception store: "
 931			      "shutting down merge");
 932			down_write(&s->lock);
 933			s->merge_failed = 1;
 934			up_write(&s->lock);
 935		}
 936		goto shut;
 937	}
 938
 939	/* Adjust old_chunk and new_chunk to reflect start of linear region */
 940	old_chunk = old_chunk + 1 - linear_chunks;
 941	new_chunk = new_chunk + 1 - linear_chunks;
 942
 943	/*
 944	 * Use one (potentially large) I/O to copy all 'linear_chunks'
 945	 * from the exception store to the origin
 946	 */
 947	io_size = linear_chunks * s->store->chunk_size;
 948
 949	dest.bdev = s->origin->bdev;
 950	dest.sector = chunk_to_sector(s->store, old_chunk);
 951	dest.count = min(io_size, get_dev_size(dest.bdev) - dest.sector);
 952
 953	src.bdev = s->cow->bdev;
 954	src.sector = chunk_to_sector(s->store, new_chunk);
 955	src.count = dest.count;
 956
 957	/*
 958	 * Reallocate any exceptions needed in other snapshots then
 959	 * wait for the pending exceptions to complete.
 960	 * Each time any pending exception (globally on the system)
 961	 * completes we are woken and repeat the process to find out
 962	 * if we can proceed.  While this may not seem a particularly
 963	 * efficient algorithm, it is not expected to have any
 964	 * significant impact on performance.
 965	 */
 966	previous_count = read_pending_exceptions_done_count();
 967	while (origin_write_extent(s, dest.sector, io_size)) {
 968		wait_event(_pending_exceptions_done,
 969			   (read_pending_exceptions_done_count() !=
 970			    previous_count));
 971		/* Retry after the wait, until all exceptions are done. */
 972		previous_count = read_pending_exceptions_done_count();
 973	}
 974
 975	down_write(&s->lock);
 976	s->first_merging_chunk = old_chunk;
 977	s->num_merging_chunks = linear_chunks;
 978	up_write(&s->lock);
 979
 980	/* Wait until writes to all 'linear_chunks' drain */
 981	for (i = 0; i < linear_chunks; i++)
 982		__check_for_conflicting_io(s, old_chunk + i);
 983
 984	dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, merge_callback, s);
 985	return;
 986
 987shut:
 988	merge_shutdown(s);
 989}
 990
 991static void error_bios(struct bio *bio);
 992
 993static void merge_callback(int read_err, unsigned long write_err, void *context)
 994{
 995	struct dm_snapshot *s = context;
 996	struct bio *b = NULL;
 997
 998	if (read_err || write_err) {
 999		if (read_err)
1000			DMERR("Read error: shutting down merge.");
1001		else
1002			DMERR("Write error: shutting down merge.");
1003		goto shut;
1004	}
1005
1006	if (s->store->type->commit_merge(s->store,
1007					 s->num_merging_chunks) < 0) {
1008		DMERR("Write error in exception store: shutting down merge");
1009		goto shut;
1010	}
1011
1012	if (remove_single_exception_chunk(s) < 0)
1013		goto shut;
1014
1015	snapshot_merge_next_chunks(s);
1016
1017	return;
1018
1019shut:
1020	down_write(&s->lock);
1021	s->merge_failed = 1;
1022	b = __release_queued_bios_after_merge(s);
1023	up_write(&s->lock);
1024	error_bios(b);
1025
1026	merge_shutdown(s);
1027}
1028
1029static void start_merge(struct dm_snapshot *s)
1030{
1031	if (!test_and_set_bit(RUNNING_MERGE, &s->state_bits))
1032		snapshot_merge_next_chunks(s);
1033}
1034
1035static int wait_schedule(void *ptr)
1036{
1037	schedule();
1038
1039	return 0;
1040}
1041
1042/*
1043 * Stop the merging process and wait until it finishes.
1044 */
1045static void stop_merge(struct dm_snapshot *s)
1046{
1047	set_bit(SHUTDOWN_MERGE, &s->state_bits);
1048	wait_on_bit(&s->state_bits, RUNNING_MERGE, wait_schedule,
1049		    TASK_UNINTERRUPTIBLE);
1050	clear_bit(SHUTDOWN_MERGE, &s->state_bits);
1051}
1052
1053/*
1054 * Construct a snapshot mapping: <origin_dev> <COW-dev> <p/n> <chunk-size>
1055 */
1056static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
1057{
1058	struct dm_snapshot *s;
1059	int i;
1060	int r = -EINVAL;
1061	char *origin_path, *cow_path;
 
1062	unsigned args_used, num_flush_bios = 1;
1063	fmode_t origin_mode = FMODE_READ;
1064
1065	if (argc != 4) {
1066		ti->error = "requires exactly 4 arguments";
1067		r = -EINVAL;
1068		goto bad;
1069	}
1070
1071	if (dm_target_is_snapshot_merge(ti)) {
1072		num_flush_bios = 2;
1073		origin_mode = FMODE_WRITE;
1074	}
1075
1076	s = kmalloc(sizeof(*s), GFP_KERNEL);
1077	if (!s) {
1078		ti->error = "Cannot allocate private snapshot structure";
1079		r = -ENOMEM;
1080		goto bad;
1081	}
1082
1083	origin_path = argv[0];
1084	argv++;
1085	argc--;
1086
1087	r = dm_get_device(ti, origin_path, origin_mode, &s->origin);
1088	if (r) {
1089		ti->error = "Cannot get origin device";
1090		goto bad_origin;
1091	}
 
1092
1093	cow_path = argv[0];
1094	argv++;
1095	argc--;
1096
 
 
 
 
 
 
 
1097	r = dm_get_device(ti, cow_path, dm_table_get_mode(ti->table), &s->cow);
1098	if (r) {
1099		ti->error = "Cannot get COW device";
1100		goto bad_cow;
1101	}
1102
1103	r = dm_exception_store_create(ti, argc, argv, s, &args_used, &s->store);
1104	if (r) {
1105		ti->error = "Couldn't create exception store";
1106		r = -EINVAL;
1107		goto bad_store;
1108	}
1109
1110	argv += args_used;
1111	argc -= args_used;
1112
1113	s->ti = ti;
1114	s->valid = 1;
 
1115	s->active = 0;
1116	atomic_set(&s->pending_exceptions_count, 0);
1117	s->exception_start_sequence = 0;
1118	s->exception_complete_sequence = 0;
1119	INIT_LIST_HEAD(&s->out_of_order_list);
1120	init_rwsem(&s->lock);
1121	INIT_LIST_HEAD(&s->list);
1122	spin_lock_init(&s->pe_lock);
1123	s->state_bits = 0;
1124	s->merge_failed = 0;
1125	s->first_merging_chunk = 0;
1126	s->num_merging_chunks = 0;
1127	bio_list_init(&s->bios_queued_during_merge);
1128
1129	/* Allocate hash table for COW data */
1130	if (init_hash_tables(s)) {
1131		ti->error = "Unable to allocate hash table space";
1132		r = -ENOMEM;
1133		goto bad_hash_tables;
1134	}
1135
1136	s->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle);
1137	if (IS_ERR(s->kcopyd_client)) {
1138		r = PTR_ERR(s->kcopyd_client);
1139		ti->error = "Could not create kcopyd client";
1140		goto bad_kcopyd;
1141	}
1142
1143	s->pending_pool = mempool_create_slab_pool(MIN_IOS, pending_cache);
1144	if (!s->pending_pool) {
1145		ti->error = "Could not allocate mempool for pending exceptions";
1146		r = -ENOMEM;
1147		goto bad_pending_pool;
1148	}
1149
1150	for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++)
1151		INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]);
1152
1153	spin_lock_init(&s->tracked_chunk_lock);
1154
1155	ti->private = s;
1156	ti->num_flush_bios = num_flush_bios;
1157	ti->per_bio_data_size = sizeof(struct dm_snap_tracked_chunk);
1158
1159	/* Add snapshot to the list of snapshots for this origin */
1160	/* Exceptions aren't triggered till snapshot_resume() is called */
1161	r = register_snapshot(s);
1162	if (r == -ENOMEM) {
1163		ti->error = "Snapshot origin struct allocation failed";
1164		goto bad_load_and_register;
1165	} else if (r < 0) {
1166		/* invalid handover, register_snapshot has set ti->error */
1167		goto bad_load_and_register;
1168	}
1169
1170	/*
1171	 * Metadata must only be loaded into one table at once, so skip this
1172	 * if metadata will be handed over during resume.
1173	 * Chunk size will be set during the handover - set it to zero to
1174	 * ensure it's ignored.
1175	 */
1176	if (r > 0) {
1177		s->store->chunk_size = 0;
1178		return 0;
1179	}
1180
1181	r = s->store->type->read_metadata(s->store, dm_add_exception,
1182					  (void *)s);
1183	if (r < 0) {
1184		ti->error = "Failed to read snapshot metadata";
1185		goto bad_read_metadata;
1186	} else if (r > 0) {
1187		s->valid = 0;
1188		DMWARN("Snapshot is marked invalid.");
1189	}
1190
1191	if (!s->store->chunk_size) {
1192		ti->error = "Chunk size not set";
1193		goto bad_read_metadata;
1194	}
1195
1196	r = dm_set_target_max_io_len(ti, s->store->chunk_size);
1197	if (r)
1198		goto bad_read_metadata;
1199
1200	return 0;
1201
1202bad_read_metadata:
1203	unregister_snapshot(s);
1204
1205bad_load_and_register:
1206	mempool_destroy(s->pending_pool);
1207
1208bad_pending_pool:
1209	dm_kcopyd_client_destroy(s->kcopyd_client);
1210
1211bad_kcopyd:
1212	dm_exception_table_exit(&s->pending, pending_cache);
1213	dm_exception_table_exit(&s->complete, exception_cache);
1214
1215bad_hash_tables:
1216	dm_exception_store_destroy(s->store);
1217
1218bad_store:
1219	dm_put_device(ti, s->cow);
1220
1221bad_cow:
1222	dm_put_device(ti, s->origin);
1223
1224bad_origin:
1225	kfree(s);
1226
1227bad:
1228	return r;
1229}
1230
1231static void __free_exceptions(struct dm_snapshot *s)
1232{
1233	dm_kcopyd_client_destroy(s->kcopyd_client);
1234	s->kcopyd_client = NULL;
1235
1236	dm_exception_table_exit(&s->pending, pending_cache);
1237	dm_exception_table_exit(&s->complete, exception_cache);
1238}
1239
1240static void __handover_exceptions(struct dm_snapshot *snap_src,
1241				  struct dm_snapshot *snap_dest)
1242{
1243	union {
1244		struct dm_exception_table table_swap;
1245		struct dm_exception_store *store_swap;
1246	} u;
1247
1248	/*
1249	 * Swap all snapshot context information between the two instances.
1250	 */
1251	u.table_swap = snap_dest->complete;
1252	snap_dest->complete = snap_src->complete;
1253	snap_src->complete = u.table_swap;
1254
1255	u.store_swap = snap_dest->store;
1256	snap_dest->store = snap_src->store;
 
1257	snap_src->store = u.store_swap;
1258
1259	snap_dest->store->snap = snap_dest;
1260	snap_src->store->snap = snap_src;
1261
1262	snap_dest->ti->max_io_len = snap_dest->store->chunk_size;
1263	snap_dest->valid = snap_src->valid;
 
1264
1265	/*
1266	 * Set source invalid to ensure it receives no further I/O.
1267	 */
1268	snap_src->valid = 0;
1269}
1270
1271static void snapshot_dtr(struct dm_target *ti)
1272{
1273#ifdef CONFIG_DM_DEBUG
1274	int i;
1275#endif
1276	struct dm_snapshot *s = ti->private;
1277	struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
1278
1279	down_read(&_origins_lock);
1280	/* Check whether exception handover must be cancelled */
1281	(void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
1282	if (snap_src && snap_dest && (s == snap_src)) {
1283		down_write(&snap_dest->lock);
1284		snap_dest->valid = 0;
1285		up_write(&snap_dest->lock);
1286		DMERR("Cancelling snapshot handover.");
1287	}
1288	up_read(&_origins_lock);
1289
1290	if (dm_target_is_snapshot_merge(ti))
1291		stop_merge(s);
1292
1293	/* Prevent further origin writes from using this snapshot. */
1294	/* After this returns there can be no new kcopyd jobs. */
1295	unregister_snapshot(s);
1296
1297	while (atomic_read(&s->pending_exceptions_count))
1298		msleep(1);
1299	/*
1300	 * Ensure instructions in mempool_destroy aren't reordered
1301	 * before atomic_read.
1302	 */
1303	smp_mb();
1304
1305#ifdef CONFIG_DM_DEBUG
1306	for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++)
1307		BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i]));
1308#endif
1309
1310	__free_exceptions(s);
1311
1312	mempool_destroy(s->pending_pool);
1313
1314	dm_exception_store_destroy(s->store);
1315
1316	dm_put_device(ti, s->cow);
1317
1318	dm_put_device(ti, s->origin);
1319
1320	kfree(s);
1321}
1322
1323/*
1324 * Flush a list of buffers.
1325 */
1326static void flush_bios(struct bio *bio)
1327{
1328	struct bio *n;
1329
1330	while (bio) {
1331		n = bio->bi_next;
1332		bio->bi_next = NULL;
1333		generic_make_request(bio);
1334		bio = n;
1335	}
1336}
1337
1338static int do_origin(struct dm_dev *origin, struct bio *bio);
1339
1340/*
1341 * Flush a list of buffers.
1342 */
1343static void retry_origin_bios(struct dm_snapshot *s, struct bio *bio)
1344{
1345	struct bio *n;
1346	int r;
1347
1348	while (bio) {
1349		n = bio->bi_next;
1350		bio->bi_next = NULL;
1351		r = do_origin(s->origin, bio);
1352		if (r == DM_MAPIO_REMAPPED)
1353			generic_make_request(bio);
1354		bio = n;
1355	}
1356}
1357
1358/*
1359 * Error a list of buffers.
1360 */
1361static void error_bios(struct bio *bio)
1362{
1363	struct bio *n;
1364
1365	while (bio) {
1366		n = bio->bi_next;
1367		bio->bi_next = NULL;
1368		bio_io_error(bio);
1369		bio = n;
1370	}
1371}
1372
1373static void __invalidate_snapshot(struct dm_snapshot *s, int err)
1374{
1375	if (!s->valid)
1376		return;
1377
1378	if (err == -EIO)
1379		DMERR("Invalidating snapshot: Error reading/writing.");
1380	else if (err == -ENOMEM)
1381		DMERR("Invalidating snapshot: Unable to allocate exception.");
1382
1383	if (s->store->type->drop_snapshot)
1384		s->store->type->drop_snapshot(s->store);
1385
1386	s->valid = 0;
1387
1388	dm_table_event(s->ti->table);
1389}
1390
1391static void pending_complete(struct dm_snap_pending_exception *pe, int success)
1392{
 
1393	struct dm_exception *e;
1394	struct dm_snapshot *s = pe->snap;
1395	struct bio *origin_bios = NULL;
1396	struct bio *snapshot_bios = NULL;
1397	struct bio *full_bio = NULL;
1398	int error = 0;
1399
1400	if (!success) {
1401		/* Read/write error - snapshot is unusable */
1402		down_write(&s->lock);
1403		__invalidate_snapshot(s, -EIO);
1404		error = 1;
1405		goto out;
1406	}
1407
1408	e = alloc_completed_exception(GFP_NOIO);
1409	if (!e) {
1410		down_write(&s->lock);
1411		__invalidate_snapshot(s, -ENOMEM);
1412		error = 1;
1413		goto out;
1414	}
1415	*e = pe->e;
1416
1417	down_write(&s->lock);
1418	if (!s->valid) {
1419		free_completed_exception(e);
1420		error = 1;
1421		goto out;
1422	}
1423
1424	/* Check for conflicting reads */
1425	__check_for_conflicting_io(s, pe->e.old_chunk);
1426
1427	/*
1428	 * Add a proper exception, and remove the
1429	 * in-flight exception from the list.
1430	 */
1431	dm_insert_exception(&s->complete, e);
1432
1433out:
1434	dm_remove_exception(&pe->e);
1435	snapshot_bios = bio_list_get(&pe->snapshot_bios);
1436	origin_bios = bio_list_get(&pe->origin_bios);
1437	full_bio = pe->full_bio;
1438	if (full_bio) {
1439		full_bio->bi_end_io = pe->full_bio_end_io;
1440		full_bio->bi_private = pe->full_bio_private;
1441		atomic_inc(&full_bio->bi_remaining);
1442	}
1443	free_pending_exception(pe);
1444
1445	increment_pending_exceptions_done_count();
1446
1447	up_write(&s->lock);
1448
1449	/* Submit any pending write bios */
1450	if (error) {
1451		if (full_bio)
1452			bio_io_error(full_bio);
1453		error_bios(snapshot_bios);
1454	} else {
1455		if (full_bio)
1456			bio_endio(full_bio, 0);
1457		flush_bios(snapshot_bios);
1458	}
1459
1460	retry_origin_bios(s, origin_bios);
1461}
1462
1463static void commit_callback(void *context, int success)
1464{
1465	struct dm_snap_pending_exception *pe = context;
1466
1467	pending_complete(pe, success);
1468}
1469
1470static void complete_exception(struct dm_snap_pending_exception *pe)
1471{
1472	struct dm_snapshot *s = pe->snap;
1473
1474	if (unlikely(pe->copy_error))
1475		pending_complete(pe, 0);
1476
1477	else
1478		/* Update the metadata if we are persistent */
1479		s->store->type->commit_exception(s->store, &pe->e,
1480						 commit_callback, pe);
1481}
1482
1483/*
1484 * Called when the copy I/O has finished.  kcopyd actually runs
1485 * this code so don't block.
1486 */
1487static void copy_callback(int read_err, unsigned long write_err, void *context)
1488{
1489	struct dm_snap_pending_exception *pe = context;
1490	struct dm_snapshot *s = pe->snap;
1491
1492	pe->copy_error = read_err || write_err;
1493
1494	if (pe->exception_sequence == s->exception_complete_sequence) {
1495		s->exception_complete_sequence++;
1496		complete_exception(pe);
1497
1498		while (!list_empty(&s->out_of_order_list)) {
1499			pe = list_entry(s->out_of_order_list.next,
1500					struct dm_snap_pending_exception, out_of_order_entry);
1501			if (pe->exception_sequence != s->exception_complete_sequence)
1502				break;
1503			s->exception_complete_sequence++;
1504			list_del(&pe->out_of_order_entry);
1505			complete_exception(pe);
1506		}
1507	} else {
1508		struct list_head *lh;
1509		struct dm_snap_pending_exception *pe2;
1510
1511		list_for_each_prev(lh, &s->out_of_order_list) {
1512			pe2 = list_entry(lh, struct dm_snap_pending_exception, out_of_order_entry);
1513			if (pe2->exception_sequence < pe->exception_sequence)
1514				break;
1515		}
1516		list_add(&pe->out_of_order_entry, lh);
1517	}
1518}
1519
1520/*
1521 * Dispatches the copy operation to kcopyd.
1522 */
1523static void start_copy(struct dm_snap_pending_exception *pe)
1524{
1525	struct dm_snapshot *s = pe->snap;
1526	struct dm_io_region src, dest;
1527	struct block_device *bdev = s->origin->bdev;
1528	sector_t dev_size;
1529
1530	dev_size = get_dev_size(bdev);
1531
1532	src.bdev = bdev;
1533	src.sector = chunk_to_sector(s->store, pe->e.old_chunk);
1534	src.count = min((sector_t)s->store->chunk_size, dev_size - src.sector);
1535
1536	dest.bdev = s->cow->bdev;
1537	dest.sector = chunk_to_sector(s->store, pe->e.new_chunk);
1538	dest.count = src.count;
1539
1540	/* Hand over to kcopyd */
1541	dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe);
1542}
1543
1544static void full_bio_end_io(struct bio *bio, int error)
1545{
1546	void *callback_data = bio->bi_private;
1547
1548	dm_kcopyd_do_callback(callback_data, 0, error ? 1 : 0);
1549}
1550
1551static void start_full_bio(struct dm_snap_pending_exception *pe,
1552			   struct bio *bio)
1553{
1554	struct dm_snapshot *s = pe->snap;
1555	void *callback_data;
1556
1557	pe->full_bio = bio;
1558	pe->full_bio_end_io = bio->bi_end_io;
1559	pe->full_bio_private = bio->bi_private;
1560
1561	callback_data = dm_kcopyd_prepare_callback(s->kcopyd_client,
1562						   copy_callback, pe);
1563
1564	bio->bi_end_io = full_bio_end_io;
1565	bio->bi_private = callback_data;
1566
1567	generic_make_request(bio);
1568}
1569
1570static struct dm_snap_pending_exception *
1571__lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk)
1572{
1573	struct dm_exception *e = dm_lookup_exception(&s->pending, chunk);
1574
1575	if (!e)
1576		return NULL;
1577
1578	return container_of(e, struct dm_snap_pending_exception, e);
1579}
1580
1581/*
1582 * Looks to see if this snapshot already has a pending exception
1583 * for this chunk, otherwise it allocates a new one and inserts
1584 * it into the pending table.
1585 *
1586 * NOTE: a write lock must be held on snap->lock before calling
1587 * this.
1588 */
1589static struct dm_snap_pending_exception *
1590__find_pending_exception(struct dm_snapshot *s,
1591			 struct dm_snap_pending_exception *pe, chunk_t chunk)
1592{
1593	struct dm_snap_pending_exception *pe2;
1594
1595	pe2 = __lookup_pending_exception(s, chunk);
1596	if (pe2) {
1597		free_pending_exception(pe);
1598		return pe2;
1599	}
1600
1601	pe->e.old_chunk = chunk;
1602	bio_list_init(&pe->origin_bios);
1603	bio_list_init(&pe->snapshot_bios);
1604	pe->started = 0;
1605	pe->full_bio = NULL;
1606
1607	if (s->store->type->prepare_exception(s->store, &pe->e)) {
1608		free_pending_exception(pe);
1609		return NULL;
1610	}
1611
1612	pe->exception_sequence = s->exception_start_sequence++;
1613
1614	dm_insert_exception(&s->pending, &pe->e);
1615
1616	return pe;
1617}
1618
1619static void remap_exception(struct dm_snapshot *s, struct dm_exception *e,
1620			    struct bio *bio, chunk_t chunk)
1621{
1622	bio->bi_bdev = s->cow->bdev;
1623	bio->bi_iter.bi_sector =
1624		chunk_to_sector(s->store, dm_chunk_number(e->new_chunk) +
1625				(chunk - e->old_chunk)) +
1626		(bio->bi_iter.bi_sector & s->store->chunk_mask);
1627}
1628
1629static int snapshot_map(struct dm_target *ti, struct bio *bio)
1630{
1631	struct dm_exception *e;
1632	struct dm_snapshot *s = ti->private;
1633	int r = DM_MAPIO_REMAPPED;
1634	chunk_t chunk;
1635	struct dm_snap_pending_exception *pe = NULL;
1636
1637	init_tracked_chunk(bio);
1638
1639	if (bio->bi_rw & REQ_FLUSH) {
1640		bio->bi_bdev = s->cow->bdev;
1641		return DM_MAPIO_REMAPPED;
1642	}
1643
1644	chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector);
1645
1646	/* Full snapshots are not usable */
1647	/* To get here the table must be live so s->active is always set. */
1648	if (!s->valid)
1649		return -EIO;
1650
1651	/* FIXME: should only take write lock if we need
1652	 * to copy an exception */
1653	down_write(&s->lock);
1654
1655	if (!s->valid) {
1656		r = -EIO;
1657		goto out_unlock;
1658	}
1659
1660	/* If the block is already remapped - use that, else remap it */
1661	e = dm_lookup_exception(&s->complete, chunk);
1662	if (e) {
1663		remap_exception(s, e, bio, chunk);
1664		goto out_unlock;
1665	}
1666
1667	/*
1668	 * Write to snapshot - higher level takes care of RW/RO
1669	 * flags so we should only get this if we are
1670	 * writeable.
1671	 */
1672	if (bio_rw(bio) == WRITE) {
1673		pe = __lookup_pending_exception(s, chunk);
1674		if (!pe) {
1675			up_write(&s->lock);
1676			pe = alloc_pending_exception(s);
1677			down_write(&s->lock);
1678
1679			if (!s->valid) {
1680				free_pending_exception(pe);
1681				r = -EIO;
1682				goto out_unlock;
1683			}
1684
1685			e = dm_lookup_exception(&s->complete, chunk);
1686			if (e) {
1687				free_pending_exception(pe);
1688				remap_exception(s, e, bio, chunk);
1689				goto out_unlock;
1690			}
1691
1692			pe = __find_pending_exception(s, pe, chunk);
1693			if (!pe) {
1694				__invalidate_snapshot(s, -ENOMEM);
 
 
 
 
1695				r = -EIO;
1696				goto out_unlock;
1697			}
1698		}
1699
1700		remap_exception(s, &pe->e, bio, chunk);
1701
1702		r = DM_MAPIO_SUBMITTED;
1703
1704		if (!pe->started &&
1705		    bio->bi_iter.bi_size ==
1706		    (s->store->chunk_size << SECTOR_SHIFT)) {
1707			pe->started = 1;
1708			up_write(&s->lock);
1709			start_full_bio(pe, bio);
1710			goto out;
1711		}
1712
1713		bio_list_add(&pe->snapshot_bios, bio);
1714
1715		if (!pe->started) {
1716			/* this is protected by snap->lock */
1717			pe->started = 1;
1718			up_write(&s->lock);
1719			start_copy(pe);
1720			goto out;
1721		}
1722	} else {
1723		bio->bi_bdev = s->origin->bdev;
1724		track_chunk(s, bio, chunk);
1725	}
1726
1727out_unlock:
1728	up_write(&s->lock);
1729out:
1730	return r;
1731}
1732
1733/*
1734 * A snapshot-merge target behaves like a combination of a snapshot
1735 * target and a snapshot-origin target.  It only generates new
1736 * exceptions in other snapshots and not in the one that is being
1737 * merged.
1738 *
1739 * For each chunk, if there is an existing exception, it is used to
1740 * redirect I/O to the cow device.  Otherwise I/O is sent to the origin,
1741 * which in turn might generate exceptions in other snapshots.
1742 * If merging is currently taking place on the chunk in question, the
1743 * I/O is deferred by adding it to s->bios_queued_during_merge.
1744 */
1745static int snapshot_merge_map(struct dm_target *ti, struct bio *bio)
1746{
1747	struct dm_exception *e;
1748	struct dm_snapshot *s = ti->private;
1749	int r = DM_MAPIO_REMAPPED;
1750	chunk_t chunk;
1751
1752	init_tracked_chunk(bio);
1753
1754	if (bio->bi_rw & REQ_FLUSH) {
1755		if (!dm_bio_get_target_bio_nr(bio))
1756			bio->bi_bdev = s->origin->bdev;
1757		else
1758			bio->bi_bdev = s->cow->bdev;
1759		return DM_MAPIO_REMAPPED;
1760	}
1761
1762	chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector);
1763
1764	down_write(&s->lock);
1765
1766	/* Full merging snapshots are redirected to the origin */
1767	if (!s->valid)
1768		goto redirect_to_origin;
1769
1770	/* If the block is already remapped - use that */
1771	e = dm_lookup_exception(&s->complete, chunk);
1772	if (e) {
1773		/* Queue writes overlapping with chunks being merged */
1774		if (bio_rw(bio) == WRITE &&
1775		    chunk >= s->first_merging_chunk &&
1776		    chunk < (s->first_merging_chunk +
1777			     s->num_merging_chunks)) {
1778			bio->bi_bdev = s->origin->bdev;
1779			bio_list_add(&s->bios_queued_during_merge, bio);
1780			r = DM_MAPIO_SUBMITTED;
1781			goto out_unlock;
1782		}
1783
1784		remap_exception(s, e, bio, chunk);
1785
1786		if (bio_rw(bio) == WRITE)
1787			track_chunk(s, bio, chunk);
1788		goto out_unlock;
1789	}
1790
1791redirect_to_origin:
1792	bio->bi_bdev = s->origin->bdev;
1793
1794	if (bio_rw(bio) == WRITE) {
1795		up_write(&s->lock);
1796		return do_origin(s->origin, bio);
1797	}
1798
1799out_unlock:
1800	up_write(&s->lock);
1801
1802	return r;
1803}
1804
1805static int snapshot_end_io(struct dm_target *ti, struct bio *bio, int error)
1806{
1807	struct dm_snapshot *s = ti->private;
1808
1809	if (is_bio_tracked(bio))
1810		stop_tracking_chunk(s, bio);
1811
1812	return 0;
1813}
1814
1815static void snapshot_merge_presuspend(struct dm_target *ti)
1816{
1817	struct dm_snapshot *s = ti->private;
1818
1819	stop_merge(s);
1820}
1821
1822static int snapshot_preresume(struct dm_target *ti)
1823{
1824	int r = 0;
1825	struct dm_snapshot *s = ti->private;
1826	struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
1827
1828	down_read(&_origins_lock);
1829	(void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
1830	if (snap_src && snap_dest) {
1831		down_read(&snap_src->lock);
1832		if (s == snap_src) {
1833			DMERR("Unable to resume snapshot source until "
1834			      "handover completes.");
1835			r = -EINVAL;
1836		} else if (!dm_suspended(snap_src->ti)) {
1837			DMERR("Unable to perform snapshot handover until "
1838			      "source is suspended.");
1839			r = -EINVAL;
1840		}
1841		up_read(&snap_src->lock);
1842	}
1843	up_read(&_origins_lock);
1844
1845	return r;
1846}
1847
1848static void snapshot_resume(struct dm_target *ti)
1849{
1850	struct dm_snapshot *s = ti->private;
1851	struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1852
1853	down_read(&_origins_lock);
 
1854	(void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
1855	if (snap_src && snap_dest) {
1856		down_write(&snap_src->lock);
1857		down_write_nested(&snap_dest->lock, SINGLE_DEPTH_NESTING);
1858		__handover_exceptions(snap_src, snap_dest);
1859		up_write(&snap_dest->lock);
1860		up_write(&snap_src->lock);
1861	}
 
1862	up_read(&_origins_lock);
1863
 
 
 
 
 
 
 
1864	/* Now we have correct chunk size, reregister */
1865	reregister_snapshot(s);
1866
1867	down_write(&s->lock);
1868	s->active = 1;
1869	up_write(&s->lock);
1870}
1871
1872static uint32_t get_origin_minimum_chunksize(struct block_device *bdev)
1873{
1874	uint32_t min_chunksize;
1875
1876	down_read(&_origins_lock);
1877	min_chunksize = __minimum_chunk_size(__lookup_origin(bdev));
1878	up_read(&_origins_lock);
1879
1880	return min_chunksize;
1881}
1882
1883static void snapshot_merge_resume(struct dm_target *ti)
1884{
1885	struct dm_snapshot *s = ti->private;
1886
1887	/*
1888	 * Handover exceptions from existing snapshot.
1889	 */
1890	snapshot_resume(ti);
1891
1892	/*
1893	 * snapshot-merge acts as an origin, so set ti->max_io_len
1894	 */
1895	ti->max_io_len = get_origin_minimum_chunksize(s->origin->bdev);
1896
1897	start_merge(s);
1898}
1899
1900static void snapshot_status(struct dm_target *ti, status_type_t type,
1901			    unsigned status_flags, char *result, unsigned maxlen)
1902{
1903	unsigned sz = 0;
1904	struct dm_snapshot *snap = ti->private;
1905
1906	switch (type) {
1907	case STATUSTYPE_INFO:
1908
1909		down_write(&snap->lock);
1910
1911		if (!snap->valid)
1912			DMEMIT("Invalid");
1913		else if (snap->merge_failed)
1914			DMEMIT("Merge failed");
 
 
1915		else {
1916			if (snap->store->type->usage) {
1917				sector_t total_sectors, sectors_allocated,
1918					 metadata_sectors;
1919				snap->store->type->usage(snap->store,
1920							 &total_sectors,
1921							 &sectors_allocated,
1922							 &metadata_sectors);
1923				DMEMIT("%llu/%llu %llu",
1924				       (unsigned long long)sectors_allocated,
1925				       (unsigned long long)total_sectors,
1926				       (unsigned long long)metadata_sectors);
1927			}
1928			else
1929				DMEMIT("Unknown");
1930		}
1931
1932		up_write(&snap->lock);
1933
1934		break;
1935
1936	case STATUSTYPE_TABLE:
1937		/*
1938		 * kdevname returns a static pointer so we need
1939		 * to make private copies if the output is to
1940		 * make sense.
1941		 */
1942		DMEMIT("%s %s", snap->origin->name, snap->cow->name);
1943		snap->store->type->status(snap->store, type, result + sz,
1944					  maxlen - sz);
1945		break;
1946	}
1947}
1948
1949static int snapshot_iterate_devices(struct dm_target *ti,
1950				    iterate_devices_callout_fn fn, void *data)
1951{
1952	struct dm_snapshot *snap = ti->private;
1953	int r;
1954
1955	r = fn(ti, snap->origin, 0, ti->len, data);
1956
1957	if (!r)
1958		r = fn(ti, snap->cow, 0, get_dev_size(snap->cow->bdev), data);
1959
1960	return r;
1961}
1962
1963
1964/*-----------------------------------------------------------------
1965 * Origin methods
1966 *---------------------------------------------------------------*/
1967
1968/*
1969 * If no exceptions need creating, DM_MAPIO_REMAPPED is returned and any
1970 * supplied bio was ignored.  The caller may submit it immediately.
1971 * (No remapping actually occurs as the origin is always a direct linear
1972 * map.)
1973 *
1974 * If further exceptions are required, DM_MAPIO_SUBMITTED is returned
1975 * and any supplied bio is added to a list to be submitted once all
1976 * the necessary exceptions exist.
1977 */
1978static int __origin_write(struct list_head *snapshots, sector_t sector,
1979			  struct bio *bio)
1980{
1981	int r = DM_MAPIO_REMAPPED;
1982	struct dm_snapshot *snap;
1983	struct dm_exception *e;
1984	struct dm_snap_pending_exception *pe;
1985	struct dm_snap_pending_exception *pe_to_start_now = NULL;
1986	struct dm_snap_pending_exception *pe_to_start_last = NULL;
1987	chunk_t chunk;
1988
1989	/* Do all the snapshots on this origin */
1990	list_for_each_entry (snap, snapshots, list) {
1991		/*
1992		 * Don't make new exceptions in a merging snapshot
1993		 * because it has effectively been deleted
1994		 */
1995		if (dm_target_is_snapshot_merge(snap->ti))
1996			continue;
1997
1998		down_write(&snap->lock);
1999
2000		/* Only deal with valid and active snapshots */
2001		if (!snap->valid || !snap->active)
2002			goto next_snapshot;
2003
2004		/* Nothing to do if writing beyond end of snapshot */
2005		if (sector >= dm_table_get_size(snap->ti->table))
2006			goto next_snapshot;
2007
2008		/*
2009		 * Remember, different snapshots can have
2010		 * different chunk sizes.
2011		 */
2012		chunk = sector_to_chunk(snap->store, sector);
2013
2014		/*
2015		 * Check exception table to see if block
2016		 * is already remapped in this snapshot
2017		 * and trigger an exception if not.
2018		 */
2019		e = dm_lookup_exception(&snap->complete, chunk);
2020		if (e)
2021			goto next_snapshot;
2022
2023		pe = __lookup_pending_exception(snap, chunk);
2024		if (!pe) {
2025			up_write(&snap->lock);
2026			pe = alloc_pending_exception(snap);
2027			down_write(&snap->lock);
2028
2029			if (!snap->valid) {
2030				free_pending_exception(pe);
2031				goto next_snapshot;
2032			}
2033
2034			e = dm_lookup_exception(&snap->complete, chunk);
2035			if (e) {
2036				free_pending_exception(pe);
2037				goto next_snapshot;
2038			}
2039
2040			pe = __find_pending_exception(snap, pe, chunk);
2041			if (!pe) {
2042				__invalidate_snapshot(snap, -ENOMEM);
2043				goto next_snapshot;
2044			}
2045		}
2046
2047		r = DM_MAPIO_SUBMITTED;
2048
2049		/*
2050		 * If an origin bio was supplied, queue it to wait for the
2051		 * completion of this exception, and start this one last,
2052		 * at the end of the function.
2053		 */
2054		if (bio) {
2055			bio_list_add(&pe->origin_bios, bio);
2056			bio = NULL;
2057
2058			if (!pe->started) {
2059				pe->started = 1;
2060				pe_to_start_last = pe;
2061			}
2062		}
2063
2064		if (!pe->started) {
2065			pe->started = 1;
2066			pe_to_start_now = pe;
2067		}
2068
2069next_snapshot:
2070		up_write(&snap->lock);
2071
2072		if (pe_to_start_now) {
2073			start_copy(pe_to_start_now);
2074			pe_to_start_now = NULL;
2075		}
2076	}
2077
2078	/*
2079	 * Submit the exception against which the bio is queued last,
2080	 * to give the other exceptions a head start.
2081	 */
2082	if (pe_to_start_last)
2083		start_copy(pe_to_start_last);
2084
2085	return r;
2086}
2087
2088/*
2089 * Called on a write from the origin driver.
2090 */
2091static int do_origin(struct dm_dev *origin, struct bio *bio)
2092{
2093	struct origin *o;
2094	int r = DM_MAPIO_REMAPPED;
2095
2096	down_read(&_origins_lock);
2097	o = __lookup_origin(origin->bdev);
2098	if (o)
2099		r = __origin_write(&o->snapshots, bio->bi_iter.bi_sector, bio);
2100	up_read(&_origins_lock);
2101
2102	return r;
2103}
2104
2105/*
2106 * Trigger exceptions in all non-merging snapshots.
2107 *
2108 * The chunk size of the merging snapshot may be larger than the chunk
2109 * size of some other snapshot so we may need to reallocate multiple
2110 * chunks in other snapshots.
2111 *
2112 * We scan all the overlapping exceptions in the other snapshots.
2113 * Returns 1 if anything was reallocated and must be waited for,
2114 * otherwise returns 0.
2115 *
2116 * size must be a multiple of merging_snap's chunk_size.
2117 */
2118static int origin_write_extent(struct dm_snapshot *merging_snap,
2119			       sector_t sector, unsigned size)
2120{
2121	int must_wait = 0;
2122	sector_t n;
2123	struct origin *o;
2124
2125	/*
2126	 * The origin's __minimum_chunk_size() got stored in max_io_len
2127	 * by snapshot_merge_resume().
2128	 */
2129	down_read(&_origins_lock);
2130	o = __lookup_origin(merging_snap->origin->bdev);
2131	for (n = 0; n < size; n += merging_snap->ti->max_io_len)
2132		if (__origin_write(&o->snapshots, sector + n, NULL) ==
2133		    DM_MAPIO_SUBMITTED)
2134			must_wait = 1;
2135	up_read(&_origins_lock);
2136
2137	return must_wait;
2138}
2139
2140/*
2141 * Origin: maps a linear range of a device, with hooks for snapshotting.
2142 */
2143
2144/*
2145 * Construct an origin mapping: <dev_path>
2146 * The context for an origin is merely a 'struct dm_dev *'
2147 * pointing to the real device.
2148 */
2149static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv)
2150{
2151	int r;
2152	struct dm_dev *dev;
2153
2154	if (argc != 1) {
2155		ti->error = "origin: incorrect number of arguments";
2156		return -EINVAL;
2157	}
2158
2159	r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &dev);
 
 
 
 
 
 
 
2160	if (r) {
2161		ti->error = "Cannot get target device";
2162		return r;
2163	}
2164
2165	ti->private = dev;
 
2166	ti->num_flush_bios = 1;
2167
2168	return 0;
 
 
 
 
 
2169}
2170
2171static void origin_dtr(struct dm_target *ti)
2172{
2173	struct dm_dev *dev = ti->private;
2174	dm_put_device(ti, dev);
 
 
2175}
2176
2177static int origin_map(struct dm_target *ti, struct bio *bio)
2178{
2179	struct dm_dev *dev = ti->private;
2180	bio->bi_bdev = dev->bdev;
 
 
 
 
 
2181
2182	if (bio->bi_rw & REQ_FLUSH)
2183		return DM_MAPIO_REMAPPED;
2184
 
 
 
 
 
 
2185	/* Only tell snapshots if this is a write */
2186	return (bio_rw(bio) == WRITE) ? do_origin(dev, bio) : DM_MAPIO_REMAPPED;
2187}
2188
2189/*
2190 * Set the target "max_io_len" field to the minimum of all the snapshots'
2191 * chunk sizes.
2192 */
2193static void origin_resume(struct dm_target *ti)
2194{
2195	struct dm_dev *dev = ti->private;
 
 
2196
2197	ti->max_io_len = get_origin_minimum_chunksize(dev->bdev);
 
 
 
 
 
 
 
 
 
 
 
2198}
2199
2200static void origin_status(struct dm_target *ti, status_type_t type,
2201			  unsigned status_flags, char *result, unsigned maxlen)
2202{
2203	struct dm_dev *dev = ti->private;
2204
2205	switch (type) {
2206	case STATUSTYPE_INFO:
2207		result[0] = '\0';
2208		break;
2209
2210	case STATUSTYPE_TABLE:
2211		snprintf(result, maxlen, "%s", dev->name);
2212		break;
2213	}
2214}
2215
2216static int origin_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
2217			struct bio_vec *biovec, int max_size)
2218{
2219	struct dm_dev *dev = ti->private;
2220	struct request_queue *q = bdev_get_queue(dev->bdev);
2221
2222	if (!q->merge_bvec_fn)
2223		return max_size;
2224
2225	bvm->bi_bdev = dev->bdev;
2226
2227	return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
2228}
2229
2230static int origin_iterate_devices(struct dm_target *ti,
2231				  iterate_devices_callout_fn fn, void *data)
2232{
2233	struct dm_dev *dev = ti->private;
2234
2235	return fn(ti, dev, 0, ti->len, data);
2236}
2237
2238static struct target_type origin_target = {
2239	.name    = "snapshot-origin",
2240	.version = {1, 8, 1},
2241	.module  = THIS_MODULE,
2242	.ctr     = origin_ctr,
2243	.dtr     = origin_dtr,
2244	.map     = origin_map,
2245	.resume  = origin_resume,
 
2246	.status  = origin_status,
2247	.merge	 = origin_merge,
2248	.iterate_devices = origin_iterate_devices,
2249};
2250
2251static struct target_type snapshot_target = {
2252	.name    = "snapshot",
2253	.version = {1, 12, 0},
2254	.module  = THIS_MODULE,
2255	.ctr     = snapshot_ctr,
2256	.dtr     = snapshot_dtr,
2257	.map     = snapshot_map,
2258	.end_io  = snapshot_end_io,
2259	.preresume  = snapshot_preresume,
2260	.resume  = snapshot_resume,
2261	.status  = snapshot_status,
2262	.iterate_devices = snapshot_iterate_devices,
2263};
2264
2265static struct target_type merge_target = {
2266	.name    = dm_snapshot_merge_target_name,
2267	.version = {1, 2, 0},
2268	.module  = THIS_MODULE,
2269	.ctr     = snapshot_ctr,
2270	.dtr     = snapshot_dtr,
2271	.map     = snapshot_merge_map,
2272	.end_io  = snapshot_end_io,
2273	.presuspend = snapshot_merge_presuspend,
2274	.preresume  = snapshot_preresume,
2275	.resume  = snapshot_merge_resume,
2276	.status  = snapshot_status,
2277	.iterate_devices = snapshot_iterate_devices,
2278};
2279
2280static int __init dm_snapshot_init(void)
2281{
2282	int r;
2283
2284	r = dm_exception_store_init();
2285	if (r) {
2286		DMERR("Failed to initialize exception stores");
2287		return r;
2288	}
2289
2290	r = dm_register_target(&snapshot_target);
2291	if (r < 0) {
2292		DMERR("snapshot target register failed %d", r);
2293		goto bad_register_snapshot_target;
2294	}
2295
2296	r = dm_register_target(&origin_target);
2297	if (r < 0) {
2298		DMERR("Origin target register failed %d", r);
2299		goto bad_register_origin_target;
2300	}
2301
2302	r = dm_register_target(&merge_target);
2303	if (r < 0) {
2304		DMERR("Merge target register failed %d", r);
2305		goto bad_register_merge_target;
2306	}
2307
2308	r = init_origin_hash();
2309	if (r) {
2310		DMERR("init_origin_hash failed.");
2311		goto bad_origin_hash;
2312	}
2313
2314	exception_cache = KMEM_CACHE(dm_exception, 0);
2315	if (!exception_cache) {
2316		DMERR("Couldn't create exception cache.");
2317		r = -ENOMEM;
2318		goto bad_exception_cache;
2319	}
2320
2321	pending_cache = KMEM_CACHE(dm_snap_pending_exception, 0);
2322	if (!pending_cache) {
2323		DMERR("Couldn't create pending cache.");
2324		r = -ENOMEM;
2325		goto bad_pending_cache;
2326	}
2327
2328	return 0;
2329
2330bad_pending_cache:
2331	kmem_cache_destroy(exception_cache);
2332bad_exception_cache:
2333	exit_origin_hash();
2334bad_origin_hash:
2335	dm_unregister_target(&merge_target);
2336bad_register_merge_target:
2337	dm_unregister_target(&origin_target);
2338bad_register_origin_target:
2339	dm_unregister_target(&snapshot_target);
2340bad_register_snapshot_target:
2341	dm_exception_store_exit();
2342
2343	return r;
2344}
2345
2346static void __exit dm_snapshot_exit(void)
2347{
2348	dm_unregister_target(&snapshot_target);
2349	dm_unregister_target(&origin_target);
2350	dm_unregister_target(&merge_target);
2351
2352	exit_origin_hash();
2353	kmem_cache_destroy(pending_cache);
2354	kmem_cache_destroy(exception_cache);
2355
2356	dm_exception_store_exit();
2357}
2358
2359/* Module hooks */
2360module_init(dm_snapshot_init);
2361module_exit(dm_snapshot_exit);
2362
2363MODULE_DESCRIPTION(DM_NAME " snapshot target");
2364MODULE_AUTHOR("Joe Thornber");
2365MODULE_LICENSE("GPL");
2366MODULE_ALIAS("dm-snapshot-origin");
2367MODULE_ALIAS("dm-snapshot-merge");