1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 2012 Alexander Block.  All rights reserved.
   4 */
   5
   6#include <linux/bsearch.h>
   7#include <linux/fs.h>
   8#include <linux/file.h>
   9#include <linux/sort.h>
  10#include <linux/mount.h>
  11#include <linux/xattr.h>
  12#include <linux/posix_acl_xattr.h>
  13#include <linux/radix-tree.h>
  14#include <linux/vmalloc.h>
  15#include <linux/string.h>
  16#include <linux/compat.h>
  17#include <linux/crc32c.h>
  18#include <linux/fsverity.h>
  19
  20#include "send.h"
  21#include "ctree.h"
  22#include "backref.h"
  23#include "locking.h"
  24#include "disk-io.h"
  25#include "btrfs_inode.h"
  26#include "transaction.h"
  27#include "compression.h"
  28#include "print-tree.h"
  29#include "accessors.h"
  30#include "dir-item.h"
  31#include "file-item.h"
  32#include "ioctl.h"
  33#include "verity.h"
  34#include "lru_cache.h"
  35
  36/*
  37 * Maximum number of references an extent can have in order for us to attempt to
  38 * issue clone operations instead of write operations. This currently exists to
  39 * avoid hitting limitations of the backreference walking code (taking a lot of
  40 * time and using too much memory for extents with large number of references).
  41 */
  42#define SEND_MAX_EXTENT_REFS	1024
  43
  44/*
  45 * A fs_path is a helper to dynamically build path names with unknown size.
  46 * It reallocates the internal buffer on demand.
  47 * It allows fast adding of path elements on the right side (normal path) and
  48 * fast adding to the left side (reversed path). A reversed path can also be
  49 * unreversed if needed.
  50 */
  51struct fs_path {
  52	union {
  53		struct {
  54			char *start;
  55			char *end;
  56
  57			char *buf;
  58			unsigned short buf_len:15;
  59			unsigned short reversed:1;
  60			char inline_buf[];
  61		};
  62		/*
  63		 * Average path length does not exceed 200 bytes, we'll have
  64		 * better packing in the slab and higher chance to satisfy
  65		 * an allocation later during send.
  66		 */
  67		char pad[256];
  68	};
  69};
  70#define FS_PATH_INLINE_SIZE \
  71	(sizeof(struct fs_path) - offsetof(struct fs_path, inline_buf))
  72
  73
  74/* reused for each extent */
  75struct clone_root {
  76	struct btrfs_root *root;
  77	u64 ino;
  78	u64 offset;
  79	u64 num_bytes;
  80	bool found_ref;
  81};
  82
  83#define SEND_MAX_NAME_CACHE_SIZE			256
  84
  85/*
  86 * Limit the root_ids array of struct backref_cache_entry to 17 elements.
  87 * This makes the size of a cache entry to be exactly 192 bytes on x86_64, which
  88 * can be satisfied from the kmalloc-192 slab, without wasting any space.
  89 * The most common case is to have a single root for cloning, which corresponds
  90 * to the send root. Having the user specify more than 16 clone roots is not
  91 * common, and in such rare cases we simply don't use caching if the number of
  92 * cloning roots that lead down to a leaf is more than 17.
  93 */
  94#define SEND_MAX_BACKREF_CACHE_ROOTS			17
  95
  96/*
  97 * Max number of entries in the cache.
  98 * With SEND_MAX_BACKREF_CACHE_ROOTS as 17, the size in bytes, excluding
  99 * maple tree's internal nodes, is 24K.
 100 */
 101#define SEND_MAX_BACKREF_CACHE_SIZE 128
 102
 103/*
 104 * A backref cache entry maps a leaf to a list of IDs of roots from which the
 105 * leaf is accessible and we can use for clone operations.
 106 * With SEND_MAX_BACKREF_CACHE_ROOTS as 12, each cache entry is 128 bytes (on
 107 * x86_64).
 108 */
 109struct backref_cache_entry {
 110	struct btrfs_lru_cache_entry entry;
 111	u64 root_ids[SEND_MAX_BACKREF_CACHE_ROOTS];
 112	/* Number of valid elements in the root_ids array. */
 113	int num_roots;
 114};
 115
 116/* See the comment at lru_cache.h about struct btrfs_lru_cache_entry. */
 117static_assert(offsetof(struct backref_cache_entry, entry) == 0);
 118
 119/*
 120 * Max number of entries in the cache that stores directories that were already
 121 * created. The cache uses raw struct btrfs_lru_cache_entry entries, so it uses
 122 * at most 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but
 123 * the kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64).
 124 */
 125#define SEND_MAX_DIR_CREATED_CACHE_SIZE			64
 126
 127/*
 128 * Max number of entries in the cache that stores directories that were already
 129 * created. The cache uses raw struct btrfs_lru_cache_entry entries, so it uses
 130 * at most 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but
 131 * the kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64).
 132 */
 133#define SEND_MAX_DIR_UTIMES_CACHE_SIZE			64
 134
 135struct send_ctx {
 136	struct file *send_filp;
 137	loff_t send_off;
 138	char *send_buf;
 139	u32 send_size;
 140	u32 send_max_size;
 141	/*
 142	 * Whether BTRFS_SEND_A_DATA attribute was already added to current
 143	 * command (since protocol v2, data must be the last attribute).
 144	 */
 145	bool put_data;
 146	struct page **send_buf_pages;
 147	u64 flags;	/* 'flags' member of btrfs_ioctl_send_args is u64 */
 148	/* Protocol version compatibility requested */
 149	u32 proto;
 150
 151	struct btrfs_root *send_root;
 152	struct btrfs_root *parent_root;
 153	struct clone_root *clone_roots;
 154	int clone_roots_cnt;
 155
 156	/* current state of the compare_tree call */
 157	struct btrfs_path *left_path;
 158	struct btrfs_path *right_path;
 159	struct btrfs_key *cmp_key;
 160
 161	/*
 162	 * Keep track of the generation of the last transaction that was used
 163	 * for relocating a block group. This is periodically checked in order
 164	 * to detect if a relocation happened since the last check, so that we
 165	 * don't operate on stale extent buffers for nodes (level >= 1) or on
 166	 * stale disk_bytenr values of file extent items.
 167	 */
 168	u64 last_reloc_trans;
 169
 170	/*
 171	 * infos of the currently processed inode. In case of deleted inodes,
 172	 * these are the values from the deleted inode.
 173	 */
 174	u64 cur_ino;
 175	u64 cur_inode_gen;
 
 
 
 176	u64 cur_inode_size;
 177	u64 cur_inode_mode;
 178	u64 cur_inode_rdev;
 179	u64 cur_inode_last_extent;
 180	u64 cur_inode_next_write_offset;
 181	bool cur_inode_new;
 182	bool cur_inode_new_gen;
 183	bool cur_inode_deleted;
 184	bool ignore_cur_inode;
 185	bool cur_inode_needs_verity;
 186	void *verity_descriptor;
 187
 188	u64 send_progress;
 189
 190	struct list_head new_refs;
 191	struct list_head deleted_refs;
 192
 193	struct btrfs_lru_cache name_cache;
 
 
 194
 195	/*
 196	 * The inode we are currently processing. It's not NULL only when we
 197	 * need to issue write commands for data extents from this inode.
 198	 */
 199	struct inode *cur_inode;
 200	struct file_ra_state ra;
 201	u64 page_cache_clear_start;
 202	bool clean_page_cache;
 203
 204	/*
 205	 * We process inodes by their increasing order, so if before an
 206	 * incremental send we reverse the parent/child relationship of
 207	 * directories such that a directory with a lower inode number was
 208	 * the parent of a directory with a higher inode number, and the one
 209	 * becoming the new parent got renamed too, we can't rename/move the
 210	 * directory with lower inode number when we finish processing it - we
 211	 * must process the directory with higher inode number first, then
 212	 * rename/move it and then rename/move the directory with lower inode
 213	 * number. Example follows.
 214	 *
 215	 * Tree state when the first send was performed:
 216	 *
 217	 * .
 218	 * |-- a                   (ino 257)
 219	 *     |-- b               (ino 258)
 220	 *         |
 221	 *         |
 222	 *         |-- c           (ino 259)
 223	 *         |   |-- d       (ino 260)
 224	 *         |
 225	 *         |-- c2          (ino 261)
 226	 *
 227	 * Tree state when the second (incremental) send is performed:
 228	 *
 229	 * .
 230	 * |-- a                   (ino 257)
 231	 *     |-- b               (ino 258)
 232	 *         |-- c2          (ino 261)
 233	 *             |-- d2      (ino 260)
 234	 *                 |-- cc  (ino 259)
 235	 *
 236	 * The sequence of steps that lead to the second state was:
 237	 *
 238	 * mv /a/b/c/d /a/b/c2/d2
 239	 * mv /a/b/c /a/b/c2/d2/cc
 240	 *
 241	 * "c" has lower inode number, but we can't move it (2nd mv operation)
 242	 * before we move "d", which has higher inode number.
 243	 *
 244	 * So we just memorize which move/rename operations must be performed
 245	 * later when their respective parent is processed and moved/renamed.
 246	 */
 247
 248	/* Indexed by parent directory inode number. */
 249	struct rb_root pending_dir_moves;
 250
 251	/*
 252	 * Reverse index, indexed by the inode number of a directory that
 253	 * is waiting for the move/rename of its immediate parent before its
 254	 * own move/rename can be performed.
 255	 */
 256	struct rb_root waiting_dir_moves;
 257
 258	/*
 259	 * A directory that is going to be rm'ed might have a child directory
 260	 * which is in the pending directory moves index above. In this case,
 261	 * the directory can only be removed after the move/rename of its child
 262	 * is performed. Example:
 263	 *
 264	 * Parent snapshot:
 265	 *
 266	 * .                        (ino 256)
 267	 * |-- a/                   (ino 257)
 268	 *     |-- b/               (ino 258)
 269	 *         |-- c/           (ino 259)
 270	 *         |   |-- x/       (ino 260)
 271	 *         |
 272	 *         |-- y/           (ino 261)
 273	 *
 274	 * Send snapshot:
 275	 *
 276	 * .                        (ino 256)
 277	 * |-- a/                   (ino 257)
 278	 *     |-- b/               (ino 258)
 279	 *         |-- YY/          (ino 261)
 280	 *              |-- x/      (ino 260)
 281	 *
 282	 * Sequence of steps that lead to the send snapshot:
 283	 * rm -f /a/b/c/foo.txt
 284	 * mv /a/b/y /a/b/YY
 285	 * mv /a/b/c/x /a/b/YY
 286	 * rmdir /a/b/c
 287	 *
 288	 * When the child is processed, its move/rename is delayed until its
 289	 * parent is processed (as explained above), but all other operations
 290	 * like update utimes, chown, chgrp, etc, are performed and the paths
 291	 * that it uses for those operations must use the orphanized name of
 292	 * its parent (the directory we're going to rm later), so we need to
 293	 * memorize that name.
 294	 *
 295	 * Indexed by the inode number of the directory to be deleted.
 296	 */
 297	struct rb_root orphan_dirs;
 298
 299	struct rb_root rbtree_new_refs;
 300	struct rb_root rbtree_deleted_refs;
 301
 302	struct btrfs_lru_cache backref_cache;
 303	u64 backref_cache_last_reloc_trans;
 304
 305	struct btrfs_lru_cache dir_created_cache;
 306	struct btrfs_lru_cache dir_utimes_cache;
 307};
 308
 309struct pending_dir_move {
 310	struct rb_node node;
 311	struct list_head list;
 312	u64 parent_ino;
 313	u64 ino;
 314	u64 gen;
 315	struct list_head update_refs;
 316};
 317
 318struct waiting_dir_move {
 319	struct rb_node node;
 320	u64 ino;
 321	/*
 322	 * There might be some directory that could not be removed because it
 323	 * was waiting for this directory inode to be moved first. Therefore
 324	 * after this directory is moved, we can try to rmdir the ino rmdir_ino.
 325	 */
 326	u64 rmdir_ino;
 327	u64 rmdir_gen;
 328	bool orphanized;
 329};
 330
 331struct orphan_dir_info {
 332	struct rb_node node;
 333	u64 ino;
 334	u64 gen;
 335	u64 last_dir_index_offset;
 336	u64 dir_high_seq_ino;
 337};
 338
 339struct name_cache_entry {
 
 340	/*
 341	 * The key in the entry is an inode number, and the generation matches
 342	 * the inode's generation.
 
 
 
 
 343	 */
 344	struct btrfs_lru_cache_entry entry;
 
 
 345	u64 parent_ino;
 346	u64 parent_gen;
 347	int ret;
 348	int need_later_update;
 349	/* Name length without NUL terminator. */
 350	int name_len;
 351	/* Not NUL terminated. */
 352	char name[] __counted_by(name_len) __nonstring;
 353};
 354
 355/* See the comment at lru_cache.h about struct btrfs_lru_cache_entry. */
 356static_assert(offsetof(struct name_cache_entry, entry) == 0);
 357
 358#define ADVANCE							1
 359#define ADVANCE_ONLY_NEXT					-1
 360
 361enum btrfs_compare_tree_result {
 362	BTRFS_COMPARE_TREE_NEW,
 363	BTRFS_COMPARE_TREE_DELETED,
 364	BTRFS_COMPARE_TREE_CHANGED,
 365	BTRFS_COMPARE_TREE_SAME,
 366};
 
 
 
 
 
 367
 368__cold
 369static void inconsistent_snapshot_error(struct send_ctx *sctx,
 370					enum btrfs_compare_tree_result result,
 371					const char *what)
 372{
 373	const char *result_string;
 374
 375	switch (result) {
 376	case BTRFS_COMPARE_TREE_NEW:
 377		result_string = "new";
 378		break;
 379	case BTRFS_COMPARE_TREE_DELETED:
 380		result_string = "deleted";
 381		break;
 382	case BTRFS_COMPARE_TREE_CHANGED:
 383		result_string = "updated";
 384		break;
 385	case BTRFS_COMPARE_TREE_SAME:
 386		ASSERT(0);
 387		result_string = "unchanged";
 388		break;
 389	default:
 390		ASSERT(0);
 391		result_string = "unexpected";
 392	}
 393
 394	btrfs_err(sctx->send_root->fs_info,
 395		  "Send: inconsistent snapshot, found %s %s for inode %llu without updated inode item, send root is %llu, parent root is %llu",
 396		  result_string, what, sctx->cmp_key->objectid,
 397		  btrfs_root_id(sctx->send_root),
 398		  (sctx->parent_root ?  btrfs_root_id(sctx->parent_root) : 0));
 399}
 400
 401__maybe_unused
 402static bool proto_cmd_ok(const struct send_ctx *sctx, int cmd)
 403{
 404	switch (sctx->proto) {
 405	case 1:	 return cmd <= BTRFS_SEND_C_MAX_V1;
 406	case 2:	 return cmd <= BTRFS_SEND_C_MAX_V2;
 407	case 3:	 return cmd <= BTRFS_SEND_C_MAX_V3;
 408	default: return false;
 409	}
 410}
 411
 412static int is_waiting_for_move(struct send_ctx *sctx, u64 ino);
 413
 414static struct waiting_dir_move *
 415get_waiting_dir_move(struct send_ctx *sctx, u64 ino);
 416
 417static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen);
 418
 419static int need_send_hole(struct send_ctx *sctx)
 420{
 421	return (sctx->parent_root && !sctx->cur_inode_new &&
 422		!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted &&
 423		S_ISREG(sctx->cur_inode_mode));
 424}
 425
 426static void fs_path_reset(struct fs_path *p)
 427{
 428	if (p->reversed) {
 429		p->start = p->buf + p->buf_len - 1;
 430		p->end = p->start;
 431		*p->start = 0;
 432	} else {
 433		p->start = p->buf;
 434		p->end = p->start;
 435		*p->start = 0;
 436	}
 437}
 438
 439static struct fs_path *fs_path_alloc(void)
 440{
 441	struct fs_path *p;
 442
 443	p = kmalloc(sizeof(*p), GFP_KERNEL);
 444	if (!p)
 445		return NULL;
 446	p->reversed = 0;
 447	p->buf = p->inline_buf;
 448	p->buf_len = FS_PATH_INLINE_SIZE;
 449	fs_path_reset(p);
 450	return p;
 451}
 452
 453static struct fs_path *fs_path_alloc_reversed(void)
 454{
 455	struct fs_path *p;
 456
 457	p = fs_path_alloc();
 458	if (!p)
 459		return NULL;
 460	p->reversed = 1;
 461	fs_path_reset(p);
 462	return p;
 463}
 464
 465static void fs_path_free(struct fs_path *p)
 466{
 467	if (!p)
 468		return;
 469	if (p->buf != p->inline_buf)
 470		kfree(p->buf);
 471	kfree(p);
 472}
 473
 474static int fs_path_len(struct fs_path *p)
 475{
 476	return p->end - p->start;
 477}
 478
 479static int fs_path_ensure_buf(struct fs_path *p, int len)
 480{
 481	char *tmp_buf;
 482	int path_len;
 483	int old_buf_len;
 484
 485	len++;
 486
 487	if (p->buf_len >= len)
 488		return 0;
 489
 490	if (len > PATH_MAX) {
 491		WARN_ON(1);
 492		return -ENOMEM;
 493	}
 494
 495	path_len = p->end - p->start;
 496	old_buf_len = p->buf_len;
 497
 498	/*
 499	 * Allocate to the next largest kmalloc bucket size, to let
 500	 * the fast path happen most of the time.
 501	 */
 502	len = kmalloc_size_roundup(len);
 503	/*
 504	 * First time the inline_buf does not suffice
 505	 */
 506	if (p->buf == p->inline_buf) {
 507		tmp_buf = kmalloc(len, GFP_KERNEL);
 508		if (tmp_buf)
 509			memcpy(tmp_buf, p->buf, old_buf_len);
 510	} else {
 511		tmp_buf = krealloc(p->buf, len, GFP_KERNEL);
 512	}
 513	if (!tmp_buf)
 514		return -ENOMEM;
 515	p->buf = tmp_buf;
 516	p->buf_len = len;
 
 
 
 
 517
 518	if (p->reversed) {
 519		tmp_buf = p->buf + old_buf_len - path_len - 1;
 520		p->end = p->buf + p->buf_len - 1;
 521		p->start = p->end - path_len;
 522		memmove(p->start, tmp_buf, path_len + 1);
 523	} else {
 524		p->start = p->buf;
 525		p->end = p->start + path_len;
 526	}
 527	return 0;
 528}
 529
 530static int fs_path_prepare_for_add(struct fs_path *p, int name_len,
 531				   char **prepared)
 532{
 533	int ret;
 534	int new_len;
 535
 536	new_len = p->end - p->start + name_len;
 537	if (p->start != p->end)
 538		new_len++;
 539	ret = fs_path_ensure_buf(p, new_len);
 540	if (ret < 0)
 541		goto out;
 542
 543	if (p->reversed) {
 544		if (p->start != p->end)
 545			*--p->start = '/';
 546		p->start -= name_len;
 547		*prepared = p->start;
 548	} else {
 549		if (p->start != p->end)
 550			*p->end++ = '/';
 551		*prepared = p->end;
 552		p->end += name_len;
 553		*p->end = 0;
 554	}
 555
 556out:
 557	return ret;
 558}
 559
 560static int fs_path_add(struct fs_path *p, const char *name, int name_len)
 561{
 562	int ret;
 563	char *prepared;
 564
 565	ret = fs_path_prepare_for_add(p, name_len, &prepared);
 566	if (ret < 0)
 567		goto out;
 568	memcpy(prepared, name, name_len);
 569
 570out:
 571	return ret;
 572}
 573
 574static int fs_path_add_path(struct fs_path *p, struct fs_path *p2)
 575{
 576	int ret;
 577	char *prepared;
 578
 579	ret = fs_path_prepare_for_add(p, p2->end - p2->start, &prepared);
 580	if (ret < 0)
 581		goto out;
 582	memcpy(prepared, p2->start, p2->end - p2->start);
 583
 584out:
 585	return ret;
 586}
 587
 588static int fs_path_add_from_extent_buffer(struct fs_path *p,
 589					  struct extent_buffer *eb,
 590					  unsigned long off, int len)
 591{
 592	int ret;
 593	char *prepared;
 594
 595	ret = fs_path_prepare_for_add(p, len, &prepared);
 596	if (ret < 0)
 597		goto out;
 598
 599	read_extent_buffer(eb, prepared, off, len);
 600
 601out:
 602	return ret;
 603}
 604
 605static int fs_path_copy(struct fs_path *p, struct fs_path *from)
 606{
 
 
 607	p->reversed = from->reversed;
 608	fs_path_reset(p);
 609
 610	return fs_path_add_path(p, from);
 
 
 611}
 612
 
 613static void fs_path_unreverse(struct fs_path *p)
 614{
 615	char *tmp;
 616	int len;
 617
 618	if (!p->reversed)
 619		return;
 620
 621	tmp = p->start;
 622	len = p->end - p->start;
 623	p->start = p->buf;
 624	p->end = p->start + len;
 625	memmove(p->start, tmp, len + 1);
 626	p->reversed = 0;
 627}
 628
 629static struct btrfs_path *alloc_path_for_send(void)
 630{
 631	struct btrfs_path *path;
 632
 633	path = btrfs_alloc_path();
 634	if (!path)
 635		return NULL;
 636	path->search_commit_root = 1;
 637	path->skip_locking = 1;
 638	path->need_commit_sem = 1;
 639	return path;
 640}
 641
 642static int write_buf(struct file *filp, const void *buf, u32 len, loff_t *off)
 643{
 644	int ret;
 645	u32 pos = 0;
 646
 647	while (pos < len) {
 648		ret = kernel_write(filp, buf + pos, len - pos, off);
 
 
 
 
 649		if (ret < 0)
 650			return ret;
 651		if (ret == 0)
 652			return -EIO;
 
 653		pos += ret;
 654	}
 655
 656	return 0;
 657}
 658
 659static int tlv_put(struct send_ctx *sctx, u16 attr, const void *data, int len)
 660{
 661	struct btrfs_tlv_header *hdr;
 662	int total_len = sizeof(*hdr) + len;
 663	int left = sctx->send_max_size - sctx->send_size;
 664
 665	if (WARN_ON_ONCE(sctx->put_data))
 666		return -EINVAL;
 667
 668	if (unlikely(left < total_len))
 669		return -EOVERFLOW;
 670
 671	hdr = (struct btrfs_tlv_header *) (sctx->send_buf + sctx->send_size);
 672	put_unaligned_le16(attr, &hdr->tlv_type);
 673	put_unaligned_le16(len, &hdr->tlv_len);
 674	memcpy(hdr + 1, data, len);
 675	sctx->send_size += total_len;
 676
 677	return 0;
 678}
 679
 680#define TLV_PUT_DEFINE_INT(bits) \
 681	static int tlv_put_u##bits(struct send_ctx *sctx,	 	\
 682			u##bits attr, u##bits value)			\
 683	{								\
 684		__le##bits __tmp = cpu_to_le##bits(value);		\
 685		return tlv_put(sctx, attr, &__tmp, sizeof(__tmp));	\
 686	}
 687
 688TLV_PUT_DEFINE_INT(8)
 689TLV_PUT_DEFINE_INT(32)
 690TLV_PUT_DEFINE_INT(64)
 691
 692static int tlv_put_string(struct send_ctx *sctx, u16 attr,
 693			  const char *str, int len)
 694{
 695	if (len == -1)
 696		len = strlen(str);
 697	return tlv_put(sctx, attr, str, len);
 698}
 699
 700static int tlv_put_uuid(struct send_ctx *sctx, u16 attr,
 701			const u8 *uuid)
 702{
 703	return tlv_put(sctx, attr, uuid, BTRFS_UUID_SIZE);
 704}
 705
 706static int tlv_put_btrfs_timespec(struct send_ctx *sctx, u16 attr,
 707				  struct extent_buffer *eb,
 708				  struct btrfs_timespec *ts)
 709{
 710	struct btrfs_timespec bts;
 711	read_extent_buffer(eb, &bts, (unsigned long)ts, sizeof(bts));
 712	return tlv_put(sctx, attr, &bts, sizeof(bts));
 713}
 714
 715
 716#define TLV_PUT(sctx, attrtype, data, attrlen) \
 717	do { \
 718		ret = tlv_put(sctx, attrtype, data, attrlen); \
 719		if (ret < 0) \
 720			goto tlv_put_failure; \
 721	} while (0)
 722
 723#define TLV_PUT_INT(sctx, attrtype, bits, value) \
 724	do { \
 725		ret = tlv_put_u##bits(sctx, attrtype, value); \
 726		if (ret < 0) \
 727			goto tlv_put_failure; \
 728	} while (0)
 729
 730#define TLV_PUT_U8(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 8, data)
 731#define TLV_PUT_U16(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 16, data)
 732#define TLV_PUT_U32(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 32, data)
 733#define TLV_PUT_U64(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 64, data)
 734#define TLV_PUT_STRING(sctx, attrtype, str, len) \
 735	do { \
 736		ret = tlv_put_string(sctx, attrtype, str, len); \
 737		if (ret < 0) \
 738			goto tlv_put_failure; \
 739	} while (0)
 740#define TLV_PUT_PATH(sctx, attrtype, p) \
 741	do { \
 742		ret = tlv_put_string(sctx, attrtype, p->start, \
 743			p->end - p->start); \
 744		if (ret < 0) \
 745			goto tlv_put_failure; \
 746	} while(0)
 747#define TLV_PUT_UUID(sctx, attrtype, uuid) \
 748	do { \
 749		ret = tlv_put_uuid(sctx, attrtype, uuid); \
 750		if (ret < 0) \
 751			goto tlv_put_failure; \
 752	} while (0)
 753#define TLV_PUT_BTRFS_TIMESPEC(sctx, attrtype, eb, ts) \
 754	do { \
 755		ret = tlv_put_btrfs_timespec(sctx, attrtype, eb, ts); \
 756		if (ret < 0) \
 757			goto tlv_put_failure; \
 758	} while (0)
 759
 760static int send_header(struct send_ctx *sctx)
 761{
 762	struct btrfs_stream_header hdr;
 763
 764	strcpy(hdr.magic, BTRFS_SEND_STREAM_MAGIC);
 765	hdr.version = cpu_to_le32(sctx->proto);
 
 766	return write_buf(sctx->send_filp, &hdr, sizeof(hdr),
 767					&sctx->send_off);
 768}
 769
 770/*
 771 * For each command/item we want to send to userspace, we call this function.
 772 */
 773static int begin_cmd(struct send_ctx *sctx, int cmd)
 774{
 775	struct btrfs_cmd_header *hdr;
 776
 777	if (WARN_ON(!sctx->send_buf))
 778		return -EINVAL;
 779
 780	if (unlikely(sctx->send_size != 0)) {
 781		btrfs_err(sctx->send_root->fs_info,
 782			  "send: command header buffer not empty cmd %d offset %llu",
 783			  cmd, sctx->send_off);
 784		return -EINVAL;
 785	}
 786
 787	sctx->send_size += sizeof(*hdr);
 788	hdr = (struct btrfs_cmd_header *)sctx->send_buf;
 789	put_unaligned_le16(cmd, &hdr->cmd);
 790
 791	return 0;
 792}
 793
 794static int send_cmd(struct send_ctx *sctx)
 795{
 796	int ret;
 797	struct btrfs_cmd_header *hdr;
 798	u32 crc;
 799
 800	hdr = (struct btrfs_cmd_header *)sctx->send_buf;
 801	put_unaligned_le32(sctx->send_size - sizeof(*hdr), &hdr->len);
 802	put_unaligned_le32(0, &hdr->crc);
 803
 804	crc = crc32c(0, (unsigned char *)sctx->send_buf, sctx->send_size);
 805	put_unaligned_le32(crc, &hdr->crc);
 806
 807	ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,
 808					&sctx->send_off);
 809
 
 
 810	sctx->send_size = 0;
 811	sctx->put_data = false;
 812
 813	return ret;
 814}
 815
 816/*
 817 * Sends a move instruction to user space
 818 */
 819static int send_rename(struct send_ctx *sctx,
 820		     struct fs_path *from, struct fs_path *to)
 821{
 822	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
 823	int ret;
 824
 825	btrfs_debug(fs_info, "send_rename %s -> %s", from->start, to->start);
 826
 827	ret = begin_cmd(sctx, BTRFS_SEND_C_RENAME);
 828	if (ret < 0)
 829		goto out;
 830
 831	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, from);
 832	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_TO, to);
 833
 834	ret = send_cmd(sctx);
 835
 836tlv_put_failure:
 837out:
 838	return ret;
 839}
 840
 841/*
 842 * Sends a link instruction to user space
 843 */
 844static int send_link(struct send_ctx *sctx,
 845		     struct fs_path *path, struct fs_path *lnk)
 846{
 847	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
 848	int ret;
 849
 850	btrfs_debug(fs_info, "send_link %s -> %s", path->start, lnk->start);
 851
 852	ret = begin_cmd(sctx, BTRFS_SEND_C_LINK);
 853	if (ret < 0)
 854		goto out;
 855
 856	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
 857	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, lnk);
 858
 859	ret = send_cmd(sctx);
 860
 861tlv_put_failure:
 862out:
 863	return ret;
 864}
 865
 866/*
 867 * Sends an unlink instruction to user space
 868 */
 869static int send_unlink(struct send_ctx *sctx, struct fs_path *path)
 870{
 871	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
 872	int ret;
 873
 874	btrfs_debug(fs_info, "send_unlink %s", path->start);
 875
 876	ret = begin_cmd(sctx, BTRFS_SEND_C_UNLINK);
 877	if (ret < 0)
 878		goto out;
 879
 880	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
 881
 882	ret = send_cmd(sctx);
 883
 884tlv_put_failure:
 885out:
 886	return ret;
 887}
 888
 889/*
 890 * Sends a rmdir instruction to user space
 891 */
 892static int send_rmdir(struct send_ctx *sctx, struct fs_path *path)
 893{
 894	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
 895	int ret;
 896
 897	btrfs_debug(fs_info, "send_rmdir %s", path->start);
 898
 899	ret = begin_cmd(sctx, BTRFS_SEND_C_RMDIR);
 900	if (ret < 0)
 901		goto out;
 902
 903	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
 904
 905	ret = send_cmd(sctx);
 906
 907tlv_put_failure:
 908out:
 909	return ret;
 910}
 911
 912struct btrfs_inode_info {
 913	u64 size;
 914	u64 gen;
 915	u64 mode;
 916	u64 uid;
 917	u64 gid;
 918	u64 rdev;
 919	u64 fileattr;
 920	u64 nlink;
 921};
 922
 923/*
 924 * Helper function to retrieve some fields from an inode item.
 925 */
 926static int get_inode_info(struct btrfs_root *root, u64 ino,
 927			  struct btrfs_inode_info *info)
 
 928{
 929	int ret;
 930	struct btrfs_path *path;
 931	struct btrfs_inode_item *ii;
 932	struct btrfs_key key;
 933
 934	path = alloc_path_for_send();
 935	if (!path)
 936		return -ENOMEM;
 937
 938	key.objectid = ino;
 939	key.type = BTRFS_INODE_ITEM_KEY;
 940	key.offset = 0;
 941	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
 942	if (ret) {
 943		if (ret > 0)
 944			ret = -ENOENT;
 945		goto out;
 946	}
 947
 948	if (!info)
 949		goto out;
 950
 951	ii = btrfs_item_ptr(path->nodes[0], path->slots[0],
 952			struct btrfs_inode_item);
 953	info->size = btrfs_inode_size(path->nodes[0], ii);
 954	info->gen = btrfs_inode_generation(path->nodes[0], ii);
 955	info->mode = btrfs_inode_mode(path->nodes[0], ii);
 956	info->uid = btrfs_inode_uid(path->nodes[0], ii);
 957	info->gid = btrfs_inode_gid(path->nodes[0], ii);
 958	info->rdev = btrfs_inode_rdev(path->nodes[0], ii);
 959	info->nlink = btrfs_inode_nlink(path->nodes[0], ii);
 960	/*
 961	 * Transfer the unchanged u64 value of btrfs_inode_item::flags, that's
 962	 * otherwise logically split to 32/32 parts.
 963	 */
 964	info->fileattr = btrfs_inode_flags(path->nodes[0], ii);
 965
 966out:
 967	btrfs_free_path(path);
 968	return ret;
 969}
 970
 971static int get_inode_gen(struct btrfs_root *root, u64 ino, u64 *gen)
 
 
 
 972{
 
 973	int ret;
 974	struct btrfs_inode_info info = { 0 };
 975
 976	ASSERT(gen);
 977
 978	ret = get_inode_info(root, ino, &info);
 979	*gen = info.gen;
 
 
 980	return ret;
 981}
 982
 983typedef int (*iterate_inode_ref_t)(u64 dir, struct fs_path *p, void *ctx);
 
 
 984
 985/*
 986 * Helper function to iterate the entries in ONE btrfs_inode_ref or
 987 * btrfs_inode_extref.
 988 * The iterate callback may return a non zero value to stop iteration. This can
 989 * be a negative value for error codes or 1 to simply stop it.
 990 *
 991 * path must point to the INODE_REF or INODE_EXTREF when called.
 992 */
 993static int iterate_inode_ref(struct btrfs_root *root, struct btrfs_path *path,
 994			     struct btrfs_key *found_key, int resolve,
 995			     iterate_inode_ref_t iterate, void *ctx)
 996{
 997	struct extent_buffer *eb = path->nodes[0];
 
 998	struct btrfs_inode_ref *iref;
 999	struct btrfs_inode_extref *extref;
1000	struct btrfs_path *tmp_path;
1001	struct fs_path *p;
1002	u32 cur = 0;
1003	u32 total;
1004	int slot = path->slots[0];
1005	u32 name_len;
1006	char *start;
1007	int ret = 0;
 
 
1008	u64 dir;
1009	unsigned long name_off;
1010	unsigned long elem_size;
1011	unsigned long ptr;
1012
1013	p = fs_path_alloc_reversed();
1014	if (!p)
1015		return -ENOMEM;
1016
1017	tmp_path = alloc_path_for_send();
1018	if (!tmp_path) {
1019		fs_path_free(p);
1020		return -ENOMEM;
1021	}
1022
1023
1024	if (found_key->type == BTRFS_INODE_REF_KEY) {
1025		ptr = (unsigned long)btrfs_item_ptr(eb, slot,
1026						    struct btrfs_inode_ref);
1027		total = btrfs_item_size(eb, slot);
 
1028		elem_size = sizeof(*iref);
1029	} else {
1030		ptr = btrfs_item_ptr_offset(eb, slot);
1031		total = btrfs_item_size(eb, slot);
1032		elem_size = sizeof(*extref);
1033	}
1034
1035	while (cur < total) {
1036		fs_path_reset(p);
1037
1038		if (found_key->type == BTRFS_INODE_REF_KEY) {
1039			iref = (struct btrfs_inode_ref *)(ptr + cur);
1040			name_len = btrfs_inode_ref_name_len(eb, iref);
1041			name_off = (unsigned long)(iref + 1);
 
1042			dir = found_key->offset;
1043		} else {
1044			extref = (struct btrfs_inode_extref *)(ptr + cur);
1045			name_len = btrfs_inode_extref_name_len(eb, extref);
1046			name_off = (unsigned long)&extref->name;
 
1047			dir = btrfs_inode_extref_parent(eb, extref);
1048		}
1049
1050		if (resolve) {
1051			start = btrfs_ref_to_path(root, tmp_path, name_len,
1052						  name_off, eb, dir,
1053						  p->buf, p->buf_len);
1054			if (IS_ERR(start)) {
1055				ret = PTR_ERR(start);
1056				goto out;
1057			}
1058			if (start < p->buf) {
1059				/* overflow , try again with larger buffer */
1060				ret = fs_path_ensure_buf(p,
1061						p->buf_len + p->buf - start);
1062				if (ret < 0)
1063					goto out;
1064				start = btrfs_ref_to_path(root, tmp_path,
1065							  name_len, name_off,
1066							  eb, dir,
1067							  p->buf, p->buf_len);
1068				if (IS_ERR(start)) {
1069					ret = PTR_ERR(start);
1070					goto out;
1071				}
1072				if (unlikely(start < p->buf)) {
1073					btrfs_err(root->fs_info,
1074			"send: path ref buffer underflow for key (%llu %u %llu)",
1075						  found_key->objectid,
1076						  found_key->type,
1077						  found_key->offset);
1078					ret = -EINVAL;
1079					goto out;
1080				}
1081			}
1082			p->start = start;
1083		} else {
1084			ret = fs_path_add_from_extent_buffer(p, eb, name_off,
1085							     name_len);
1086			if (ret < 0)
1087				goto out;
1088		}
1089
1090		cur += elem_size + name_len;
1091		ret = iterate(dir, p, ctx);
1092		if (ret)
1093			goto out;
 
1094	}
1095
1096out:
1097	btrfs_free_path(tmp_path);
1098	fs_path_free(p);
1099	return ret;
1100}
1101
1102typedef int (*iterate_dir_item_t)(int num, struct btrfs_key *di_key,
1103				  const char *name, int name_len,
1104				  const char *data, int data_len,
1105				  void *ctx);
1106
1107/*
1108 * Helper function to iterate the entries in ONE btrfs_dir_item.
1109 * The iterate callback may return a non zero value to stop iteration. This can
1110 * be a negative value for error codes or 1 to simply stop it.
1111 *
1112 * path must point to the dir item when called.
1113 */
1114static int iterate_dir_item(struct btrfs_root *root, struct btrfs_path *path,
1115			    iterate_dir_item_t iterate, void *ctx)
1116{
1117	int ret = 0;
1118	struct extent_buffer *eb;
 
1119	struct btrfs_dir_item *di;
1120	struct btrfs_key di_key;
1121	char *buf = NULL;
1122	int buf_len;
1123	u32 name_len;
1124	u32 data_len;
1125	u32 cur;
1126	u32 len;
1127	u32 total;
1128	int slot;
1129	int num;
 
1130
1131	/*
1132	 * Start with a small buffer (1 page). If later we end up needing more
1133	 * space, which can happen for xattrs on a fs with a leaf size greater
1134	 * than the page size, attempt to increase the buffer. Typically xattr
1135	 * values are small.
1136	 */
1137	buf_len = PATH_MAX;
1138	buf = kmalloc(buf_len, GFP_KERNEL);
1139	if (!buf) {
1140		ret = -ENOMEM;
1141		goto out;
1142	}
1143
1144	eb = path->nodes[0];
1145	slot = path->slots[0];
 
1146	di = btrfs_item_ptr(eb, slot, struct btrfs_dir_item);
1147	cur = 0;
1148	len = 0;
1149	total = btrfs_item_size(eb, slot);
1150
1151	num = 0;
1152	while (cur < total) {
1153		name_len = btrfs_dir_name_len(eb, di);
1154		data_len = btrfs_dir_data_len(eb, di);
 
1155		btrfs_dir_item_key_to_cpu(eb, di, &di_key);
1156
1157		if (btrfs_dir_ftype(eb, di) == BTRFS_FT_XATTR) {
1158			if (name_len > XATTR_NAME_MAX) {
1159				ret = -ENAMETOOLONG;
1160				goto out;
1161			}
1162			if (name_len + data_len >
1163					BTRFS_MAX_XATTR_SIZE(root->fs_info)) {
1164				ret = -E2BIG;
1165				goto out;
1166			}
1167		} else {
1168			/*
1169			 * Path too long
1170			 */
1171			if (name_len + data_len > PATH_MAX) {
1172				ret = -ENAMETOOLONG;
1173				goto out;
1174			}
1175		}
1176
1177		if (name_len + data_len > buf_len) {
1178			buf_len = name_len + data_len;
1179			if (is_vmalloc_addr(buf)) {
1180				vfree(buf);
1181				buf = NULL;
1182			} else {
1183				char *tmp = krealloc(buf, buf_len,
1184						GFP_KERNEL | __GFP_NOWARN);
1185
1186				if (!tmp)
1187					kfree(buf);
1188				buf = tmp;
1189			}
1190			if (!buf) {
1191				buf = kvmalloc(buf_len, GFP_KERNEL);
1192				if (!buf) {
1193					ret = -ENOMEM;
1194					goto out;
1195				}
1196			}
1197		}
1198
1199		read_extent_buffer(eb, buf, (unsigned long)(di + 1),
1200				name_len + data_len);
1201
1202		len = sizeof(*di) + name_len + data_len;
1203		di = (struct btrfs_dir_item *)((char *)di + len);
1204		cur += len;
1205
1206		ret = iterate(num, &di_key, buf, name_len, buf + name_len,
1207			      data_len, ctx);
1208		if (ret < 0)
1209			goto out;
1210		if (ret) {
1211			ret = 0;
1212			goto out;
1213		}
1214
1215		num++;
1216	}
1217
1218out:
1219	kvfree(buf);
1220	return ret;
1221}
1222
1223static int __copy_first_ref(u64 dir, struct fs_path *p, void *ctx)
 
1224{
1225	int ret;
1226	struct fs_path *pt = ctx;
1227
1228	ret = fs_path_copy(pt, p);
1229	if (ret < 0)
1230		return ret;
1231
1232	/* we want the first only */
1233	return 1;
1234}
1235
1236/*
1237 * Retrieve the first path of an inode. If an inode has more then one
1238 * ref/hardlink, this is ignored.
1239 */
1240static int get_inode_path(struct btrfs_root *root,
1241			  u64 ino, struct fs_path *path)
1242{
1243	int ret;
1244	struct btrfs_key key, found_key;
1245	struct btrfs_path *p;
1246
1247	p = alloc_path_for_send();
1248	if (!p)
1249		return -ENOMEM;
1250
1251	fs_path_reset(path);
1252
1253	key.objectid = ino;
1254	key.type = BTRFS_INODE_REF_KEY;
1255	key.offset = 0;
1256
1257	ret = btrfs_search_slot_for_read(root, &key, p, 1, 0);
1258	if (ret < 0)
1259		goto out;
1260	if (ret) {
1261		ret = 1;
1262		goto out;
1263	}
1264	btrfs_item_key_to_cpu(p->nodes[0], &found_key, p->slots[0]);
1265	if (found_key.objectid != ino ||
1266	    (found_key.type != BTRFS_INODE_REF_KEY &&
1267	     found_key.type != BTRFS_INODE_EXTREF_KEY)) {
1268		ret = -ENOENT;
1269		goto out;
1270	}
1271
1272	ret = iterate_inode_ref(root, p, &found_key, 1,
1273				__copy_first_ref, path);
1274	if (ret < 0)
1275		goto out;
1276	ret = 0;
1277
1278out:
1279	btrfs_free_path(p);
1280	return ret;
1281}
1282
1283struct backref_ctx {
1284	struct send_ctx *sctx;
1285
1286	/* number of total found references */
1287	u64 found;
1288
1289	/*
1290	 * used for clones found in send_root. clones found behind cur_objectid
1291	 * and cur_offset are not considered as allowed clones.
1292	 */
1293	u64 cur_objectid;
1294	u64 cur_offset;
1295
1296	/* may be truncated in case it's the last extent in a file */
1297	u64 extent_len;
1298
1299	/* The bytenr the file extent item we are processing refers to. */
1300	u64 bytenr;
1301	/* The owner (root id) of the data backref for the current extent. */
1302	u64 backref_owner;
1303	/* The offset of the data backref for the current extent. */
1304	u64 backref_offset;
1305};
1306
1307static int __clone_root_cmp_bsearch(const void *key, const void *elt)
1308{
1309	u64 root = (u64)(uintptr_t)key;
1310	const struct clone_root *cr = elt;
1311
1312	if (root < btrfs_root_id(cr->root))
1313		return -1;
1314	if (root > btrfs_root_id(cr->root))
1315		return 1;
1316	return 0;
1317}
1318
1319static int __clone_root_cmp_sort(const void *e1, const void *e2)
1320{
1321	const struct clone_root *cr1 = e1;
1322	const struct clone_root *cr2 = e2;
1323
1324	if (btrfs_root_id(cr1->root) < btrfs_root_id(cr2->root))
1325		return -1;
1326	if (btrfs_root_id(cr1->root) > btrfs_root_id(cr2->root))
1327		return 1;
1328	return 0;
1329}
1330
1331/*
1332 * Called for every backref that is found for the current extent.
1333 * Results are collected in sctx->clone_roots->ino/offset.
1334 */
1335static int iterate_backrefs(u64 ino, u64 offset, u64 num_bytes, u64 root_id,
1336			    void *ctx_)
1337{
1338	struct backref_ctx *bctx = ctx_;
1339	struct clone_root *clone_root;
1340
1341	/* First check if the root is in the list of accepted clone sources */
1342	clone_root = bsearch((void *)(uintptr_t)root_id, bctx->sctx->clone_roots,
1343			     bctx->sctx->clone_roots_cnt,
1344			     sizeof(struct clone_root),
1345			     __clone_root_cmp_bsearch);
1346	if (!clone_root)
1347		return 0;
1348
1349	/* This is our own reference, bail out as we can't clone from it. */
1350	if (clone_root->root == bctx->sctx->send_root &&
1351	    ino == bctx->cur_objectid &&
1352	    offset == bctx->cur_offset)
1353		return 0;
 
1354
1355	/*
1356	 * Make sure we don't consider clones from send_root that are
1357	 * behind the current inode/offset.
1358	 */
1359	if (clone_root->root == bctx->sctx->send_root) {
1360		/*
1361		 * If the source inode was not yet processed we can't issue a
1362		 * clone operation, as the source extent does not exist yet at
1363		 * the destination of the stream.
1364		 */
1365		if (ino > bctx->cur_objectid)
1366			return 0;
1367		/*
1368		 * We clone from the inode currently being sent as long as the
1369		 * source extent is already processed, otherwise we could try
1370		 * to clone from an extent that does not exist yet at the
1371		 * destination of the stream.
1372		 */
1373		if (ino == bctx->cur_objectid &&
1374		    offset + bctx->extent_len >
1375		    bctx->sctx->cur_inode_next_write_offset)
1376			return 0;
1377	}
1378
1379	bctx->found++;
1380	clone_root->found_ref = true;
1381
1382	/*
1383	 * If the given backref refers to a file extent item with a larger
1384	 * number of bytes than what we found before, use the new one so that
1385	 * we clone more optimally and end up doing less writes and getting
1386	 * less exclusive, non-shared extents at the destination.
1387	 */
1388	if (num_bytes > clone_root->num_bytes) {
1389		clone_root->ino = ino;
1390		clone_root->offset = offset;
1391		clone_root->num_bytes = num_bytes;
1392
1393		/*
1394		 * Found a perfect candidate, so there's no need to continue
1395		 * backref walking.
1396		 */
1397		if (num_bytes >= bctx->extent_len)
1398			return BTRFS_ITERATE_EXTENT_INODES_STOP;
1399	}
1400
1401	return 0;
1402}
1403
1404static bool lookup_backref_cache(u64 leaf_bytenr, void *ctx,
1405				 const u64 **root_ids_ret, int *root_count_ret)
1406{
1407	struct backref_ctx *bctx = ctx;
1408	struct send_ctx *sctx = bctx->sctx;
1409	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1410	const u64 key = leaf_bytenr >> fs_info->sectorsize_bits;
1411	struct btrfs_lru_cache_entry *raw_entry;
1412	struct backref_cache_entry *entry;
1413
1414	if (sctx->backref_cache.size == 0)
1415		return false;
1416
1417	/*
1418	 * If relocation happened since we first filled the cache, then we must
1419	 * empty the cache and can not use it, because even though we operate on
1420	 * read-only roots, their leaves and nodes may have been reallocated and
1421	 * now be used for different nodes/leaves of the same tree or some other
1422	 * tree.
1423	 *
1424	 * We are called from iterate_extent_inodes() while either holding a
1425	 * transaction handle or holding fs_info->commit_root_sem, so no need
1426	 * to take any lock here.
1427	 */
1428	if (fs_info->last_reloc_trans > sctx->backref_cache_last_reloc_trans) {
1429		btrfs_lru_cache_clear(&sctx->backref_cache);
1430		return false;
1431	}
1432
1433	raw_entry = btrfs_lru_cache_lookup(&sctx->backref_cache, key, 0);
1434	if (!raw_entry)
1435		return false;
1436
1437	entry = container_of(raw_entry, struct backref_cache_entry, entry);
1438	*root_ids_ret = entry->root_ids;
1439	*root_count_ret = entry->num_roots;
1440
1441	return true;
1442}
1443
1444static void store_backref_cache(u64 leaf_bytenr, const struct ulist *root_ids,
1445				void *ctx)
1446{
1447	struct backref_ctx *bctx = ctx;
1448	struct send_ctx *sctx = bctx->sctx;
1449	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1450	struct backref_cache_entry *new_entry;
1451	struct ulist_iterator uiter;
1452	struct ulist_node *node;
1453	int ret;
1454
1455	/*
1456	 * We're called while holding a transaction handle or while holding
1457	 * fs_info->commit_root_sem (at iterate_extent_inodes()), so must do a
1458	 * NOFS allocation.
1459	 */
1460	new_entry = kmalloc(sizeof(struct backref_cache_entry), GFP_NOFS);
1461	/* No worries, cache is optional. */
1462	if (!new_entry)
1463		return;
1464
1465	new_entry->entry.key = leaf_bytenr >> fs_info->sectorsize_bits;
1466	new_entry->entry.gen = 0;
1467	new_entry->num_roots = 0;
1468	ULIST_ITER_INIT(&uiter);
1469	while ((node = ulist_next(root_ids, &uiter)) != NULL) {
1470		const u64 root_id = node->val;
1471		struct clone_root *root;
1472
1473		root = bsearch((void *)(uintptr_t)root_id, sctx->clone_roots,
1474			       sctx->clone_roots_cnt, sizeof(struct clone_root),
1475			       __clone_root_cmp_bsearch);
1476		if (!root)
1477			continue;
1478
1479		/* Too many roots, just exit, no worries as caching is optional. */
1480		if (new_entry->num_roots >= SEND_MAX_BACKREF_CACHE_ROOTS) {
1481			kfree(new_entry);
1482			return;
1483		}
1484
1485		new_entry->root_ids[new_entry->num_roots] = root_id;
1486		new_entry->num_roots++;
1487	}
1488
1489	/*
1490	 * We may have not added any roots to the new cache entry, which means
1491	 * none of the roots is part of the list of roots from which we are
1492	 * allowed to clone. Cache the new entry as it's still useful to avoid
1493	 * backref walking to determine which roots have a path to the leaf.
1494	 *
1495	 * Also use GFP_NOFS because we're called while holding a transaction
1496	 * handle or while holding fs_info->commit_root_sem.
1497	 */
1498	ret = btrfs_lru_cache_store(&sctx->backref_cache, &new_entry->entry,
1499				    GFP_NOFS);
1500	ASSERT(ret == 0 || ret == -ENOMEM);
1501	if (ret) {
1502		/* Caching is optional, no worries. */
1503		kfree(new_entry);
1504		return;
1505	}
1506
1507	/*
1508	 * We are called from iterate_extent_inodes() while either holding a
1509	 * transaction handle or holding fs_info->commit_root_sem, so no need
1510	 * to take any lock here.
1511	 */
1512	if (sctx->backref_cache.size == 1)
1513		sctx->backref_cache_last_reloc_trans = fs_info->last_reloc_trans;
1514}
1515
1516static int check_extent_item(u64 bytenr, const struct btrfs_extent_item *ei,
1517			     const struct extent_buffer *leaf, void *ctx)
1518{
1519	const u64 refs = btrfs_extent_refs(leaf, ei);
1520	const struct backref_ctx *bctx = ctx;
1521	const struct send_ctx *sctx = bctx->sctx;
1522
1523	if (bytenr == bctx->bytenr) {
1524		const u64 flags = btrfs_extent_flags(leaf, ei);
1525
1526		if (WARN_ON(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK))
1527			return -EUCLEAN;
1528
1529		/*
1530		 * If we have only one reference and only the send root as a
1531		 * clone source - meaning no clone roots were given in the
1532		 * struct btrfs_ioctl_send_args passed to the send ioctl - then
1533		 * it's our reference and there's no point in doing backref
1534		 * walking which is expensive, so exit early.
1535		 */
1536		if (refs == 1 && sctx->clone_roots_cnt == 1)
1537			return -ENOENT;
1538	}
1539
1540	/*
1541	 * Backreference walking (iterate_extent_inodes() below) is currently
1542	 * too expensive when an extent has a large number of references, both
1543	 * in time spent and used memory. So for now just fallback to write
1544	 * operations instead of clone operations when an extent has more than
1545	 * a certain amount of references.
1546	 */
1547	if (refs > SEND_MAX_EXTENT_REFS)
1548		return -ENOENT;
1549
1550	return 0;
1551}
1552
1553static bool skip_self_data_ref(u64 root, u64 ino, u64 offset, void *ctx)
1554{
1555	const struct backref_ctx *bctx = ctx;
1556
1557	if (ino == bctx->cur_objectid &&
1558	    root == bctx->backref_owner &&
1559	    offset == bctx->backref_offset)
1560		return true;
1561
1562	return false;
1563}
1564
1565/*
1566 * Given an inode, offset and extent item, it finds a good clone for a clone
1567 * instruction. Returns -ENOENT when none could be found. The function makes
1568 * sure that the returned clone is usable at the point where sending is at the
1569 * moment. This means, that no clones are accepted which lie behind the current
1570 * inode+offset.
1571 *
1572 * path must point to the extent item when called.
1573 */
1574static int find_extent_clone(struct send_ctx *sctx,
1575			     struct btrfs_path *path,
1576			     u64 ino, u64 data_offset,
1577			     u64 ino_size,
1578			     struct clone_root **found)
1579{
1580	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1581	int ret;
1582	int extent_type;
1583	u64 logical;
1584	u64 disk_byte;
1585	u64 num_bytes;
 
 
1586	struct btrfs_file_extent_item *fi;
1587	struct extent_buffer *eb = path->nodes[0];
1588	struct backref_ctx backref_ctx = { 0 };
1589	struct btrfs_backref_walk_ctx backref_walk_ctx = { 0 };
1590	struct clone_root *cur_clone_root;
 
 
1591	int compressed;
1592	u32 i;
1593
1594	/*
1595	 * With fallocate we can get prealloc extents beyond the inode's i_size,
1596	 * so we don't do anything here because clone operations can not clone
1597	 * to a range beyond i_size without increasing the i_size of the
1598	 * destination inode.
1599	 */
1600	if (data_offset >= ino_size)
1601		return 0;
1602
1603	fi = btrfs_item_ptr(eb, path->slots[0], struct btrfs_file_extent_item);
1604	extent_type = btrfs_file_extent_type(eb, fi);
1605	if (extent_type == BTRFS_FILE_EXTENT_INLINE)
1606		return -ENOENT;
1607
1608	disk_byte = btrfs_file_extent_disk_bytenr(eb, fi);
1609	if (disk_byte == 0)
1610		return -ENOENT;
 
 
1611
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1612	compressed = btrfs_file_extent_compression(eb, fi);
 
1613	num_bytes = btrfs_file_extent_num_bytes(eb, fi);
 
 
 
 
 
1614	logical = disk_byte + btrfs_file_extent_offset(eb, fi);
1615
 
 
 
 
 
 
 
 
 
 
 
 
 
1616	/*
1617	 * Setup the clone roots.
1618	 */
1619	for (i = 0; i < sctx->clone_roots_cnt; i++) {
1620		cur_clone_root = sctx->clone_roots + i;
1621		cur_clone_root->ino = (u64)-1;
1622		cur_clone_root->offset = 0;
1623		cur_clone_root->num_bytes = 0;
1624		cur_clone_root->found_ref = false;
1625	}
1626
1627	backref_ctx.sctx = sctx;
1628	backref_ctx.cur_objectid = ino;
1629	backref_ctx.cur_offset = data_offset;
1630	backref_ctx.bytenr = disk_byte;
1631	/*
1632	 * Use the header owner and not the send root's id, because in case of a
1633	 * snapshot we can have shared subtrees.
 
 
 
 
 
 
 
1634	 */
1635	backref_ctx.backref_owner = btrfs_header_owner(eb);
1636	backref_ctx.backref_offset = data_offset - btrfs_file_extent_offset(eb, fi);
 
 
1637
1638	/*
1639	 * The last extent of a file may be too large due to page alignment.
1640	 * We need to adjust extent_len in this case so that the checks in
1641	 * iterate_backrefs() work.
1642	 */
1643	if (data_offset + num_bytes >= ino_size)
1644		backref_ctx.extent_len = ino_size - data_offset;
1645	else
1646		backref_ctx.extent_len = num_bytes;
1647
1648	/*
1649	 * Now collect all backrefs.
1650	 */
1651	backref_walk_ctx.bytenr = disk_byte;
1652	if (compressed == BTRFS_COMPRESS_NONE)
1653		backref_walk_ctx.extent_item_pos = btrfs_file_extent_offset(eb, fi);
1654	backref_walk_ctx.fs_info = fs_info;
1655	backref_walk_ctx.cache_lookup = lookup_backref_cache;
1656	backref_walk_ctx.cache_store = store_backref_cache;
1657	backref_walk_ctx.indirect_ref_iterator = iterate_backrefs;
1658	backref_walk_ctx.check_extent_item = check_extent_item;
1659	backref_walk_ctx.user_ctx = &backref_ctx;
1660
1661	/*
1662	 * If have a single clone root, then it's the send root and we can tell
1663	 * the backref walking code to skip our own backref and not resolve it,
1664	 * since we can not use it for cloning - the source and destination
1665	 * ranges can't overlap and in case the leaf is shared through a subtree
1666	 * due to snapshots, we can't use those other roots since they are not
1667	 * in the list of clone roots.
1668	 */
1669	if (sctx->clone_roots_cnt == 1)
1670		backref_walk_ctx.skip_data_ref = skip_self_data_ref;
1671
1672	ret = iterate_extent_inodes(&backref_walk_ctx, true, iterate_backrefs,
1673				    &backref_ctx);
1674	if (ret < 0)
1675		return ret;
1676
1677	down_read(&fs_info->commit_root_sem);
1678	if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
1679		/*
1680		 * A transaction commit for a transaction in which block group
1681		 * relocation was done just happened.
1682		 * The disk_bytenr of the file extent item we processed is
1683		 * possibly stale, referring to the extent's location before
1684		 * relocation. So act as if we haven't found any clone sources
1685		 * and fallback to write commands, which will read the correct
1686		 * data from the new extent location. Otherwise we will fail
1687		 * below because we haven't found our own back reference or we
1688		 * could be getting incorrect sources in case the old extent
1689		 * was already reallocated after the relocation.
1690		 */
1691		up_read(&fs_info->commit_root_sem);
1692		return -ENOENT;
1693	}
1694	up_read(&fs_info->commit_root_sem);
1695
1696	btrfs_debug(fs_info,
1697		    "find_extent_clone: data_offset=%llu, ino=%llu, num_bytes=%llu, logical=%llu",
1698		    data_offset, ino, num_bytes, logical);
1699
1700	if (!backref_ctx.found) {
1701		btrfs_debug(fs_info, "no clones found");
1702		return -ENOENT;
1703	}
1704
1705	cur_clone_root = NULL;
1706	for (i = 0; i < sctx->clone_roots_cnt; i++) {
1707		struct clone_root *clone_root = &sctx->clone_roots[i];
1708
1709		if (!clone_root->found_ref)
1710			continue;
1711
1712		/*
1713		 * Choose the root from which we can clone more bytes, to
1714		 * minimize write operations and therefore have more extent
1715		 * sharing at the destination (the same as in the source).
1716		 */
1717		if (!cur_clone_root ||
1718		    clone_root->num_bytes > cur_clone_root->num_bytes) {
1719			cur_clone_root = clone_root;
1720
1721			/*
1722			 * We found an optimal clone candidate (any inode from
1723			 * any root is fine), so we're done.
1724			 */
1725			if (clone_root->num_bytes >= backref_ctx.extent_len)
1726				break;
1727		}
 
1728	}
1729
1730	if (cur_clone_root) {
1731		*found = cur_clone_root;
1732		ret = 0;
1733	} else {
1734		ret = -ENOENT;
1735	}
1736
 
 
 
1737	return ret;
1738}
1739
1740static int read_symlink(struct btrfs_root *root,
1741			u64 ino,
1742			struct fs_path *dest)
1743{
1744	int ret;
1745	struct btrfs_path *path;
1746	struct btrfs_key key;
1747	struct btrfs_file_extent_item *ei;
1748	u8 type;
1749	u8 compression;
1750	unsigned long off;
1751	int len;
1752
1753	path = alloc_path_for_send();
1754	if (!path)
1755		return -ENOMEM;
1756
1757	key.objectid = ino;
1758	key.type = BTRFS_EXTENT_DATA_KEY;
1759	key.offset = 0;
1760	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1761	if (ret < 0)
1762		goto out;
1763	if (ret) {
1764		/*
1765		 * An empty symlink inode. Can happen in rare error paths when
1766		 * creating a symlink (transaction committed before the inode
1767		 * eviction handler removed the symlink inode items and a crash
1768		 * happened in between or the subvol was snapshoted in between).
1769		 * Print an informative message to dmesg/syslog so that the user
1770		 * can delete the symlink.
1771		 */
1772		btrfs_err(root->fs_info,
1773			  "Found empty symlink inode %llu at root %llu",
1774			  ino, btrfs_root_id(root));
1775		ret = -EIO;
1776		goto out;
1777	}
1778
1779	ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
1780			struct btrfs_file_extent_item);
1781	type = btrfs_file_extent_type(path->nodes[0], ei);
1782	if (unlikely(type != BTRFS_FILE_EXTENT_INLINE)) {
1783		ret = -EUCLEAN;
1784		btrfs_crit(root->fs_info,
1785"send: found symlink extent that is not inline, ino %llu root %llu extent type %d",
1786			   ino, btrfs_root_id(root), type);
1787		goto out;
1788	}
1789	compression = btrfs_file_extent_compression(path->nodes[0], ei);
1790	if (unlikely(compression != BTRFS_COMPRESS_NONE)) {
1791		ret = -EUCLEAN;
1792		btrfs_crit(root->fs_info,
1793"send: found symlink extent with compression, ino %llu root %llu compression type %d",
1794			   ino, btrfs_root_id(root), compression);
1795		goto out;
1796	}
1797
1798	off = btrfs_file_extent_inline_start(ei);
1799	len = btrfs_file_extent_ram_bytes(path->nodes[0], ei);
1800
1801	ret = fs_path_add_from_extent_buffer(dest, path->nodes[0], off, len);
1802
1803out:
1804	btrfs_free_path(path);
1805	return ret;
1806}
1807
1808/*
1809 * Helper function to generate a file name that is unique in the root of
1810 * send_root and parent_root. This is used to generate names for orphan inodes.
1811 */
1812static int gen_unique_name(struct send_ctx *sctx,
1813			   u64 ino, u64 gen,
1814			   struct fs_path *dest)
1815{
1816	int ret = 0;
1817	struct btrfs_path *path;
1818	struct btrfs_dir_item *di;
1819	char tmp[64];
1820	int len;
1821	u64 idx = 0;
1822
1823	path = alloc_path_for_send();
1824	if (!path)
1825		return -ENOMEM;
1826
1827	while (1) {
1828		struct fscrypt_str tmp_name;
1829
1830		len = snprintf(tmp, sizeof(tmp), "o%llu-%llu-%llu",
1831				ino, gen, idx);
1832		ASSERT(len < sizeof(tmp));
1833		tmp_name.name = tmp;
1834		tmp_name.len = strlen(tmp);
1835
1836		di = btrfs_lookup_dir_item(NULL, sctx->send_root,
1837				path, BTRFS_FIRST_FREE_OBJECTID,
1838				&tmp_name, 0);
1839		btrfs_release_path(path);
1840		if (IS_ERR(di)) {
1841			ret = PTR_ERR(di);
1842			goto out;
1843		}
1844		if (di) {
1845			/* not unique, try again */
1846			idx++;
1847			continue;
1848		}
1849
1850		if (!sctx->parent_root) {
1851			/* unique */
1852			ret = 0;
1853			break;
1854		}
1855
1856		di = btrfs_lookup_dir_item(NULL, sctx->parent_root,
1857				path, BTRFS_FIRST_FREE_OBJECTID,
1858				&tmp_name, 0);
1859		btrfs_release_path(path);
1860		if (IS_ERR(di)) {
1861			ret = PTR_ERR(di);
1862			goto out;
1863		}
1864		if (di) {
1865			/* not unique, try again */
1866			idx++;
1867			continue;
1868		}
1869		/* unique */
1870		break;
1871	}
1872
1873	ret = fs_path_add(dest, tmp, strlen(tmp));
1874
1875out:
1876	btrfs_free_path(path);
1877	return ret;
1878}
1879
1880enum inode_state {
1881	inode_state_no_change,
1882	inode_state_will_create,
1883	inode_state_did_create,
1884	inode_state_will_delete,
1885	inode_state_did_delete,
1886};
1887
1888static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen,
1889			       u64 *send_gen, u64 *parent_gen)
1890{
1891	int ret;
1892	int left_ret;
1893	int right_ret;
1894	u64 left_gen;
1895	u64 right_gen = 0;
1896	struct btrfs_inode_info info;
1897
1898	ret = get_inode_info(sctx->send_root, ino, &info);
 
1899	if (ret < 0 && ret != -ENOENT)
1900		goto out;
1901	left_ret = (info.nlink == 0) ? -ENOENT : ret;
1902	left_gen = info.gen;
1903	if (send_gen)
1904		*send_gen = ((left_ret == -ENOENT) ? 0 : info.gen);
1905
1906	if (!sctx->parent_root) {
1907		right_ret = -ENOENT;
1908	} else {
1909		ret = get_inode_info(sctx->parent_root, ino, &info);
 
1910		if (ret < 0 && ret != -ENOENT)
1911			goto out;
1912		right_ret = (info.nlink == 0) ? -ENOENT : ret;
1913		right_gen = info.gen;
1914		if (parent_gen)
1915			*parent_gen = ((right_ret == -ENOENT) ? 0 : info.gen);
1916	}
1917
1918	if (!left_ret && !right_ret) {
1919		if (left_gen == gen && right_gen == gen) {
1920			ret = inode_state_no_change;
1921		} else if (left_gen == gen) {
1922			if (ino < sctx->send_progress)
1923				ret = inode_state_did_create;
1924			else
1925				ret = inode_state_will_create;
1926		} else if (right_gen == gen) {
1927			if (ino < sctx->send_progress)
1928				ret = inode_state_did_delete;
1929			else
1930				ret = inode_state_will_delete;
1931		} else  {
1932			ret = -ENOENT;
1933		}
1934	} else if (!left_ret) {
1935		if (left_gen == gen) {
1936			if (ino < sctx->send_progress)
1937				ret = inode_state_did_create;
1938			else
1939				ret = inode_state_will_create;
1940		} else {
1941			ret = -ENOENT;
1942		}
1943	} else if (!right_ret) {
1944		if (right_gen == gen) {
1945			if (ino < sctx->send_progress)
1946				ret = inode_state_did_delete;
1947			else
1948				ret = inode_state_will_delete;
1949		} else {
1950			ret = -ENOENT;
1951		}
1952	} else {
1953		ret = -ENOENT;
1954	}
1955
1956out:
1957	return ret;
1958}
1959
1960static int is_inode_existent(struct send_ctx *sctx, u64 ino, u64 gen,
1961			     u64 *send_gen, u64 *parent_gen)
1962{
1963	int ret;
1964
1965	if (ino == BTRFS_FIRST_FREE_OBJECTID)
1966		return 1;
1967
1968	ret = get_cur_inode_state(sctx, ino, gen, send_gen, parent_gen);
1969	if (ret < 0)
1970		goto out;
1971
1972	if (ret == inode_state_no_change ||
1973	    ret == inode_state_did_create ||
1974	    ret == inode_state_will_delete)
1975		ret = 1;
1976	else
1977		ret = 0;
1978
1979out:
1980	return ret;
1981}
1982
1983/*
1984 * Helper function to lookup a dir item in a dir.
1985 */
1986static int lookup_dir_item_inode(struct btrfs_root *root,
1987				 u64 dir, const char *name, int name_len,
1988				 u64 *found_inode)
 
1989{
1990	int ret = 0;
1991	struct btrfs_dir_item *di;
1992	struct btrfs_key key;
1993	struct btrfs_path *path;
1994	struct fscrypt_str name_str = FSTR_INIT((char *)name, name_len);
1995
1996	path = alloc_path_for_send();
1997	if (!path)
1998		return -ENOMEM;
1999
2000	di = btrfs_lookup_dir_item(NULL, root, path, dir, &name_str, 0);
 
2001	if (IS_ERR_OR_NULL(di)) {
2002		ret = di ? PTR_ERR(di) : -ENOENT;
2003		goto out;
2004	}
2005	btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key);
2006	if (key.type == BTRFS_ROOT_ITEM_KEY) {
2007		ret = -ENOENT;
2008		goto out;
2009	}
2010	*found_inode = key.objectid;
 
2011
2012out:
2013	btrfs_free_path(path);
2014	return ret;
2015}
2016
2017/*
2018 * Looks up the first btrfs_inode_ref of a given ino. It returns the parent dir,
2019 * generation of the parent dir and the name of the dir entry.
2020 */
2021static int get_first_ref(struct btrfs_root *root, u64 ino,
2022			 u64 *dir, u64 *dir_gen, struct fs_path *name)
2023{
2024	int ret;
2025	struct btrfs_key key;
2026	struct btrfs_key found_key;
2027	struct btrfs_path *path;
2028	int len;
2029	u64 parent_dir;
2030
2031	path = alloc_path_for_send();
2032	if (!path)
2033		return -ENOMEM;
2034
2035	key.objectid = ino;
2036	key.type = BTRFS_INODE_REF_KEY;
2037	key.offset = 0;
2038
2039	ret = btrfs_search_slot_for_read(root, &key, path, 1, 0);
2040	if (ret < 0)
2041		goto out;
2042	if (!ret)
2043		btrfs_item_key_to_cpu(path->nodes[0], &found_key,
2044				path->slots[0]);
2045	if (ret || found_key.objectid != ino ||
2046	    (found_key.type != BTRFS_INODE_REF_KEY &&
2047	     found_key.type != BTRFS_INODE_EXTREF_KEY)) {
2048		ret = -ENOENT;
2049		goto out;
2050	}
2051
2052	if (found_key.type == BTRFS_INODE_REF_KEY) {
2053		struct btrfs_inode_ref *iref;
2054		iref = btrfs_item_ptr(path->nodes[0], path->slots[0],
2055				      struct btrfs_inode_ref);
2056		len = btrfs_inode_ref_name_len(path->nodes[0], iref);
2057		ret = fs_path_add_from_extent_buffer(name, path->nodes[0],
2058						     (unsigned long)(iref + 1),
2059						     len);
2060		parent_dir = found_key.offset;
2061	} else {
2062		struct btrfs_inode_extref *extref;
2063		extref = btrfs_item_ptr(path->nodes[0], path->slots[0],
2064					struct btrfs_inode_extref);
2065		len = btrfs_inode_extref_name_len(path->nodes[0], extref);
2066		ret = fs_path_add_from_extent_buffer(name, path->nodes[0],
2067					(unsigned long)&extref->name, len);
2068		parent_dir = btrfs_inode_extref_parent(path->nodes[0], extref);
2069	}
2070	if (ret < 0)
2071		goto out;
2072	btrfs_release_path(path);
2073
2074	if (dir_gen) {
2075		ret = get_inode_gen(root, parent_dir, dir_gen);
 
2076		if (ret < 0)
2077			goto out;
2078	}
2079
2080	*dir = parent_dir;
2081
2082out:
2083	btrfs_free_path(path);
2084	return ret;
2085}
2086
2087static int is_first_ref(struct btrfs_root *root,
2088			u64 ino, u64 dir,
2089			const char *name, int name_len)
2090{
2091	int ret;
2092	struct fs_path *tmp_name;
2093	u64 tmp_dir;
2094
2095	tmp_name = fs_path_alloc();
2096	if (!tmp_name)
2097		return -ENOMEM;
2098
2099	ret = get_first_ref(root, ino, &tmp_dir, NULL, tmp_name);
2100	if (ret < 0)
2101		goto out;
2102
2103	if (dir != tmp_dir || name_len != fs_path_len(tmp_name)) {
2104		ret = 0;
2105		goto out;
2106	}
2107
2108	ret = !memcmp(tmp_name->start, name, name_len);
2109
2110out:
2111	fs_path_free(tmp_name);
2112	return ret;
2113}
2114
2115/*
2116 * Used by process_recorded_refs to determine if a new ref would overwrite an
2117 * already existing ref. In case it detects an overwrite, it returns the
2118 * inode/gen in who_ino/who_gen.
2119 * When an overwrite is detected, process_recorded_refs does proper orphanizing
2120 * to make sure later references to the overwritten inode are possible.
2121 * Orphanizing is however only required for the first ref of an inode.
2122 * process_recorded_refs does an additional is_first_ref check to see if
2123 * orphanizing is really required.
2124 */
2125static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen,
2126			      const char *name, int name_len,
2127			      u64 *who_ino, u64 *who_gen, u64 *who_mode)
2128{
2129	int ret;
2130	u64 parent_root_dir_gen;
2131	u64 other_inode = 0;
2132	struct btrfs_inode_info info;
2133
2134	if (!sctx->parent_root)
2135		return 0;
2136
2137	ret = is_inode_existent(sctx, dir, dir_gen, NULL, &parent_root_dir_gen);
2138	if (ret <= 0)
2139		return 0;
2140
2141	/*
2142	 * If we have a parent root we need to verify that the parent dir was
2143	 * not deleted and then re-created, if it was then we have no overwrite
2144	 * and we can just unlink this entry.
2145	 *
2146	 * @parent_root_dir_gen was set to 0 if the inode does not exist in the
2147	 * parent root.
2148	 */
2149	if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID &&
2150	    parent_root_dir_gen != dir_gen)
2151		return 0;
 
 
 
 
 
 
 
 
 
2152
2153	ret = lookup_dir_item_inode(sctx->parent_root, dir, name, name_len,
2154				    &other_inode);
2155	if (ret == -ENOENT)
2156		return 0;
2157	else if (ret < 0)
2158		return ret;
 
 
2159
2160	/*
2161	 * Check if the overwritten ref was already processed. If yes, the ref
2162	 * was already unlinked/moved, so we can safely assume that we will not
2163	 * overwrite anything at this point in time.
2164	 */
2165	if (other_inode > sctx->send_progress ||
2166	    is_waiting_for_move(sctx, other_inode)) {
2167		ret = get_inode_info(sctx->parent_root, other_inode, &info);
 
2168		if (ret < 0)
2169			return ret;
2170
 
2171		*who_ino = other_inode;
2172		*who_gen = info.gen;
2173		*who_mode = info.mode;
2174		return 1;
2175	}
2176
2177	return 0;
 
2178}
2179
2180/*
2181 * Checks if the ref was overwritten by an already processed inode. This is
2182 * used by __get_cur_name_and_parent to find out if the ref was orphanized and
2183 * thus the orphan name needs be used.
2184 * process_recorded_refs also uses it to avoid unlinking of refs that were
2185 * overwritten.
2186 */
2187static int did_overwrite_ref(struct send_ctx *sctx,
2188			    u64 dir, u64 dir_gen,
2189			    u64 ino, u64 ino_gen,
2190			    const char *name, int name_len)
2191{
2192	int ret;
 
2193	u64 ow_inode;
2194	u64 ow_gen = 0;
2195	u64 send_root_dir_gen;
2196
2197	if (!sctx->parent_root)
2198		return 0;
2199
2200	ret = is_inode_existent(sctx, dir, dir_gen, &send_root_dir_gen, NULL);
2201	if (ret <= 0)
2202		return ret;
2203
2204	/*
2205	 * @send_root_dir_gen was set to 0 if the inode does not exist in the
2206	 * send root.
2207	 */
2208	if (dir != BTRFS_FIRST_FREE_OBJECTID && send_root_dir_gen != dir_gen)
2209		return 0;
 
 
 
 
 
 
2210
2211	/* check if the ref was overwritten by another ref */
2212	ret = lookup_dir_item_inode(sctx->send_root, dir, name, name_len,
2213				    &ow_inode);
2214	if (ret == -ENOENT) {
 
 
2215		/* was never and will never be overwritten */
2216		return 0;
2217	} else if (ret < 0) {
2218		return ret;
2219	}
2220
2221	if (ow_inode == ino) {
2222		ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen);
2223		if (ret < 0)
2224			return ret;
2225
2226		/* It's the same inode, so no overwrite happened. */
2227		if (ow_gen == ino_gen)
2228			return 0;
2229	}
2230
2231	/*
2232	 * We know that it is or will be overwritten. Check this now.
2233	 * The current inode being processed might have been the one that caused
2234	 * inode 'ino' to be orphanized, therefore check if ow_inode matches
2235	 * the current inode being processed.
2236	 */
2237	if (ow_inode < sctx->send_progress)
2238		return 1;
2239
2240	if (ino != sctx->cur_ino && ow_inode == sctx->cur_ino) {
2241		if (ow_gen == 0) {
2242			ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen);
2243			if (ret < 0)
2244				return ret;
2245		}
2246		if (ow_gen == sctx->cur_inode_gen)
2247			return 1;
2248	}
2249
2250	return 0;
 
2251}
2252
2253/*
2254 * Same as did_overwrite_ref, but also checks if it is the first ref of an inode
2255 * that got overwritten. This is used by process_recorded_refs to determine
2256 * if it has to use the path as returned by get_cur_path or the orphan name.
2257 */
2258static int did_overwrite_first_ref(struct send_ctx *sctx, u64 ino, u64 gen)
2259{
2260	int ret = 0;
2261	struct fs_path *name = NULL;
2262	u64 dir;
2263	u64 dir_gen;
2264
2265	if (!sctx->parent_root)
2266		goto out;
2267
2268	name = fs_path_alloc();
2269	if (!name)
2270		return -ENOMEM;
2271
2272	ret = get_first_ref(sctx->parent_root, ino, &dir, &dir_gen, name);
2273	if (ret < 0)
2274		goto out;
2275
2276	ret = did_overwrite_ref(sctx, dir, dir_gen, ino, gen,
2277			name->start, fs_path_len(name));
2278
2279out:
2280	fs_path_free(name);
2281	return ret;
2282}
2283
2284static inline struct name_cache_entry *name_cache_search(struct send_ctx *sctx,
2285							 u64 ino, u64 gen)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2286{
2287	struct btrfs_lru_cache_entry *entry;
 
2288
2289	entry = btrfs_lru_cache_lookup(&sctx->name_cache, ino, gen);
2290	if (!entry)
2291		return NULL;
2292
2293	return container_of(entry, struct name_cache_entry, entry);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2294}
2295
2296/*
2297 * Used by get_cur_path for each ref up to the root.
2298 * Returns 0 if it succeeded.
2299 * Returns 1 if the inode is not existent or got overwritten. In that case, the
2300 * name is an orphan name. This instructs get_cur_path to stop iterating. If 1
2301 * is returned, parent_ino/parent_gen are not guaranteed to be valid.
2302 * Returns <0 in case of error.
2303 */
2304static int __get_cur_name_and_parent(struct send_ctx *sctx,
2305				     u64 ino, u64 gen,
2306				     u64 *parent_ino,
2307				     u64 *parent_gen,
2308				     struct fs_path *dest)
2309{
2310	int ret;
2311	int nce_ret;
2312	struct name_cache_entry *nce;
2313
2314	/*
2315	 * First check if we already did a call to this function with the same
2316	 * ino/gen. If yes, check if the cache entry is still up-to-date. If yes
2317	 * return the cached result.
2318	 */
2319	nce = name_cache_search(sctx, ino, gen);
2320	if (nce) {
2321		if (ino < sctx->send_progress && nce->need_later_update) {
2322			btrfs_lru_cache_remove(&sctx->name_cache, &nce->entry);
 
2323			nce = NULL;
2324		} else {
 
2325			*parent_ino = nce->parent_ino;
2326			*parent_gen = nce->parent_gen;
2327			ret = fs_path_add(dest, nce->name, nce->name_len);
2328			if (ret < 0)
2329				goto out;
2330			ret = nce->ret;
2331			goto out;
2332		}
2333	}
2334
2335	/*
2336	 * If the inode is not existent yet, add the orphan name and return 1.
2337	 * This should only happen for the parent dir that we determine in
2338	 * record_new_ref_if_needed().
2339	 */
2340	ret = is_inode_existent(sctx, ino, gen, NULL, NULL);
2341	if (ret < 0)
2342		goto out;
2343
2344	if (!ret) {
2345		ret = gen_unique_name(sctx, ino, gen, dest);
2346		if (ret < 0)
2347			goto out;
2348		ret = 1;
2349		goto out_cache;
2350	}
2351
2352	/*
2353	 * Depending on whether the inode was already processed or not, use
2354	 * send_root or parent_root for ref lookup.
2355	 */
2356	if (ino < sctx->send_progress)
2357		ret = get_first_ref(sctx->send_root, ino,
2358				    parent_ino, parent_gen, dest);
2359	else
2360		ret = get_first_ref(sctx->parent_root, ino,
2361				    parent_ino, parent_gen, dest);
2362	if (ret < 0)
2363		goto out;
2364
2365	/*
2366	 * Check if the ref was overwritten by an inode's ref that was processed
2367	 * earlier. If yes, treat as orphan and return 1.
2368	 */
2369	ret = did_overwrite_ref(sctx, *parent_ino, *parent_gen, ino, gen,
2370			dest->start, dest->end - dest->start);
2371	if (ret < 0)
2372		goto out;
2373	if (ret) {
2374		fs_path_reset(dest);
2375		ret = gen_unique_name(sctx, ino, gen, dest);
2376		if (ret < 0)
2377			goto out;
2378		ret = 1;
2379	}
2380
2381out_cache:
2382	/*
2383	 * Store the result of the lookup in the name cache.
2384	 */
2385	nce = kmalloc(sizeof(*nce) + fs_path_len(dest), GFP_KERNEL);
2386	if (!nce) {
2387		ret = -ENOMEM;
2388		goto out;
2389	}
2390
2391	nce->entry.key = ino;
2392	nce->entry.gen = gen;
2393	nce->parent_ino = *parent_ino;
2394	nce->parent_gen = *parent_gen;
2395	nce->name_len = fs_path_len(dest);
2396	nce->ret = ret;
2397	memcpy(nce->name, dest->start, nce->name_len);
2398
2399	if (ino < sctx->send_progress)
2400		nce->need_later_update = 0;
2401	else
2402		nce->need_later_update = 1;
2403
2404	nce_ret = btrfs_lru_cache_store(&sctx->name_cache, &nce->entry, GFP_KERNEL);
2405	if (nce_ret < 0) {
2406		kfree(nce);
2407		ret = nce_ret;
2408	}
2409
2410out:
2411	return ret;
2412}
2413
2414/*
2415 * Magic happens here. This function returns the first ref to an inode as it
2416 * would look like while receiving the stream at this point in time.
2417 * We walk the path up to the root. For every inode in between, we check if it
2418 * was already processed/sent. If yes, we continue with the parent as found
2419 * in send_root. If not, we continue with the parent as found in parent_root.
2420 * If we encounter an inode that was deleted at this point in time, we use the
2421 * inodes "orphan" name instead of the real name and stop. Same with new inodes
2422 * that were not created yet and overwritten inodes/refs.
2423 *
2424 * When do we have orphan inodes:
2425 * 1. When an inode is freshly created and thus no valid refs are available yet
2426 * 2. When a directory lost all it's refs (deleted) but still has dir items
2427 *    inside which were not processed yet (pending for move/delete). If anyone
2428 *    tried to get the path to the dir items, it would get a path inside that
2429 *    orphan directory.
2430 * 3. When an inode is moved around or gets new links, it may overwrite the ref
2431 *    of an unprocessed inode. If in that case the first ref would be
2432 *    overwritten, the overwritten inode gets "orphanized". Later when we
2433 *    process this overwritten inode, it is restored at a new place by moving
2434 *    the orphan inode.
2435 *
2436 * sctx->send_progress tells this function at which point in time receiving
2437 * would be.
2438 */
2439static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen,
2440			struct fs_path *dest)
2441{
2442	int ret = 0;
2443	struct fs_path *name = NULL;
2444	u64 parent_inode = 0;
2445	u64 parent_gen = 0;
2446	int stop = 0;
2447
2448	name = fs_path_alloc();
2449	if (!name) {
2450		ret = -ENOMEM;
2451		goto out;
2452	}
2453
2454	dest->reversed = 1;
2455	fs_path_reset(dest);
2456
2457	while (!stop && ino != BTRFS_FIRST_FREE_OBJECTID) {
2458		struct waiting_dir_move *wdm;
2459
2460		fs_path_reset(name);
2461
2462		if (is_waiting_for_rm(sctx, ino, gen)) {
2463			ret = gen_unique_name(sctx, ino, gen, name);
2464			if (ret < 0)
2465				goto out;
2466			ret = fs_path_add_path(dest, name);
2467			break;
2468		}
2469
2470		wdm = get_waiting_dir_move(sctx, ino);
2471		if (wdm && wdm->orphanized) {
2472			ret = gen_unique_name(sctx, ino, gen, name);
2473			stop = 1;
2474		} else if (wdm) {
2475			ret = get_first_ref(sctx->parent_root, ino,
2476					    &parent_inode, &parent_gen, name);
2477		} else {
2478			ret = __get_cur_name_and_parent(sctx, ino, gen,
2479							&parent_inode,
2480							&parent_gen, name);
2481			if (ret)
2482				stop = 1;
2483		}
2484
2485		if (ret < 0)
2486			goto out;
2487
2488		ret = fs_path_add_path(dest, name);
2489		if (ret < 0)
2490			goto out;
2491
2492		ino = parent_inode;
2493		gen = parent_gen;
2494	}
2495
2496out:
2497	fs_path_free(name);
2498	if (!ret)
2499		fs_path_unreverse(dest);
2500	return ret;
2501}
2502
2503/*
2504 * Sends a BTRFS_SEND_C_SUBVOL command/item to userspace
2505 */
2506static int send_subvol_begin(struct send_ctx *sctx)
2507{
2508	int ret;
2509	struct btrfs_root *send_root = sctx->send_root;
2510	struct btrfs_root *parent_root = sctx->parent_root;
2511	struct btrfs_path *path;
2512	struct btrfs_key key;
2513	struct btrfs_root_ref *ref;
2514	struct extent_buffer *leaf;
2515	char *name = NULL;
2516	int namelen;
2517
2518	path = btrfs_alloc_path();
2519	if (!path)
2520		return -ENOMEM;
2521
2522	name = kmalloc(BTRFS_PATH_NAME_MAX, GFP_KERNEL);
2523	if (!name) {
2524		btrfs_free_path(path);
2525		return -ENOMEM;
2526	}
2527
2528	key.objectid = btrfs_root_id(send_root);
2529	key.type = BTRFS_ROOT_BACKREF_KEY;
2530	key.offset = 0;
2531
2532	ret = btrfs_search_slot_for_read(send_root->fs_info->tree_root,
2533				&key, path, 1, 0);
2534	if (ret < 0)
2535		goto out;
2536	if (ret) {
2537		ret = -ENOENT;
2538		goto out;
2539	}
2540
2541	leaf = path->nodes[0];
2542	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2543	if (key.type != BTRFS_ROOT_BACKREF_KEY ||
2544	    key.objectid != btrfs_root_id(send_root)) {
2545		ret = -ENOENT;
2546		goto out;
2547	}
2548	ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
2549	namelen = btrfs_root_ref_name_len(leaf, ref);
2550	read_extent_buffer(leaf, name, (unsigned long)(ref + 1), namelen);
2551	btrfs_release_path(path);
2552
2553	if (parent_root) {
2554		ret = begin_cmd(sctx, BTRFS_SEND_C_SNAPSHOT);
2555		if (ret < 0)
2556			goto out;
2557	} else {
2558		ret = begin_cmd(sctx, BTRFS_SEND_C_SUBVOL);
2559		if (ret < 0)
2560			goto out;
2561	}
2562
2563	TLV_PUT_STRING(sctx, BTRFS_SEND_A_PATH, name, namelen);
2564
2565	if (!btrfs_is_empty_uuid(sctx->send_root->root_item.received_uuid))
2566		TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID,
2567			    sctx->send_root->root_item.received_uuid);
2568	else
2569		TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID,
2570			    sctx->send_root->root_item.uuid);
2571
2572	TLV_PUT_U64(sctx, BTRFS_SEND_A_CTRANSID,
2573		    btrfs_root_ctransid(&sctx->send_root->root_item));
2574	if (parent_root) {
2575		if (!btrfs_is_empty_uuid(parent_root->root_item.received_uuid))
2576			TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
2577				     parent_root->root_item.received_uuid);
2578		else
2579			TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
2580				     parent_root->root_item.uuid);
2581		TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID,
2582			    btrfs_root_ctransid(&sctx->parent_root->root_item));
2583	}
2584
2585	ret = send_cmd(sctx);
2586
2587tlv_put_failure:
2588out:
2589	btrfs_free_path(path);
2590	kfree(name);
2591	return ret;
2592}
2593
2594static int send_truncate(struct send_ctx *sctx, u64 ino, u64 gen, u64 size)
2595{
2596	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2597	int ret = 0;
2598	struct fs_path *p;
2599
2600	btrfs_debug(fs_info, "send_truncate %llu size=%llu", ino, size);
2601
2602	p = fs_path_alloc();
2603	if (!p)
2604		return -ENOMEM;
2605
2606	ret = begin_cmd(sctx, BTRFS_SEND_C_TRUNCATE);
2607	if (ret < 0)
2608		goto out;
2609
2610	ret = get_cur_path(sctx, ino, gen, p);
2611	if (ret < 0)
2612		goto out;
2613	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2614	TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, size);
2615
2616	ret = send_cmd(sctx);
2617
2618tlv_put_failure:
2619out:
2620	fs_path_free(p);
2621	return ret;
2622}
2623
2624static int send_chmod(struct send_ctx *sctx, u64 ino, u64 gen, u64 mode)
2625{
2626	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2627	int ret = 0;
2628	struct fs_path *p;
2629
2630	btrfs_debug(fs_info, "send_chmod %llu mode=%llu", ino, mode);
2631
2632	p = fs_path_alloc();
2633	if (!p)
2634		return -ENOMEM;
2635
2636	ret = begin_cmd(sctx, BTRFS_SEND_C_CHMOD);
2637	if (ret < 0)
2638		goto out;
2639
2640	ret = get_cur_path(sctx, ino, gen, p);
2641	if (ret < 0)
2642		goto out;
2643	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2644	TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode & 07777);
2645
2646	ret = send_cmd(sctx);
2647
2648tlv_put_failure:
2649out:
2650	fs_path_free(p);
2651	return ret;
2652}
2653
2654static int send_fileattr(struct send_ctx *sctx, u64 ino, u64 gen, u64 fileattr)
2655{
2656	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2657	int ret = 0;
2658	struct fs_path *p;
2659
2660	if (sctx->proto < 2)
2661		return 0;
2662
2663	btrfs_debug(fs_info, "send_fileattr %llu fileattr=%llu", ino, fileattr);
2664
2665	p = fs_path_alloc();
2666	if (!p)
2667		return -ENOMEM;
2668
2669	ret = begin_cmd(sctx, BTRFS_SEND_C_FILEATTR);
2670	if (ret < 0)
2671		goto out;
2672
2673	ret = get_cur_path(sctx, ino, gen, p);
2674	if (ret < 0)
2675		goto out;
2676	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2677	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILEATTR, fileattr);
2678
2679	ret = send_cmd(sctx);
2680
2681tlv_put_failure:
2682out:
2683	fs_path_free(p);
2684	return ret;
2685}
2686
2687static int send_chown(struct send_ctx *sctx, u64 ino, u64 gen, u64 uid, u64 gid)
2688{
2689	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2690	int ret = 0;
2691	struct fs_path *p;
2692
2693	btrfs_debug(fs_info, "send_chown %llu uid=%llu, gid=%llu",
2694		    ino, uid, gid);
2695
2696	p = fs_path_alloc();
2697	if (!p)
2698		return -ENOMEM;
2699
2700	ret = begin_cmd(sctx, BTRFS_SEND_C_CHOWN);
2701	if (ret < 0)
2702		goto out;
2703
2704	ret = get_cur_path(sctx, ino, gen, p);
2705	if (ret < 0)
2706		goto out;
2707	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2708	TLV_PUT_U64(sctx, BTRFS_SEND_A_UID, uid);
2709	TLV_PUT_U64(sctx, BTRFS_SEND_A_GID, gid);
2710
2711	ret = send_cmd(sctx);
2712
2713tlv_put_failure:
2714out:
2715	fs_path_free(p);
2716	return ret;
2717}
2718
2719static int send_utimes(struct send_ctx *sctx, u64 ino, u64 gen)
2720{
2721	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2722	int ret = 0;
2723	struct fs_path *p = NULL;
2724	struct btrfs_inode_item *ii;
2725	struct btrfs_path *path = NULL;
2726	struct extent_buffer *eb;
2727	struct btrfs_key key;
2728	int slot;
2729
2730	btrfs_debug(fs_info, "send_utimes %llu", ino);
2731
2732	p = fs_path_alloc();
2733	if (!p)
2734		return -ENOMEM;
2735
2736	path = alloc_path_for_send();
2737	if (!path) {
2738		ret = -ENOMEM;
2739		goto out;
2740	}
2741
2742	key.objectid = ino;
2743	key.type = BTRFS_INODE_ITEM_KEY;
2744	key.offset = 0;
2745	ret = btrfs_search_slot(NULL, sctx->send_root, &key, path, 0, 0);
2746	if (ret > 0)
2747		ret = -ENOENT;
2748	if (ret < 0)
2749		goto out;
2750
2751	eb = path->nodes[0];
2752	slot = path->slots[0];
2753	ii = btrfs_item_ptr(eb, slot, struct btrfs_inode_item);
2754
2755	ret = begin_cmd(sctx, BTRFS_SEND_C_UTIMES);
2756	if (ret < 0)
2757		goto out;
2758
2759	ret = get_cur_path(sctx, ino, gen, p);
2760	if (ret < 0)
2761		goto out;
2762	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2763	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_ATIME, eb, &ii->atime);
2764	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_MTIME, eb, &ii->mtime);
2765	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_CTIME, eb, &ii->ctime);
2766	if (sctx->proto >= 2)
2767		TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_OTIME, eb, &ii->otime);
2768
2769	ret = send_cmd(sctx);
2770
2771tlv_put_failure:
2772out:
2773	fs_path_free(p);
2774	btrfs_free_path(path);
2775	return ret;
2776}
2777
2778/*
2779 * If the cache is full, we can't remove entries from it and do a call to
2780 * send_utimes() for each respective inode, because we might be finishing
2781 * processing an inode that is a directory and it just got renamed, and existing
2782 * entries in the cache may refer to inodes that have the directory in their
2783 * full path - in which case we would generate outdated paths (pre-rename)
2784 * for the inodes that the cache entries point to. Instead of prunning the
2785 * cache when inserting, do it after we finish processing each inode at
2786 * finish_inode_if_needed().
2787 */
2788static int cache_dir_utimes(struct send_ctx *sctx, u64 dir, u64 gen)
2789{
2790	struct btrfs_lru_cache_entry *entry;
2791	int ret;
2792
2793	entry = btrfs_lru_cache_lookup(&sctx->dir_utimes_cache, dir, gen);
2794	if (entry != NULL)
2795		return 0;
2796
2797	/* Caching is optional, don't fail if we can't allocate memory. */
2798	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2799	if (!entry)
2800		return send_utimes(sctx, dir, gen);
2801
2802	entry->key = dir;
2803	entry->gen = gen;
2804
2805	ret = btrfs_lru_cache_store(&sctx->dir_utimes_cache, entry, GFP_KERNEL);
2806	ASSERT(ret != -EEXIST);
2807	if (ret) {
2808		kfree(entry);
2809		return send_utimes(sctx, dir, gen);
2810	}
2811
2812	return 0;
2813}
2814
2815static int trim_dir_utimes_cache(struct send_ctx *sctx)
2816{
2817	while (sctx->dir_utimes_cache.size > SEND_MAX_DIR_UTIMES_CACHE_SIZE) {
2818		struct btrfs_lru_cache_entry *lru;
2819		int ret;
2820
2821		lru = btrfs_lru_cache_lru_entry(&sctx->dir_utimes_cache);
2822		ASSERT(lru != NULL);
2823
2824		ret = send_utimes(sctx, lru->key, lru->gen);
2825		if (ret)
2826			return ret;
2827
2828		btrfs_lru_cache_remove(&sctx->dir_utimes_cache, lru);
2829	}
2830
2831	return 0;
2832}
2833
2834/*
2835 * Sends a BTRFS_SEND_C_MKXXX or SYMLINK command to user space. We don't have
2836 * a valid path yet because we did not process the refs yet. So, the inode
2837 * is created as orphan.
2838 */
2839static int send_create_inode(struct send_ctx *sctx, u64 ino)
2840{
2841	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2842	int ret = 0;
2843	struct fs_path *p;
2844	int cmd;
2845	struct btrfs_inode_info info;
2846	u64 gen;
2847	u64 mode;
2848	u64 rdev;
2849
2850	btrfs_debug(fs_info, "send_create_inode %llu", ino);
2851
2852	p = fs_path_alloc();
2853	if (!p)
2854		return -ENOMEM;
2855
2856	if (ino != sctx->cur_ino) {
2857		ret = get_inode_info(sctx->send_root, ino, &info);
 
2858		if (ret < 0)
2859			goto out;
2860		gen = info.gen;
2861		mode = info.mode;
2862		rdev = info.rdev;
2863	} else {
2864		gen = sctx->cur_inode_gen;
2865		mode = sctx->cur_inode_mode;
2866		rdev = sctx->cur_inode_rdev;
2867	}
2868
2869	if (S_ISREG(mode)) {
2870		cmd = BTRFS_SEND_C_MKFILE;
2871	} else if (S_ISDIR(mode)) {
2872		cmd = BTRFS_SEND_C_MKDIR;
2873	} else if (S_ISLNK(mode)) {
2874		cmd = BTRFS_SEND_C_SYMLINK;
2875	} else if (S_ISCHR(mode) || S_ISBLK(mode)) {
2876		cmd = BTRFS_SEND_C_MKNOD;
2877	} else if (S_ISFIFO(mode)) {
2878		cmd = BTRFS_SEND_C_MKFIFO;
2879	} else if (S_ISSOCK(mode)) {
2880		cmd = BTRFS_SEND_C_MKSOCK;
2881	} else {
2882		btrfs_warn(sctx->send_root->fs_info, "unexpected inode type %o",
2883				(int)(mode & S_IFMT));
2884		ret = -EOPNOTSUPP;
2885		goto out;
2886	}
2887
2888	ret = begin_cmd(sctx, cmd);
2889	if (ret < 0)
2890		goto out;
2891
2892	ret = gen_unique_name(sctx, ino, gen, p);
2893	if (ret < 0)
2894		goto out;
2895
2896	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2897	TLV_PUT_U64(sctx, BTRFS_SEND_A_INO, ino);
2898
2899	if (S_ISLNK(mode)) {
2900		fs_path_reset(p);
2901		ret = read_symlink(sctx->send_root, ino, p);
2902		if (ret < 0)
2903			goto out;
2904		TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, p);
2905	} else if (S_ISCHR(mode) || S_ISBLK(mode) ||
2906		   S_ISFIFO(mode) || S_ISSOCK(mode)) {
2907		TLV_PUT_U64(sctx, BTRFS_SEND_A_RDEV, new_encode_dev(rdev));
2908		TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode);
2909	}
2910
2911	ret = send_cmd(sctx);
2912	if (ret < 0)
2913		goto out;
2914
2915
2916tlv_put_failure:
2917out:
2918	fs_path_free(p);
2919	return ret;
2920}
2921
2922static void cache_dir_created(struct send_ctx *sctx, u64 dir)
2923{
2924	struct btrfs_lru_cache_entry *entry;
2925	int ret;
2926
2927	/* Caching is optional, ignore any failures. */
2928	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2929	if (!entry)
2930		return;
2931
2932	entry->key = dir;
2933	entry->gen = 0;
2934	ret = btrfs_lru_cache_store(&sctx->dir_created_cache, entry, GFP_KERNEL);
2935	if (ret < 0)
2936		kfree(entry);
2937}
2938
2939/*
2940 * We need some special handling for inodes that get processed before the parent
2941 * directory got created. See process_recorded_refs for details.
2942 * This function does the check if we already created the dir out of order.
2943 */
2944static int did_create_dir(struct send_ctx *sctx, u64 dir)
2945{
2946	int ret = 0;
2947	int iter_ret = 0;
2948	struct btrfs_path *path = NULL;
2949	struct btrfs_key key;
2950	struct btrfs_key found_key;
2951	struct btrfs_key di_key;
 
2952	struct btrfs_dir_item *di;
2953
2954	if (btrfs_lru_cache_lookup(&sctx->dir_created_cache, dir, 0))
2955		return 1;
2956
2957	path = alloc_path_for_send();
2958	if (!path)
2959		return -ENOMEM;
 
 
2960
2961	key.objectid = dir;
2962	key.type = BTRFS_DIR_INDEX_KEY;
2963	key.offset = 0;
 
 
 
2964
2965	btrfs_for_each_slot(sctx->send_root, &key, &found_key, path, iter_ret) {
2966		struct extent_buffer *eb = path->nodes[0];
 
 
 
 
 
 
 
 
 
 
 
2967
 
2968		if (found_key.objectid != key.objectid ||
2969		    found_key.type != key.type) {
2970			ret = 0;
2971			break;
2972		}
2973
2974		di = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dir_item);
2975		btrfs_dir_item_key_to_cpu(eb, di, &di_key);
2976
2977		if (di_key.type != BTRFS_ROOT_ITEM_KEY &&
2978		    di_key.objectid < sctx->send_progress) {
2979			ret = 1;
2980			cache_dir_created(sctx, dir);
2981			break;
2982		}
 
 
2983	}
2984	/* Catch error found during iteration */
2985	if (iter_ret < 0)
2986		ret = iter_ret;
2987
 
2988	btrfs_free_path(path);
2989	return ret;
2990}
2991
2992/*
2993 * Only creates the inode if it is:
2994 * 1. Not a directory
2995 * 2. Or a directory which was not created already due to out of order
2996 *    directories. See did_create_dir and process_recorded_refs for details.
2997 */
2998static int send_create_inode_if_needed(struct send_ctx *sctx)
2999{
3000	int ret;
3001
3002	if (S_ISDIR(sctx->cur_inode_mode)) {
3003		ret = did_create_dir(sctx, sctx->cur_ino);
3004		if (ret < 0)
3005			return ret;
3006		else if (ret > 0)
3007			return 0;
 
 
3008	}
3009
3010	ret = send_create_inode(sctx, sctx->cur_ino);
 
 
3011
3012	if (ret == 0 && S_ISDIR(sctx->cur_inode_mode))
3013		cache_dir_created(sctx, sctx->cur_ino);
3014
3015	return ret;
3016}
3017
3018struct recorded_ref {
3019	struct list_head list;
3020	char *name;
3021	struct fs_path *full_path;
3022	u64 dir;
3023	u64 dir_gen;
3024	int name_len;
3025	struct rb_node node;
3026	struct rb_root *root;
3027};
3028
3029static struct recorded_ref *recorded_ref_alloc(void)
3030{
3031	struct recorded_ref *ref;
3032
3033	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
3034	if (!ref)
3035		return NULL;
3036	RB_CLEAR_NODE(&ref->node);
3037	INIT_LIST_HEAD(&ref->list);
3038	return ref;
3039}
3040
3041static void recorded_ref_free(struct recorded_ref *ref)
 
 
 
 
 
 
3042{
 
 
 
3043	if (!ref)
3044		return;
3045	if (!RB_EMPTY_NODE(&ref->node))
3046		rb_erase(&ref->node, ref->root);
3047	list_del(&ref->list);
3048	fs_path_free(ref->full_path);
3049	kfree(ref);
3050}
3051
3052static void set_ref_path(struct recorded_ref *ref, struct fs_path *path)
3053{
3054	ref->full_path = path;
3055	ref->name = (char *)kbasename(ref->full_path->start);
3056	ref->name_len = ref->full_path->end - ref->name;
3057}
3058
3059static int dup_ref(struct recorded_ref *ref, struct list_head *list)
3060{
3061	struct recorded_ref *new;
3062
3063	new = recorded_ref_alloc();
3064	if (!new)
3065		return -ENOMEM;
3066
3067	new->dir = ref->dir;
3068	new->dir_gen = ref->dir_gen;
 
 
3069	list_add_tail(&new->list, list);
3070	return 0;
3071}
3072
3073static void __free_recorded_refs(struct list_head *head)
3074{
3075	struct recorded_ref *cur;
3076
3077	while (!list_empty(head)) {
3078		cur = list_entry(head->next, struct recorded_ref, list);
3079		recorded_ref_free(cur);
 
 
3080	}
3081}
3082
3083static void free_recorded_refs(struct send_ctx *sctx)
3084{
3085	__free_recorded_refs(&sctx->new_refs);
3086	__free_recorded_refs(&sctx->deleted_refs);
3087}
3088
3089/*
3090 * Renames/moves a file/dir to its orphan name. Used when the first
3091 * ref of an unprocessed inode gets overwritten and for all non empty
3092 * directories.
3093 */
3094static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen,
3095			  struct fs_path *path)
3096{
3097	int ret;
3098	struct fs_path *orphan;
3099
3100	orphan = fs_path_alloc();
3101	if (!orphan)
3102		return -ENOMEM;
3103
3104	ret = gen_unique_name(sctx, ino, gen, orphan);
3105	if (ret < 0)
3106		goto out;
3107
3108	ret = send_rename(sctx, path, orphan);
3109
3110out:
3111	fs_path_free(orphan);
3112	return ret;
3113}
3114
3115static struct orphan_dir_info *add_orphan_dir_info(struct send_ctx *sctx,
3116						   u64 dir_ino, u64 dir_gen)
3117{
3118	struct rb_node **p = &sctx->orphan_dirs.rb_node;
3119	struct rb_node *parent = NULL;
3120	struct orphan_dir_info *entry, *odi;
3121
3122	while (*p) {
3123		parent = *p;
3124		entry = rb_entry(parent, struct orphan_dir_info, node);
3125		if (dir_ino < entry->ino)
3126			p = &(*p)->rb_left;
3127		else if (dir_ino > entry->ino)
3128			p = &(*p)->rb_right;
3129		else if (dir_gen < entry->gen)
3130			p = &(*p)->rb_left;
3131		else if (dir_gen > entry->gen)
3132			p = &(*p)->rb_right;
3133		else
3134			return entry;
 
3135	}
3136
3137	odi = kmalloc(sizeof(*odi), GFP_KERNEL);
3138	if (!odi)
3139		return ERR_PTR(-ENOMEM);
3140	odi->ino = dir_ino;
3141	odi->gen = dir_gen;
3142	odi->last_dir_index_offset = 0;
3143	odi->dir_high_seq_ino = 0;
3144
3145	rb_link_node(&odi->node, parent, p);
3146	rb_insert_color(&odi->node, &sctx->orphan_dirs);
3147	return odi;
3148}
3149
3150static struct orphan_dir_info *get_orphan_dir_info(struct send_ctx *sctx,
3151						   u64 dir_ino, u64 gen)
3152{
3153	struct rb_node *n = sctx->orphan_dirs.rb_node;
3154	struct orphan_dir_info *entry;
3155
3156	while (n) {
3157		entry = rb_entry(n, struct orphan_dir_info, node);
3158		if (dir_ino < entry->ino)
3159			n = n->rb_left;
3160		else if (dir_ino > entry->ino)
3161			n = n->rb_right;
3162		else if (gen < entry->gen)
3163			n = n->rb_left;
3164		else if (gen > entry->gen)
3165			n = n->rb_right;
3166		else
3167			return entry;
3168	}
3169	return NULL;
3170}
3171
3172static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen)
3173{
3174	struct orphan_dir_info *odi = get_orphan_dir_info(sctx, dir_ino, gen);
3175
3176	return odi != NULL;
3177}
3178
3179static void free_orphan_dir_info(struct send_ctx *sctx,
3180				 struct orphan_dir_info *odi)
3181{
3182	if (!odi)
3183		return;
3184	rb_erase(&odi->node, &sctx->orphan_dirs);
3185	kfree(odi);
3186}
3187
3188/*
3189 * Returns 1 if a directory can be removed at this point in time.
3190 * We check this by iterating all dir items and checking if the inode behind
3191 * the dir item was already processed.
3192 */
3193static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen)
 
3194{
3195	int ret = 0;
3196	int iter_ret = 0;
3197	struct btrfs_root *root = sctx->parent_root;
3198	struct btrfs_path *path;
3199	struct btrfs_key key;
3200	struct btrfs_key found_key;
3201	struct btrfs_key loc;
3202	struct btrfs_dir_item *di;
3203	struct orphan_dir_info *odi = NULL;
3204	u64 dir_high_seq_ino = 0;
3205	u64 last_dir_index_offset = 0;
3206
3207	/*
3208	 * Don't try to rmdir the top/root subvolume dir.
3209	 */
3210	if (dir == BTRFS_FIRST_FREE_OBJECTID)
3211		return 0;
3212
3213	odi = get_orphan_dir_info(sctx, dir, dir_gen);
3214	if (odi && sctx->cur_ino < odi->dir_high_seq_ino)
3215		return 0;
3216
3217	path = alloc_path_for_send();
3218	if (!path)
3219		return -ENOMEM;
3220
3221	if (!odi) {
3222		/*
3223		 * Find the inode number associated with the last dir index
3224		 * entry. This is very likely the inode with the highest number
3225		 * of all inodes that have an entry in the directory. We can
3226		 * then use it to avoid future calls to can_rmdir(), when
3227		 * processing inodes with a lower number, from having to search
3228		 * the parent root b+tree for dir index keys.
3229		 */
3230		key.objectid = dir;
3231		key.type = BTRFS_DIR_INDEX_KEY;
3232		key.offset = (u64)-1;
3233
3234		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3235		if (ret < 0) {
3236			goto out;
3237		} else if (ret > 0) {
3238			/* Can't happen, the root is never empty. */
3239			ASSERT(path->slots[0] > 0);
3240			if (WARN_ON(path->slots[0] == 0)) {
3241				ret = -EUCLEAN;
3242				goto out;
3243			}
3244			path->slots[0]--;
3245		}
3246
3247		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
3248		if (key.objectid != dir || key.type != BTRFS_DIR_INDEX_KEY) {
3249			/* No index keys, dir can be removed. */
3250			ret = 1;
3251			goto out;
3252		}
3253
3254		di = btrfs_item_ptr(path->nodes[0], path->slots[0],
3255				    struct btrfs_dir_item);
3256		btrfs_dir_item_key_to_cpu(path->nodes[0], di, &loc);
3257		dir_high_seq_ino = loc.objectid;
3258		if (sctx->cur_ino < dir_high_seq_ino) {
3259			ret = 0;
3260			goto out;
3261		}
3262
3263		btrfs_release_path(path);
3264	}
3265
3266	key.objectid = dir;
3267	key.type = BTRFS_DIR_INDEX_KEY;
3268	key.offset = (odi ? odi->last_dir_index_offset : 0);
 
 
 
 
 
 
 
 
3269
3270	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
3271		struct waiting_dir_move *dm;
3272
 
 
 
 
 
 
 
 
 
 
3273		if (found_key.objectid != key.objectid ||
3274		    found_key.type != key.type)
3275			break;
3276
3277		di = btrfs_item_ptr(path->nodes[0], path->slots[0],
3278				struct btrfs_dir_item);
3279		btrfs_dir_item_key_to_cpu(path->nodes[0], di, &loc);
3280
3281		dir_high_seq_ino = max(dir_high_seq_ino, loc.objectid);
3282		last_dir_index_offset = found_key.offset;
3283
3284		dm = get_waiting_dir_move(sctx, loc.objectid);
3285		if (dm) {
 
 
 
 
 
 
 
3286			dm->rmdir_ino = dir;
3287			dm->rmdir_gen = dir_gen;
3288			ret = 0;
3289			goto out;
3290		}
3291
3292		if (loc.objectid > sctx->cur_ino) {
 
 
 
 
 
 
 
3293			ret = 0;
3294			goto out;
3295		}
3296	}
3297	if (iter_ret < 0) {
3298		ret = iter_ret;
3299		goto out;
3300	}
3301	free_orphan_dir_info(sctx, odi);
3302
3303	ret = 1;
3304
3305out:
3306	btrfs_free_path(path);
3307
3308	if (ret)
3309		return ret;
3310
3311	if (!odi) {
3312		odi = add_orphan_dir_info(sctx, dir, dir_gen);
3313		if (IS_ERR(odi))
3314			return PTR_ERR(odi);
3315
3316		odi->gen = dir_gen;
3317	}
3318
3319	odi->last_dir_index_offset = last_dir_index_offset;
3320	odi->dir_high_seq_ino = max(odi->dir_high_seq_ino, dir_high_seq_ino);
3321
3322	return 0;
3323}
3324
3325static int is_waiting_for_move(struct send_ctx *sctx, u64 ino)
3326{
3327	struct waiting_dir_move *entry = get_waiting_dir_move(sctx, ino);
3328
3329	return entry != NULL;
3330}
3331
3332static int add_waiting_dir_move(struct send_ctx *sctx, u64 ino, bool orphanized)
3333{
3334	struct rb_node **p = &sctx->waiting_dir_moves.rb_node;
3335	struct rb_node *parent = NULL;
3336	struct waiting_dir_move *entry, *dm;
3337
3338	dm = kmalloc(sizeof(*dm), GFP_KERNEL);
3339	if (!dm)
3340		return -ENOMEM;
3341	dm->ino = ino;
3342	dm->rmdir_ino = 0;
3343	dm->rmdir_gen = 0;
3344	dm->orphanized = orphanized;
3345
3346	while (*p) {
3347		parent = *p;
3348		entry = rb_entry(parent, struct waiting_dir_move, node);
3349		if (ino < entry->ino) {
3350			p = &(*p)->rb_left;
3351		} else if (ino > entry->ino) {
3352			p = &(*p)->rb_right;
3353		} else {
3354			kfree(dm);
3355			return -EEXIST;
3356		}
3357	}
3358
3359	rb_link_node(&dm->node, parent, p);
3360	rb_insert_color(&dm->node, &sctx->waiting_dir_moves);
3361	return 0;
3362}
3363
3364static struct waiting_dir_move *
3365get_waiting_dir_move(struct send_ctx *sctx, u64 ino)
3366{
3367	struct rb_node *n = sctx->waiting_dir_moves.rb_node;
3368	struct waiting_dir_move *entry;
3369
3370	while (n) {
3371		entry = rb_entry(n, struct waiting_dir_move, node);
3372		if (ino < entry->ino)
3373			n = n->rb_left;
3374		else if (ino > entry->ino)
3375			n = n->rb_right;
3376		else
3377			return entry;
3378	}
3379	return NULL;
3380}
3381
3382static void free_waiting_dir_move(struct send_ctx *sctx,
3383				  struct waiting_dir_move *dm)
3384{
3385	if (!dm)
3386		return;
3387	rb_erase(&dm->node, &sctx->waiting_dir_moves);
3388	kfree(dm);
3389}
3390
3391static int add_pending_dir_move(struct send_ctx *sctx,
3392				u64 ino,
3393				u64 ino_gen,
3394				u64 parent_ino,
3395				struct list_head *new_refs,
3396				struct list_head *deleted_refs,
3397				const bool is_orphan)
3398{
3399	struct rb_node **p = &sctx->pending_dir_moves.rb_node;
3400	struct rb_node *parent = NULL;
3401	struct pending_dir_move *entry = NULL, *pm;
3402	struct recorded_ref *cur;
3403	int exists = 0;
3404	int ret;
3405
3406	pm = kmalloc(sizeof(*pm), GFP_KERNEL);
3407	if (!pm)
3408		return -ENOMEM;
3409	pm->parent_ino = parent_ino;
3410	pm->ino = ino;
3411	pm->gen = ino_gen;
3412	INIT_LIST_HEAD(&pm->list);
3413	INIT_LIST_HEAD(&pm->update_refs);
3414	RB_CLEAR_NODE(&pm->node);
3415
3416	while (*p) {
3417		parent = *p;
3418		entry = rb_entry(parent, struct pending_dir_move, node);
3419		if (parent_ino < entry->parent_ino) {
3420			p = &(*p)->rb_left;
3421		} else if (parent_ino > entry->parent_ino) {
3422			p = &(*p)->rb_right;
3423		} else {
3424			exists = 1;
3425			break;
3426		}
3427	}
3428
3429	list_for_each_entry(cur, deleted_refs, list) {
3430		ret = dup_ref(cur, &pm->update_refs);
3431		if (ret < 0)
3432			goto out;
3433	}
3434	list_for_each_entry(cur, new_refs, list) {
3435		ret = dup_ref(cur, &pm->update_refs);
3436		if (ret < 0)
3437			goto out;
3438	}
3439
3440	ret = add_waiting_dir_move(sctx, pm->ino, is_orphan);
3441	if (ret)
3442		goto out;
3443
3444	if (exists) {
3445		list_add_tail(&pm->list, &entry->list);
3446	} else {
3447		rb_link_node(&pm->node, parent, p);
3448		rb_insert_color(&pm->node, &sctx->pending_dir_moves);
3449	}
3450	ret = 0;
3451out:
3452	if (ret) {
3453		__free_recorded_refs(&pm->update_refs);
3454		kfree(pm);
3455	}
3456	return ret;
3457}
3458
3459static struct pending_dir_move *get_pending_dir_moves(struct send_ctx *sctx,
3460						      u64 parent_ino)
3461{
3462	struct rb_node *n = sctx->pending_dir_moves.rb_node;
3463	struct pending_dir_move *entry;
3464
3465	while (n) {
3466		entry = rb_entry(n, struct pending_dir_move, node);
3467		if (parent_ino < entry->parent_ino)
3468			n = n->rb_left;
3469		else if (parent_ino > entry->parent_ino)
3470			n = n->rb_right;
3471		else
3472			return entry;
3473	}
3474	return NULL;
3475}
3476
3477static int path_loop(struct send_ctx *sctx, struct fs_path *name,
3478		     u64 ino, u64 gen, u64 *ancestor_ino)
3479{
3480	int ret = 0;
3481	u64 parent_inode = 0;
3482	u64 parent_gen = 0;
3483	u64 start_ino = ino;
3484
3485	*ancestor_ino = 0;
3486	while (ino != BTRFS_FIRST_FREE_OBJECTID) {
3487		fs_path_reset(name);
3488
3489		if (is_waiting_for_rm(sctx, ino, gen))
3490			break;
3491		if (is_waiting_for_move(sctx, ino)) {
3492			if (*ancestor_ino == 0)
3493				*ancestor_ino = ino;
3494			ret = get_first_ref(sctx->parent_root, ino,
3495					    &parent_inode, &parent_gen, name);
3496		} else {
3497			ret = __get_cur_name_and_parent(sctx, ino, gen,
3498							&parent_inode,
3499							&parent_gen, name);
3500			if (ret > 0) {
3501				ret = 0;
3502				break;
3503			}
3504		}
3505		if (ret < 0)
3506			break;
3507		if (parent_inode == start_ino) {
3508			ret = 1;
3509			if (*ancestor_ino == 0)
3510				*ancestor_ino = ino;
3511			break;
3512		}
3513		ino = parent_inode;
3514		gen = parent_gen;
3515	}
3516	return ret;
3517}
3518
3519static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm)
3520{
3521	struct fs_path *from_path = NULL;
3522	struct fs_path *to_path = NULL;
3523	struct fs_path *name = NULL;
3524	u64 orig_progress = sctx->send_progress;
3525	struct recorded_ref *cur;
3526	u64 parent_ino, parent_gen;
3527	struct waiting_dir_move *dm = NULL;
3528	u64 rmdir_ino = 0;
3529	u64 rmdir_gen;
3530	u64 ancestor;
3531	bool is_orphan;
3532	int ret;
3533
3534	name = fs_path_alloc();
3535	from_path = fs_path_alloc();
3536	if (!name || !from_path) {
3537		ret = -ENOMEM;
3538		goto out;
3539	}
3540
3541	dm = get_waiting_dir_move(sctx, pm->ino);
3542	ASSERT(dm);
3543	rmdir_ino = dm->rmdir_ino;
3544	rmdir_gen = dm->rmdir_gen;
3545	is_orphan = dm->orphanized;
3546	free_waiting_dir_move(sctx, dm);
3547
3548	if (is_orphan) {
3549		ret = gen_unique_name(sctx, pm->ino,
3550				      pm->gen, from_path);
3551	} else {
3552		ret = get_first_ref(sctx->parent_root, pm->ino,
3553				    &parent_ino, &parent_gen, name);
3554		if (ret < 0)
3555			goto out;
3556		ret = get_cur_path(sctx, parent_ino, parent_gen,
3557				   from_path);
3558		if (ret < 0)
3559			goto out;
3560		ret = fs_path_add_path(from_path, name);
3561	}
3562	if (ret < 0)
3563		goto out;
3564
3565	sctx->send_progress = sctx->cur_ino + 1;
3566	ret = path_loop(sctx, name, pm->ino, pm->gen, &ancestor);
3567	if (ret < 0)
3568		goto out;
3569	if (ret) {
3570		LIST_HEAD(deleted_refs);
3571		ASSERT(ancestor > BTRFS_FIRST_FREE_OBJECTID);
3572		ret = add_pending_dir_move(sctx, pm->ino, pm->gen, ancestor,
3573					   &pm->update_refs, &deleted_refs,
3574					   is_orphan);
3575		if (ret < 0)
3576			goto out;
3577		if (rmdir_ino) {
3578			dm = get_waiting_dir_move(sctx, pm->ino);
3579			ASSERT(dm);
3580			dm->rmdir_ino = rmdir_ino;
3581			dm->rmdir_gen = rmdir_gen;
3582		}
3583		goto out;
3584	}
3585	fs_path_reset(name);
3586	to_path = name;
3587	name = NULL;
3588	ret = get_cur_path(sctx, pm->ino, pm->gen, to_path);
3589	if (ret < 0)
3590		goto out;
3591
3592	ret = send_rename(sctx, from_path, to_path);
3593	if (ret < 0)
3594		goto out;
3595
3596	if (rmdir_ino) {
3597		struct orphan_dir_info *odi;
3598		u64 gen;
3599
3600		odi = get_orphan_dir_info(sctx, rmdir_ino, rmdir_gen);
3601		if (!odi) {
3602			/* already deleted */
3603			goto finish;
3604		}
3605		gen = odi->gen;
3606
3607		ret = can_rmdir(sctx, rmdir_ino, gen);
3608		if (ret < 0)
3609			goto out;
3610		if (!ret)
3611			goto finish;
3612
3613		name = fs_path_alloc();
3614		if (!name) {
3615			ret = -ENOMEM;
3616			goto out;
3617		}
3618		ret = get_cur_path(sctx, rmdir_ino, gen, name);
3619		if (ret < 0)
3620			goto out;
3621		ret = send_rmdir(sctx, name);
3622		if (ret < 0)
3623			goto out;
3624	}
3625
3626finish:
3627	ret = cache_dir_utimes(sctx, pm->ino, pm->gen);
3628	if (ret < 0)
3629		goto out;
3630
3631	/*
3632	 * After rename/move, need to update the utimes of both new parent(s)
3633	 * and old parent(s).
3634	 */
3635	list_for_each_entry(cur, &pm->update_refs, list) {
3636		/*
3637		 * The parent inode might have been deleted in the send snapshot
3638		 */
3639		ret = get_inode_info(sctx->send_root, cur->dir, NULL);
 
3640		if (ret == -ENOENT) {
3641			ret = 0;
3642			continue;
3643		}
3644		if (ret < 0)
3645			goto out;
3646
3647		ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen);
3648		if (ret < 0)
3649			goto out;
3650	}
3651
3652out:
3653	fs_path_free(name);
3654	fs_path_free(from_path);
3655	fs_path_free(to_path);
3656	sctx->send_progress = orig_progress;
3657
3658	return ret;
3659}
3660
3661static void free_pending_move(struct send_ctx *sctx, struct pending_dir_move *m)
3662{
3663	if (!list_empty(&m->list))
3664		list_del(&m->list);
3665	if (!RB_EMPTY_NODE(&m->node))
3666		rb_erase(&m->node, &sctx->pending_dir_moves);
3667	__free_recorded_refs(&m->update_refs);
3668	kfree(m);
3669}
3670
3671static void tail_append_pending_moves(struct send_ctx *sctx,
3672				      struct pending_dir_move *moves,
3673				      struct list_head *stack)
3674{
3675	if (list_empty(&moves->list)) {
3676		list_add_tail(&moves->list, stack);
3677	} else {
3678		LIST_HEAD(list);
3679		list_splice_init(&moves->list, &list);
3680		list_add_tail(&moves->list, stack);
3681		list_splice_tail(&list, stack);
3682	}
3683	if (!RB_EMPTY_NODE(&moves->node)) {
3684		rb_erase(&moves->node, &sctx->pending_dir_moves);
3685		RB_CLEAR_NODE(&moves->node);
3686	}
3687}
3688
3689static int apply_children_dir_moves(struct send_ctx *sctx)
3690{
3691	struct pending_dir_move *pm;
3692	LIST_HEAD(stack);
3693	u64 parent_ino = sctx->cur_ino;
3694	int ret = 0;
3695
3696	pm = get_pending_dir_moves(sctx, parent_ino);
3697	if (!pm)
3698		return 0;
3699
 
3700	tail_append_pending_moves(sctx, pm, &stack);
3701
3702	while (!list_empty(&stack)) {
3703		pm = list_first_entry(&stack, struct pending_dir_move, list);
3704		parent_ino = pm->ino;
3705		ret = apply_dir_move(sctx, pm);
3706		free_pending_move(sctx, pm);
3707		if (ret)
3708			goto out;
3709		pm = get_pending_dir_moves(sctx, parent_ino);
3710		if (pm)
3711			tail_append_pending_moves(sctx, pm, &stack);
3712	}
3713	return 0;
3714
3715out:
3716	while (!list_empty(&stack)) {
3717		pm = list_first_entry(&stack, struct pending_dir_move, list);
3718		free_pending_move(sctx, pm);
3719	}
3720	return ret;
3721}
3722
3723/*
3724 * We might need to delay a directory rename even when no ancestor directory
3725 * (in the send root) with a higher inode number than ours (sctx->cur_ino) was
3726 * renamed. This happens when we rename a directory to the old name (the name
3727 * in the parent root) of some other unrelated directory that got its rename
3728 * delayed due to some ancestor with higher number that got renamed.
3729 *
3730 * Example:
3731 *
3732 * Parent snapshot:
3733 * .                                       (ino 256)
3734 * |---- a/                                (ino 257)
3735 * |     |---- file                        (ino 260)
3736 * |
3737 * |---- b/                                (ino 258)
3738 * |---- c/                                (ino 259)
3739 *
3740 * Send snapshot:
3741 * .                                       (ino 256)
3742 * |---- a/                                (ino 258)
3743 * |---- x/                                (ino 259)
3744 *       |---- y/                          (ino 257)
3745 *             |----- file                 (ino 260)
3746 *
3747 * Here we can not rename 258 from 'b' to 'a' without the rename of inode 257
3748 * from 'a' to 'x/y' happening first, which in turn depends on the rename of
3749 * inode 259 from 'c' to 'x'. So the order of rename commands the send stream
3750 * must issue is:
3751 *
3752 * 1 - rename 259 from 'c' to 'x'
3753 * 2 - rename 257 from 'a' to 'x/y'
3754 * 3 - rename 258 from 'b' to 'a'
3755 *
3756 * Returns 1 if the rename of sctx->cur_ino needs to be delayed, 0 if it can
3757 * be done right away and < 0 on error.
3758 */
3759static int wait_for_dest_dir_move(struct send_ctx *sctx,
3760				  struct recorded_ref *parent_ref,
3761				  const bool is_orphan)
3762{
 
3763	struct btrfs_path *path;
3764	struct btrfs_key key;
3765	struct btrfs_key di_key;
3766	struct btrfs_dir_item *di;
3767	u64 left_gen;
3768	u64 right_gen;
3769	int ret = 0;
3770	struct waiting_dir_move *wdm;
3771
3772	if (RB_EMPTY_ROOT(&sctx->waiting_dir_moves))
3773		return 0;
3774
3775	path = alloc_path_for_send();
3776	if (!path)
3777		return -ENOMEM;
3778
3779	key.objectid = parent_ref->dir;
3780	key.type = BTRFS_DIR_ITEM_KEY;
3781	key.offset = btrfs_name_hash(parent_ref->name, parent_ref->name_len);
3782
3783	ret = btrfs_search_slot(NULL, sctx->parent_root, &key, path, 0, 0);
3784	if (ret < 0) {
3785		goto out;
3786	} else if (ret > 0) {
3787		ret = 0;
3788		goto out;
3789	}
3790
3791	di = btrfs_match_dir_item_name(path, parent_ref->name,
3792				       parent_ref->name_len);
3793	if (!di) {
3794		ret = 0;
3795		goto out;
3796	}
3797	/*
3798	 * di_key.objectid has the number of the inode that has a dentry in the
3799	 * parent directory with the same name that sctx->cur_ino is being
3800	 * renamed to. We need to check if that inode is in the send root as
3801	 * well and if it is currently marked as an inode with a pending rename,
3802	 * if it is, we need to delay the rename of sctx->cur_ino as well, so
3803	 * that it happens after that other inode is renamed.
3804	 */
3805	btrfs_dir_item_key_to_cpu(path->nodes[0], di, &di_key);
3806	if (di_key.type != BTRFS_INODE_ITEM_KEY) {
3807		ret = 0;
3808		goto out;
3809	}
3810
3811	ret = get_inode_gen(sctx->parent_root, di_key.objectid, &left_gen);
 
3812	if (ret < 0)
3813		goto out;
3814	ret = get_inode_gen(sctx->send_root, di_key.objectid, &right_gen);
 
3815	if (ret < 0) {
3816		if (ret == -ENOENT)
3817			ret = 0;
3818		goto out;
3819	}
3820
3821	/* Different inode, no need to delay the rename of sctx->cur_ino */
3822	if (right_gen != left_gen) {
3823		ret = 0;
3824		goto out;
3825	}
3826
3827	wdm = get_waiting_dir_move(sctx, di_key.objectid);
3828	if (wdm && !wdm->orphanized) {
3829		ret = add_pending_dir_move(sctx,
3830					   sctx->cur_ino,
3831					   sctx->cur_inode_gen,
3832					   di_key.objectid,
3833					   &sctx->new_refs,
3834					   &sctx->deleted_refs,
3835					   is_orphan);
3836		if (!ret)
3837			ret = 1;
3838	}
3839out:
3840	btrfs_free_path(path);
3841	return ret;
3842}
3843
3844/*
3845 * Check if inode ino2, or any of its ancestors, is inode ino1.
3846 * Return 1 if true, 0 if false and < 0 on error.
3847 */
3848static int check_ino_in_path(struct btrfs_root *root,
3849			     const u64 ino1,
3850			     const u64 ino1_gen,
3851			     const u64 ino2,
3852			     const u64 ino2_gen,
3853			     struct fs_path *fs_path)
3854{
3855	u64 ino = ino2;
3856
3857	if (ino1 == ino2)
3858		return ino1_gen == ino2_gen;
3859
3860	while (ino > BTRFS_FIRST_FREE_OBJECTID) {
3861		u64 parent;
3862		u64 parent_gen;
3863		int ret;
3864
3865		fs_path_reset(fs_path);
3866		ret = get_first_ref(root, ino, &parent, &parent_gen, fs_path);
3867		if (ret < 0)
3868			return ret;
3869		if (parent == ino1)
3870			return parent_gen == ino1_gen;
3871		ino = parent;
3872	}
3873	return 0;
3874}
3875
3876/*
3877 * Check if inode ino1 is an ancestor of inode ino2 in the given root for any
3878 * possible path (in case ino2 is not a directory and has multiple hard links).
3879 * Return 1 if true, 0 if false and < 0 on error.
3880 */
3881static int is_ancestor(struct btrfs_root *root,
3882		       const u64 ino1,
3883		       const u64 ino1_gen,
3884		       const u64 ino2,
3885		       struct fs_path *fs_path)
3886{
3887	bool free_fs_path = false;
3888	int ret = 0;
3889	int iter_ret = 0;
3890	struct btrfs_path *path = NULL;
3891	struct btrfs_key key;
3892
3893	if (!fs_path) {
3894		fs_path = fs_path_alloc();
3895		if (!fs_path)
3896			return -ENOMEM;
3897		free_fs_path = true;
3898	}
3899
3900	path = alloc_path_for_send();
3901	if (!path) {
3902		ret = -ENOMEM;
3903		goto out;
3904	}
3905
3906	key.objectid = ino2;
3907	key.type = BTRFS_INODE_REF_KEY;
3908	key.offset = 0;
3909
3910	btrfs_for_each_slot(root, &key, &key, path, iter_ret) {
 
 
 
 
3911		struct extent_buffer *leaf = path->nodes[0];
3912		int slot = path->slots[0];
3913		u32 cur_offset = 0;
3914		u32 item_size;
3915
 
 
 
 
 
 
 
 
 
 
3916		if (key.objectid != ino2)
3917			break;
3918		if (key.type != BTRFS_INODE_REF_KEY &&
3919		    key.type != BTRFS_INODE_EXTREF_KEY)
3920			break;
3921
3922		item_size = btrfs_item_size(leaf, slot);
3923		while (cur_offset < item_size) {
3924			u64 parent;
3925			u64 parent_gen;
3926
3927			if (key.type == BTRFS_INODE_EXTREF_KEY) {
3928				unsigned long ptr;
3929				struct btrfs_inode_extref *extref;
3930
3931				ptr = btrfs_item_ptr_offset(leaf, slot);
3932				extref = (struct btrfs_inode_extref *)
3933					(ptr + cur_offset);
3934				parent = btrfs_inode_extref_parent(leaf,
3935								   extref);
3936				cur_offset += sizeof(*extref);
3937				cur_offset += btrfs_inode_extref_name_len(leaf,
3938								  extref);
3939			} else {
3940				parent = key.offset;
3941				cur_offset = item_size;
3942			}
3943
3944			ret = get_inode_gen(root, parent, &parent_gen);
 
3945			if (ret < 0)
3946				goto out;
3947			ret = check_ino_in_path(root, ino1, ino1_gen,
3948						parent, parent_gen, fs_path);
3949			if (ret)
3950				goto out;
3951		}
 
3952	}
3953	ret = 0;
3954	if (iter_ret < 0)
3955		ret = iter_ret;
3956
3957out:
3958	btrfs_free_path(path);
3959	if (free_fs_path)
3960		fs_path_free(fs_path);
3961	return ret;
3962}
3963
3964static int wait_for_parent_move(struct send_ctx *sctx,
3965				struct recorded_ref *parent_ref,
3966				const bool is_orphan)
3967{
3968	int ret = 0;
3969	u64 ino = parent_ref->dir;
3970	u64 ino_gen = parent_ref->dir_gen;
3971	u64 parent_ino_before, parent_ino_after;
3972	struct fs_path *path_before = NULL;
3973	struct fs_path *path_after = NULL;
3974	int len1, len2;
3975
3976	path_after = fs_path_alloc();
3977	path_before = fs_path_alloc();
3978	if (!path_after || !path_before) {
3979		ret = -ENOMEM;
3980		goto out;
3981	}
3982
3983	/*
3984	 * Our current directory inode may not yet be renamed/moved because some
3985	 * ancestor (immediate or not) has to be renamed/moved first. So find if
3986	 * such ancestor exists and make sure our own rename/move happens after
3987	 * that ancestor is processed to avoid path build infinite loops (done
3988	 * at get_cur_path()).
3989	 */
3990	while (ino > BTRFS_FIRST_FREE_OBJECTID) {
3991		u64 parent_ino_after_gen;
3992
3993		if (is_waiting_for_move(sctx, ino)) {
3994			/*
3995			 * If the current inode is an ancestor of ino in the
3996			 * parent root, we need to delay the rename of the
3997			 * current inode, otherwise don't delayed the rename
3998			 * because we can end up with a circular dependency
3999			 * of renames, resulting in some directories never
4000			 * getting the respective rename operations issued in
4001			 * the send stream or getting into infinite path build
4002			 * loops.
4003			 */
4004			ret = is_ancestor(sctx->parent_root,
4005					  sctx->cur_ino, sctx->cur_inode_gen,
4006					  ino, path_before);
4007			if (ret)
4008				break;
4009		}
4010
4011		fs_path_reset(path_before);
4012		fs_path_reset(path_after);
4013
4014		ret = get_first_ref(sctx->send_root, ino, &parent_ino_after,
4015				    &parent_ino_after_gen, path_after);
4016		if (ret < 0)
4017			goto out;
4018		ret = get_first_ref(sctx->parent_root, ino, &parent_ino_before,
4019				    NULL, path_before);
4020		if (ret < 0 && ret != -ENOENT) {
4021			goto out;
4022		} else if (ret == -ENOENT) {
4023			ret = 0;
4024			break;
4025		}
4026
4027		len1 = fs_path_len(path_before);
4028		len2 = fs_path_len(path_after);
4029		if (ino > sctx->cur_ino &&
4030		    (parent_ino_before != parent_ino_after || len1 != len2 ||
4031		     memcmp(path_before->start, path_after->start, len1))) {
4032			u64 parent_ino_gen;
4033
4034			ret = get_inode_gen(sctx->parent_root, ino, &parent_ino_gen);
 
 
4035			if (ret < 0)
4036				goto out;
4037			if (ino_gen == parent_ino_gen) {
4038				ret = 1;
4039				break;
4040			}
4041		}
4042		ino = parent_ino_after;
4043		ino_gen = parent_ino_after_gen;
4044	}
4045
4046out:
4047	fs_path_free(path_before);
4048	fs_path_free(path_after);
4049
4050	if (ret == 1) {
4051		ret = add_pending_dir_move(sctx,
4052					   sctx->cur_ino,
4053					   sctx->cur_inode_gen,
4054					   ino,
4055					   &sctx->new_refs,
4056					   &sctx->deleted_refs,
4057					   is_orphan);
4058		if (!ret)
4059			ret = 1;
4060	}
4061
4062	return ret;
4063}
4064
4065static int update_ref_path(struct send_ctx *sctx, struct recorded_ref *ref)
4066{
4067	int ret;
4068	struct fs_path *new_path;
4069
4070	/*
4071	 * Our reference's name member points to its full_path member string, so
4072	 * we use here a new path.
4073	 */
4074	new_path = fs_path_alloc();
4075	if (!new_path)
4076		return -ENOMEM;
4077
4078	ret = get_cur_path(sctx, ref->dir, ref->dir_gen, new_path);
4079	if (ret < 0) {
4080		fs_path_free(new_path);
4081		return ret;
4082	}
4083	ret = fs_path_add(new_path, ref->name, ref->name_len);
4084	if (ret < 0) {
4085		fs_path_free(new_path);
4086		return ret;
4087	}
4088
4089	fs_path_free(ref->full_path);
4090	set_ref_path(ref, new_path);
4091
4092	return 0;
4093}
4094
4095/*
4096 * When processing the new references for an inode we may orphanize an existing
4097 * directory inode because its old name conflicts with one of the new references
4098 * of the current inode. Later, when processing another new reference of our
4099 * inode, we might need to orphanize another inode, but the path we have in the
4100 * reference reflects the pre-orphanization name of the directory we previously
4101 * orphanized. For example:
4102 *
4103 * parent snapshot looks like:
4104 *
4105 * .                                     (ino 256)
4106 * |----- f1                             (ino 257)
4107 * |----- f2                             (ino 258)
4108 * |----- d1/                            (ino 259)
4109 *        |----- d2/                     (ino 260)
4110 *
4111 * send snapshot looks like:
4112 *
4113 * .                                     (ino 256)
4114 * |----- d1                             (ino 258)
4115 * |----- f2/                            (ino 259)
4116 *        |----- f2_link/                (ino 260)
4117 *        |       |----- f1              (ino 257)
4118 *        |
4119 *        |----- d2                      (ino 258)
4120 *
4121 * When processing inode 257 we compute the name for inode 259 as "d1", and we
4122 * cache it in the name cache. Later when we start processing inode 258, when
4123 * collecting all its new references we set a full path of "d1/d2" for its new
4124 * reference with name "d2". When we start processing the new references we
4125 * start by processing the new reference with name "d1", and this results in
4126 * orphanizing inode 259, since its old reference causes a conflict. Then we
4127 * move on the next new reference, with name "d2", and we find out we must
4128 * orphanize inode 260, as its old reference conflicts with ours - but for the
4129 * orphanization we use a source path corresponding to the path we stored in the
4130 * new reference, which is "d1/d2" and not "o259-6-0/d2" - this makes the
4131 * receiver fail since the path component "d1/" no longer exists, it was renamed
4132 * to "o259-6-0/" when processing the previous new reference. So in this case we
4133 * must recompute the path in the new reference and use it for the new
4134 * orphanization operation.
4135 */
4136static int refresh_ref_path(struct send_ctx *sctx, struct recorded_ref *ref)
4137{
4138	char *name;
4139	int ret;
4140
4141	name = kmemdup(ref->name, ref->name_len, GFP_KERNEL);
4142	if (!name)
4143		return -ENOMEM;
4144
4145	fs_path_reset(ref->full_path);
4146	ret = get_cur_path(sctx, ref->dir, ref->dir_gen, ref->full_path);
4147	if (ret < 0)
4148		goto out;
4149
4150	ret = fs_path_add(ref->full_path, name, ref->name_len);
4151	if (ret < 0)
4152		goto out;
4153
4154	/* Update the reference's base name pointer. */
4155	set_ref_path(ref, ref->full_path);
4156out:
4157	kfree(name);
4158	return ret;
4159}
4160
4161/*
4162 * This does all the move/link/unlink/rmdir magic.
4163 */
4164static int process_recorded_refs(struct send_ctx *sctx, int *pending_move)
4165{
4166	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
4167	int ret = 0;
4168	struct recorded_ref *cur;
4169	struct recorded_ref *cur2;
4170	LIST_HEAD(check_dirs);
4171	struct fs_path *valid_path = NULL;
4172	u64 ow_inode = 0;
4173	u64 ow_gen;
4174	u64 ow_mode;
4175	int did_overwrite = 0;
4176	int is_orphan = 0;
4177	u64 last_dir_ino_rm = 0;
4178	bool can_rename = true;
4179	bool orphanized_dir = false;
4180	bool orphanized_ancestor = false;
4181
4182	btrfs_debug(fs_info, "process_recorded_refs %llu", sctx->cur_ino);
4183
4184	/*
4185	 * This should never happen as the root dir always has the same ref
4186	 * which is always '..'
4187	 */
4188	if (unlikely(sctx->cur_ino <= BTRFS_FIRST_FREE_OBJECTID)) {
4189		btrfs_err(fs_info,
4190			  "send: unexpected inode %llu in process_recorded_refs()",
4191			  sctx->cur_ino);
4192		ret = -EINVAL;
4193		goto out;
4194	}
4195
4196	valid_path = fs_path_alloc();
4197	if (!valid_path) {
4198		ret = -ENOMEM;
4199		goto out;
4200	}
4201
4202	/*
4203	 * First, check if the first ref of the current inode was overwritten
4204	 * before. If yes, we know that the current inode was already orphanized
4205	 * and thus use the orphan name. If not, we can use get_cur_path to
4206	 * get the path of the first ref as it would like while receiving at
4207	 * this point in time.
4208	 * New inodes are always orphan at the beginning, so force to use the
4209	 * orphan name in this case.
4210	 * The first ref is stored in valid_path and will be updated if it
4211	 * gets moved around.
4212	 */
4213	if (!sctx->cur_inode_new) {
4214		ret = did_overwrite_first_ref(sctx, sctx->cur_ino,
4215				sctx->cur_inode_gen);
4216		if (ret < 0)
4217			goto out;
4218		if (ret)
4219			did_overwrite = 1;
4220	}
4221	if (sctx->cur_inode_new || did_overwrite) {
4222		ret = gen_unique_name(sctx, sctx->cur_ino,
4223				sctx->cur_inode_gen, valid_path);
4224		if (ret < 0)
4225			goto out;
4226		is_orphan = 1;
4227	} else {
4228		ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen,
4229				valid_path);
4230		if (ret < 0)
4231			goto out;
4232	}
4233
4234	/*
4235	 * Before doing any rename and link operations, do a first pass on the
4236	 * new references to orphanize any unprocessed inodes that may have a
4237	 * reference that conflicts with one of the new references of the current
4238	 * inode. This needs to happen first because a new reference may conflict
4239	 * with the old reference of a parent directory, so we must make sure
4240	 * that the path used for link and rename commands don't use an
4241	 * orphanized name when an ancestor was not yet orphanized.
4242	 *
4243	 * Example:
4244	 *
4245	 * Parent snapshot:
4246	 *
4247	 * .                                                      (ino 256)
4248	 * |----- testdir/                                        (ino 259)
4249	 * |          |----- a                                    (ino 257)
4250	 * |
4251	 * |----- b                                               (ino 258)
4252	 *
4253	 * Send snapshot:
4254	 *
4255	 * .                                                      (ino 256)
4256	 * |----- testdir_2/                                      (ino 259)
4257	 * |          |----- a                                    (ino 260)
4258	 * |
4259	 * |----- testdir                                         (ino 257)
4260	 * |----- b                                               (ino 257)
4261	 * |----- b2                                              (ino 258)
4262	 *
4263	 * Processing the new reference for inode 257 with name "b" may happen
4264	 * before processing the new reference with name "testdir". If so, we
4265	 * must make sure that by the time we send a link command to create the
4266	 * hard link "b", inode 259 was already orphanized, since the generated
4267	 * path in "valid_path" already contains the orphanized name for 259.
4268	 * We are processing inode 257, so only later when processing 259 we do
4269	 * the rename operation to change its temporary (orphanized) name to
4270	 * "testdir_2".
4271	 */
4272	list_for_each_entry(cur, &sctx->new_refs, list) {
4273		ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL);
 
 
 
 
 
 
 
4274		if (ret < 0)
4275			goto out;
4276		if (ret == inode_state_will_create)
4277			continue;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4278
4279		/*
4280		 * Check if this new ref would overwrite the first ref of another
4281		 * unprocessed inode. If yes, orphanize the overwritten inode.
4282		 * If we find an overwritten ref that is not the first ref,
4283		 * simply unlink it.
4284		 */
4285		ret = will_overwrite_ref(sctx, cur->dir, cur->dir_gen,
4286				cur->name, cur->name_len,
4287				&ow_inode, &ow_gen, &ow_mode);
4288		if (ret < 0)
4289			goto out;
4290		if (ret) {
4291			ret = is_first_ref(sctx->parent_root,
4292					   ow_inode, cur->dir, cur->name,
4293					   cur->name_len);
4294			if (ret < 0)
4295				goto out;
4296			if (ret) {
4297				struct name_cache_entry *nce;
4298				struct waiting_dir_move *wdm;
4299
4300				if (orphanized_dir) {
4301					ret = refresh_ref_path(sctx, cur);
4302					if (ret < 0)
4303						goto out;
4304				}
4305
4306				ret = orphanize_inode(sctx, ow_inode, ow_gen,
4307						cur->full_path);
4308				if (ret < 0)
4309					goto out;
4310				if (S_ISDIR(ow_mode))
4311					orphanized_dir = true;
4312
4313				/*
4314				 * If ow_inode has its rename operation delayed
4315				 * make sure that its orphanized name is used in
4316				 * the source path when performing its rename
4317				 * operation.
4318				 */
4319				wdm = get_waiting_dir_move(sctx, ow_inode);
4320				if (wdm)
 
 
4321					wdm->orphanized = true;
 
4322
4323				/*
4324				 * Make sure we clear our orphanized inode's
4325				 * name from the name cache. This is because the
4326				 * inode ow_inode might be an ancestor of some
4327				 * other inode that will be orphanized as well
4328				 * later and has an inode number greater than
4329				 * sctx->send_progress. We need to prevent
4330				 * future name lookups from using the old name
4331				 * and get instead the orphan name.
4332				 */
4333				nce = name_cache_search(sctx, ow_inode, ow_gen);
4334				if (nce)
4335					btrfs_lru_cache_remove(&sctx->name_cache,
4336							       &nce->entry);
 
4337
4338				/*
4339				 * ow_inode might currently be an ancestor of
4340				 * cur_ino, therefore compute valid_path (the
4341				 * current path of cur_ino) again because it
4342				 * might contain the pre-orphanization name of
4343				 * ow_inode, which is no longer valid.
4344				 */
4345				ret = is_ancestor(sctx->parent_root,
4346						  ow_inode, ow_gen,
4347						  sctx->cur_ino, NULL);
4348				if (ret > 0) {
4349					orphanized_ancestor = true;
4350					fs_path_reset(valid_path);
4351					ret = get_cur_path(sctx, sctx->cur_ino,
4352							   sctx->cur_inode_gen,
4353							   valid_path);
4354				}
4355				if (ret < 0)
4356					goto out;
4357			} else {
4358				/*
4359				 * If we previously orphanized a directory that
4360				 * collided with a new reference that we already
4361				 * processed, recompute the current path because
4362				 * that directory may be part of the path.
4363				 */
4364				if (orphanized_dir) {
4365					ret = refresh_ref_path(sctx, cur);
4366					if (ret < 0)
4367						goto out;
4368				}
4369				ret = send_unlink(sctx, cur->full_path);
4370				if (ret < 0)
4371					goto out;
4372			}
4373		}
4374
4375	}
4376
4377	list_for_each_entry(cur, &sctx->new_refs, list) {
4378		/*
4379		 * We may have refs where the parent directory does not exist
4380		 * yet. This happens if the parent directories inum is higher
4381		 * than the current inum. To handle this case, we create the
4382		 * parent directory out of order. But we need to check if this
4383		 * did already happen before due to other refs in the same dir.
4384		 */
4385		ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL);
4386		if (ret < 0)
4387			goto out;
4388		if (ret == inode_state_will_create) {
4389			ret = 0;
4390			/*
4391			 * First check if any of the current inodes refs did
4392			 * already create the dir.
4393			 */
4394			list_for_each_entry(cur2, &sctx->new_refs, list) {
4395				if (cur == cur2)
4396					break;
4397				if (cur2->dir == cur->dir) {
4398					ret = 1;
4399					break;
4400				}
4401			}
4402
4403			/*
4404			 * If that did not happen, check if a previous inode
4405			 * did already create the dir.
4406			 */
4407			if (!ret)
4408				ret = did_create_dir(sctx, cur->dir);
4409			if (ret < 0)
4410				goto out;
4411			if (!ret) {
4412				ret = send_create_inode(sctx, cur->dir);
4413				if (ret < 0)
4414					goto out;
4415				cache_dir_created(sctx, cur->dir);
4416			}
4417		}
4418
4419		if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root) {
4420			ret = wait_for_dest_dir_move(sctx, cur, is_orphan);
4421			if (ret < 0)
4422				goto out;
4423			if (ret == 1) {
4424				can_rename = false;
4425				*pending_move = 1;
4426			}
4427		}
4428
4429		if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root &&
4430		    can_rename) {
4431			ret = wait_for_parent_move(sctx, cur, is_orphan);
4432			if (ret < 0)
4433				goto out;
4434			if (ret == 1) {
4435				can_rename = false;
4436				*pending_move = 1;
4437			}
4438		}
4439
4440		/*
4441		 * link/move the ref to the new place. If we have an orphan
4442		 * inode, move it and update valid_path. If not, link or move
4443		 * it depending on the inode mode.
4444		 */
4445		if (is_orphan && can_rename) {
4446			ret = send_rename(sctx, valid_path, cur->full_path);
4447			if (ret < 0)
4448				goto out;
4449			is_orphan = 0;
4450			ret = fs_path_copy(valid_path, cur->full_path);
4451			if (ret < 0)
4452				goto out;
4453		} else if (can_rename) {
4454			if (S_ISDIR(sctx->cur_inode_mode)) {
4455				/*
4456				 * Dirs can't be linked, so move it. For moved
4457				 * dirs, we always have one new and one deleted
4458				 * ref. The deleted ref is ignored later.
4459				 */
4460				ret = send_rename(sctx, valid_path,
4461						  cur->full_path);
4462				if (!ret)
4463					ret = fs_path_copy(valid_path,
4464							   cur->full_path);
4465				if (ret < 0)
4466					goto out;
4467			} else {
4468				/*
4469				 * We might have previously orphanized an inode
4470				 * which is an ancestor of our current inode,
4471				 * so our reference's full path, which was
4472				 * computed before any such orphanizations, must
4473				 * be updated.
4474				 */
4475				if (orphanized_dir) {
4476					ret = update_ref_path(sctx, cur);
4477					if (ret < 0)
4478						goto out;
4479				}
4480				ret = send_link(sctx, cur->full_path,
4481						valid_path);
4482				if (ret < 0)
4483					goto out;
4484			}
4485		}
4486		ret = dup_ref(cur, &check_dirs);
4487		if (ret < 0)
4488			goto out;
4489	}
4490
4491	if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_deleted) {
4492		/*
4493		 * Check if we can already rmdir the directory. If not,
4494		 * orphanize it. For every dir item inside that gets deleted
4495		 * later, we do this check again and rmdir it then if possible.
4496		 * See the use of check_dirs for more details.
4497		 */
4498		ret = can_rmdir(sctx, sctx->cur_ino, sctx->cur_inode_gen);
 
4499		if (ret < 0)
4500			goto out;
4501		if (ret) {
4502			ret = send_rmdir(sctx, valid_path);
4503			if (ret < 0)
4504				goto out;
4505		} else if (!is_orphan) {
4506			ret = orphanize_inode(sctx, sctx->cur_ino,
4507					sctx->cur_inode_gen, valid_path);
4508			if (ret < 0)
4509				goto out;
4510			is_orphan = 1;
4511		}
4512
4513		list_for_each_entry(cur, &sctx->deleted_refs, list) {
4514			ret = dup_ref(cur, &check_dirs);
4515			if (ret < 0)
4516				goto out;
4517		}
4518	} else if (S_ISDIR(sctx->cur_inode_mode) &&
4519		   !list_empty(&sctx->deleted_refs)) {
4520		/*
4521		 * We have a moved dir. Add the old parent to check_dirs
4522		 */
4523		cur = list_entry(sctx->deleted_refs.next, struct recorded_ref,
4524				list);
4525		ret = dup_ref(cur, &check_dirs);
4526		if (ret < 0)
4527			goto out;
4528	} else if (!S_ISDIR(sctx->cur_inode_mode)) {
4529		/*
4530		 * We have a non dir inode. Go through all deleted refs and
4531		 * unlink them if they were not already overwritten by other
4532		 * inodes.
4533		 */
4534		list_for_each_entry(cur, &sctx->deleted_refs, list) {
4535			ret = did_overwrite_ref(sctx, cur->dir, cur->dir_gen,
4536					sctx->cur_ino, sctx->cur_inode_gen,
4537					cur->name, cur->name_len);
4538			if (ret < 0)
4539				goto out;
4540			if (!ret) {
4541				/*
4542				 * If we orphanized any ancestor before, we need
4543				 * to recompute the full path for deleted names,
4544				 * since any such path was computed before we
4545				 * processed any references and orphanized any
4546				 * ancestor inode.
4547				 */
4548				if (orphanized_ancestor) {
4549					ret = update_ref_path(sctx, cur);
4550					if (ret < 0)
4551						goto out;
4552				}
4553				ret = send_unlink(sctx, cur->full_path);
4554				if (ret < 0)
4555					goto out;
4556			}
4557			ret = dup_ref(cur, &check_dirs);
4558			if (ret < 0)
4559				goto out;
4560		}
4561		/*
4562		 * If the inode is still orphan, unlink the orphan. This may
4563		 * happen when a previous inode did overwrite the first ref
4564		 * of this inode and no new refs were added for the current
4565		 * inode. Unlinking does not mean that the inode is deleted in
4566		 * all cases. There may still be links to this inode in other
4567		 * places.
4568		 */
4569		if (is_orphan) {
4570			ret = send_unlink(sctx, valid_path);
4571			if (ret < 0)
4572				goto out;
4573		}
4574	}
4575
4576	/*
4577	 * We did collect all parent dirs where cur_inode was once located. We
4578	 * now go through all these dirs and check if they are pending for
4579	 * deletion and if it's finally possible to perform the rmdir now.
4580	 * We also update the inode stats of the parent dirs here.
4581	 */
4582	list_for_each_entry(cur, &check_dirs, list) {
4583		/*
4584		 * In case we had refs into dirs that were not processed yet,
4585		 * we don't need to do the utime and rmdir logic for these dirs.
4586		 * The dir will be processed later.
4587		 */
4588		if (cur->dir > sctx->cur_ino)
4589			continue;
4590
4591		ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL);
4592		if (ret < 0)
4593			goto out;
4594
4595		if (ret == inode_state_did_create ||
4596		    ret == inode_state_no_change) {
4597			ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen);
 
4598			if (ret < 0)
4599				goto out;
4600		} else if (ret == inode_state_did_delete &&
4601			   cur->dir != last_dir_ino_rm) {
4602			ret = can_rmdir(sctx, cur->dir, cur->dir_gen);
 
4603			if (ret < 0)
4604				goto out;
4605			if (ret) {
4606				ret = get_cur_path(sctx, cur->dir,
4607						   cur->dir_gen, valid_path);
4608				if (ret < 0)
4609					goto out;
4610				ret = send_rmdir(sctx, valid_path);
4611				if (ret < 0)
4612					goto out;
4613				last_dir_ino_rm = cur->dir;
4614			}
4615		}
4616	}
4617
4618	ret = 0;
4619
4620out:
4621	__free_recorded_refs(&check_dirs);
4622	free_recorded_refs(sctx);
4623	fs_path_free(valid_path);
4624	return ret;
4625}
4626
4627static int rbtree_ref_comp(const void *k, const struct rb_node *node)
4628{
4629	const struct recorded_ref *data = k;
4630	const struct recorded_ref *ref = rb_entry(node, struct recorded_ref, node);
4631	int result;
4632
4633	if (data->dir > ref->dir)
4634		return 1;
4635	if (data->dir < ref->dir)
4636		return -1;
4637	if (data->dir_gen > ref->dir_gen)
4638		return 1;
4639	if (data->dir_gen < ref->dir_gen)
4640		return -1;
4641	if (data->name_len > ref->name_len)
4642		return 1;
4643	if (data->name_len < ref->name_len)
4644		return -1;
4645	result = strcmp(data->name, ref->name);
4646	if (result > 0)
4647		return 1;
4648	if (result < 0)
4649		return -1;
4650	return 0;
4651}
4652
4653static bool rbtree_ref_less(struct rb_node *node, const struct rb_node *parent)
4654{
4655	const struct recorded_ref *entry = rb_entry(node, struct recorded_ref, node);
4656
4657	return rbtree_ref_comp(entry, parent) < 0;
4658}
4659
4660static int record_ref_in_tree(struct rb_root *root, struct list_head *refs,
4661			      struct fs_path *name, u64 dir, u64 dir_gen,
4662			      struct send_ctx *sctx)
4663{
4664	int ret = 0;
4665	struct fs_path *path = NULL;
4666	struct recorded_ref *ref = NULL;
 
4667
4668	path = fs_path_alloc();
4669	if (!path) {
4670		ret = -ENOMEM;
4671		goto out;
4672	}
4673
4674	ref = recorded_ref_alloc();
4675	if (!ref) {
4676		ret = -ENOMEM;
4677		goto out;
4678	}
4679
4680	ret = get_cur_path(sctx, dir, dir_gen, path);
4681	if (ret < 0)
4682		goto out;
4683	ret = fs_path_add_path(path, name);
4684	if (ret < 0)
4685		goto out;
4686
4687	ref->dir = dir;
4688	ref->dir_gen = dir_gen;
4689	set_ref_path(ref, path);
4690	list_add_tail(&ref->list, refs);
4691	rb_add(&ref->node, root, rbtree_ref_less);
4692	ref->root = root;
4693out:
4694	if (ret) {
4695		if (path && (!ref || !ref->full_path))
4696			fs_path_free(path);
4697		recorded_ref_free(ref);
4698	}
4699	return ret;
4700}
4701
4702static int record_new_ref_if_needed(u64 dir, struct fs_path *name, void *ctx)
 
 
4703{
4704	int ret = 0;
4705	struct send_ctx *sctx = ctx;
4706	struct rb_node *node = NULL;
4707	struct recorded_ref data;
4708	struct recorded_ref *ref;
4709	u64 dir_gen;
4710
4711	ret = get_inode_gen(sctx->send_root, dir, &dir_gen);
4712	if (ret < 0)
4713		goto out;
4714
4715	data.dir = dir;
4716	data.dir_gen = dir_gen;
4717	set_ref_path(&data, name);
4718	node = rb_find(&data, &sctx->rbtree_deleted_refs, rbtree_ref_comp);
4719	if (node) {
4720		ref = rb_entry(node, struct recorded_ref, node);
4721		recorded_ref_free(ref);
4722	} else {
4723		ret = record_ref_in_tree(&sctx->rbtree_new_refs,
4724					 &sctx->new_refs, name, dir, dir_gen,
4725					 sctx);
4726	}
4727out:
4728	return ret;
4729}
4730
4731static int record_deleted_ref_if_needed(u64 dir, struct fs_path *name, void *ctx)
 
 
 
4732{
4733	int ret = 0;
4734	struct send_ctx *sctx = ctx;
4735	struct rb_node *node = NULL;
4736	struct recorded_ref data;
4737	struct recorded_ref *ref;
4738	u64 dir_gen;
4739
4740	ret = get_inode_gen(sctx->parent_root, dir, &dir_gen);
4741	if (ret < 0)
4742		goto out;
4743
4744	data.dir = dir;
4745	data.dir_gen = dir_gen;
4746	set_ref_path(&data, name);
4747	node = rb_find(&data, &sctx->rbtree_new_refs, rbtree_ref_comp);
4748	if (node) {
4749		ref = rb_entry(node, struct recorded_ref, node);
4750		recorded_ref_free(ref);
4751	} else {
4752		ret = record_ref_in_tree(&sctx->rbtree_deleted_refs,
4753					 &sctx->deleted_refs, name, dir,
4754					 dir_gen, sctx);
4755	}
4756out:
4757	return ret;
4758}
4759
4760static int record_new_ref(struct send_ctx *sctx)
4761{
4762	int ret;
4763
4764	ret = iterate_inode_ref(sctx->send_root, sctx->left_path,
4765				sctx->cmp_key, 0, record_new_ref_if_needed, sctx);
4766	if (ret < 0)
4767		goto out;
4768	ret = 0;
4769
4770out:
4771	return ret;
4772}
4773
4774static int record_deleted_ref(struct send_ctx *sctx)
4775{
4776	int ret;
4777
4778	ret = iterate_inode_ref(sctx->parent_root, sctx->right_path,
4779				sctx->cmp_key, 0, record_deleted_ref_if_needed,
4780				sctx);
4781	if (ret < 0)
4782		goto out;
4783	ret = 0;
4784
4785out:
4786	return ret;
4787}
4788
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4789static int record_changed_ref(struct send_ctx *sctx)
4790{
4791	int ret = 0;
4792
4793	ret = iterate_inode_ref(sctx->send_root, sctx->left_path,
4794			sctx->cmp_key, 0, record_new_ref_if_needed, sctx);
4795	if (ret < 0)
4796		goto out;
4797	ret = iterate_inode_ref(sctx->parent_root, sctx->right_path,
4798			sctx->cmp_key, 0, record_deleted_ref_if_needed, sctx);
4799	if (ret < 0)
4800		goto out;
4801	ret = 0;
4802
4803out:
4804	return ret;
4805}
4806
4807/*
4808 * Record and process all refs at once. Needed when an inode changes the
4809 * generation number, which means that it was deleted and recreated.
4810 */
4811static int process_all_refs(struct send_ctx *sctx,
4812			    enum btrfs_compare_tree_result cmd)
4813{
4814	int ret = 0;
4815	int iter_ret = 0;
4816	struct btrfs_root *root;
4817	struct btrfs_path *path;
4818	struct btrfs_key key;
4819	struct btrfs_key found_key;
 
 
4820	iterate_inode_ref_t cb;
4821	int pending_move = 0;
4822
4823	path = alloc_path_for_send();
4824	if (!path)
4825		return -ENOMEM;
4826
4827	if (cmd == BTRFS_COMPARE_TREE_NEW) {
4828		root = sctx->send_root;
4829		cb = record_new_ref_if_needed;
4830	} else if (cmd == BTRFS_COMPARE_TREE_DELETED) {
4831		root = sctx->parent_root;
4832		cb = record_deleted_ref_if_needed;
4833	} else {
4834		btrfs_err(sctx->send_root->fs_info,
4835				"Wrong command %d in process_all_refs", cmd);
4836		ret = -EINVAL;
4837		goto out;
4838	}
4839
4840	key.objectid = sctx->cmp_key->objectid;
4841	key.type = BTRFS_INODE_REF_KEY;
4842	key.offset = 0;
4843	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4844		if (found_key.objectid != key.objectid ||
4845		    (found_key.type != BTRFS_INODE_REF_KEY &&
4846		     found_key.type != BTRFS_INODE_EXTREF_KEY))
4847			break;
4848
4849		ret = iterate_inode_ref(root, path, &found_key, 0, cb, sctx);
4850		if (ret < 0)
4851			goto out;
4852	}
4853	/* Catch error found during iteration */
4854	if (iter_ret < 0) {
4855		ret = iter_ret;
4856		goto out;
4857	}
4858	btrfs_release_path(path);
4859
4860	/*
4861	 * We don't actually care about pending_move as we are simply
4862	 * re-creating this inode and will be rename'ing it into place once we
4863	 * rename the parent directory.
4864	 */
4865	ret = process_recorded_refs(sctx, &pending_move);
4866out:
4867	btrfs_free_path(path);
4868	return ret;
4869}
4870
4871static int send_set_xattr(struct send_ctx *sctx,
4872			  struct fs_path *path,
4873			  const char *name, int name_len,
4874			  const char *data, int data_len)
4875{
4876	int ret = 0;
4877
4878	ret = begin_cmd(sctx, BTRFS_SEND_C_SET_XATTR);
4879	if (ret < 0)
4880		goto out;
4881
4882	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
4883	TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len);
4884	TLV_PUT(sctx, BTRFS_SEND_A_XATTR_DATA, data, data_len);
4885
4886	ret = send_cmd(sctx);
4887
4888tlv_put_failure:
4889out:
4890	return ret;
4891}
4892
4893static int send_remove_xattr(struct send_ctx *sctx,
4894			  struct fs_path *path,
4895			  const char *name, int name_len)
4896{
4897	int ret = 0;
4898
4899	ret = begin_cmd(sctx, BTRFS_SEND_C_REMOVE_XATTR);
4900	if (ret < 0)
4901		goto out;
4902
4903	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
4904	TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len);
4905
4906	ret = send_cmd(sctx);
4907
4908tlv_put_failure:
4909out:
4910	return ret;
4911}
4912
4913static int __process_new_xattr(int num, struct btrfs_key *di_key,
4914			       const char *name, int name_len, const char *data,
4915			       int data_len, void *ctx)
 
4916{
4917	int ret;
4918	struct send_ctx *sctx = ctx;
4919	struct fs_path *p;
4920	struct posix_acl_xattr_header dummy_acl;
4921
4922	/* Capabilities are emitted by finish_inode_if_needed */
4923	if (!strncmp(name, XATTR_NAME_CAPS, name_len))
4924		return 0;
4925
4926	p = fs_path_alloc();
4927	if (!p)
4928		return -ENOMEM;
4929
4930	/*
4931	 * This hack is needed because empty acls are stored as zero byte
4932	 * data in xattrs. Problem with that is, that receiving these zero byte
4933	 * acls will fail later. To fix this, we send a dummy acl list that
4934	 * only contains the version number and no entries.
4935	 */
4936	if (!strncmp(name, XATTR_NAME_POSIX_ACL_ACCESS, name_len) ||
4937	    !strncmp(name, XATTR_NAME_POSIX_ACL_DEFAULT, name_len)) {
4938		if (data_len == 0) {
4939			dummy_acl.a_version =
4940					cpu_to_le32(POSIX_ACL_XATTR_VERSION);
4941			data = (char *)&dummy_acl;
4942			data_len = sizeof(dummy_acl);
4943		}
4944	}
4945
4946	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
4947	if (ret < 0)
4948		goto out;
4949
4950	ret = send_set_xattr(sctx, p, name, name_len, data, data_len);
4951
4952out:
4953	fs_path_free(p);
4954	return ret;
4955}
4956
4957static int __process_deleted_xattr(int num, struct btrfs_key *di_key,
4958				   const char *name, int name_len,
4959				   const char *data, int data_len, void *ctx)
 
4960{
4961	int ret;
4962	struct send_ctx *sctx = ctx;
4963	struct fs_path *p;
4964
4965	p = fs_path_alloc();
4966	if (!p)
4967		return -ENOMEM;
4968
4969	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
4970	if (ret < 0)
4971		goto out;
4972
4973	ret = send_remove_xattr(sctx, p, name, name_len);
4974
4975out:
4976	fs_path_free(p);
4977	return ret;
4978}
4979
4980static int process_new_xattr(struct send_ctx *sctx)
4981{
4982	int ret = 0;
4983
4984	ret = iterate_dir_item(sctx->send_root, sctx->left_path,
4985			       __process_new_xattr, sctx);
4986
4987	return ret;
4988}
4989
4990static int process_deleted_xattr(struct send_ctx *sctx)
4991{
4992	return iterate_dir_item(sctx->parent_root, sctx->right_path,
4993				__process_deleted_xattr, sctx);
4994}
4995
4996struct find_xattr_ctx {
4997	const char *name;
4998	int name_len;
4999	int found_idx;
5000	char *found_data;
5001	int found_data_len;
5002};
5003
5004static int __find_xattr(int num, struct btrfs_key *di_key, const char *name,
5005			int name_len, const char *data, int data_len, void *vctx)
 
 
5006{
5007	struct find_xattr_ctx *ctx = vctx;
5008
5009	if (name_len == ctx->name_len &&
5010	    strncmp(name, ctx->name, name_len) == 0) {
5011		ctx->found_idx = num;
5012		ctx->found_data_len = data_len;
5013		ctx->found_data = kmemdup(data, data_len, GFP_KERNEL);
5014		if (!ctx->found_data)
5015			return -ENOMEM;
5016		return 1;
5017	}
5018	return 0;
5019}
5020
5021static int find_xattr(struct btrfs_root *root,
5022		      struct btrfs_path *path,
5023		      struct btrfs_key *key,
5024		      const char *name, int name_len,
5025		      char **data, int *data_len)
5026{
5027	int ret;
5028	struct find_xattr_ctx ctx;
5029
5030	ctx.name = name;
5031	ctx.name_len = name_len;
5032	ctx.found_idx = -1;
5033	ctx.found_data = NULL;
5034	ctx.found_data_len = 0;
5035
5036	ret = iterate_dir_item(root, path, __find_xattr, &ctx);
5037	if (ret < 0)
5038		return ret;
5039
5040	if (ctx.found_idx == -1)
5041		return -ENOENT;
5042	if (data) {
5043		*data = ctx.found_data;
5044		*data_len = ctx.found_data_len;
5045	} else {
5046		kfree(ctx.found_data);
5047	}
5048	return ctx.found_idx;
5049}
5050
5051
5052static int __process_changed_new_xattr(int num, struct btrfs_key *di_key,
5053				       const char *name, int name_len,
5054				       const char *data, int data_len,
5055				       void *ctx)
5056{
5057	int ret;
5058	struct send_ctx *sctx = ctx;
5059	char *found_data = NULL;
5060	int found_data_len  = 0;
5061
5062	ret = find_xattr(sctx->parent_root, sctx->right_path,
5063			 sctx->cmp_key, name, name_len, &found_data,
5064			 &found_data_len);
5065	if (ret == -ENOENT) {
5066		ret = __process_new_xattr(num, di_key, name, name_len, data,
5067					  data_len, ctx);
5068	} else if (ret >= 0) {
5069		if (data_len != found_data_len ||
5070		    memcmp(data, found_data, data_len)) {
5071			ret = __process_new_xattr(num, di_key, name, name_len,
5072						  data, data_len, ctx);
5073		} else {
5074			ret = 0;
5075		}
5076	}
5077
5078	kfree(found_data);
5079	return ret;
5080}
5081
5082static int __process_changed_deleted_xattr(int num, struct btrfs_key *di_key,
5083					   const char *name, int name_len,
5084					   const char *data, int data_len,
5085					   void *ctx)
5086{
5087	int ret;
5088	struct send_ctx *sctx = ctx;
5089
5090	ret = find_xattr(sctx->send_root, sctx->left_path, sctx->cmp_key,
5091			 name, name_len, NULL, NULL);
5092	if (ret == -ENOENT)
5093		ret = __process_deleted_xattr(num, di_key, name, name_len, data,
5094					      data_len, ctx);
5095	else if (ret >= 0)
5096		ret = 0;
5097
5098	return ret;
5099}
5100
5101static int process_changed_xattr(struct send_ctx *sctx)
5102{
5103	int ret = 0;
5104
5105	ret = iterate_dir_item(sctx->send_root, sctx->left_path,
5106			__process_changed_new_xattr, sctx);
5107	if (ret < 0)
5108		goto out;
5109	ret = iterate_dir_item(sctx->parent_root, sctx->right_path,
5110			__process_changed_deleted_xattr, sctx);
5111
5112out:
5113	return ret;
5114}
5115
5116static int process_all_new_xattrs(struct send_ctx *sctx)
5117{
5118	int ret = 0;
5119	int iter_ret = 0;
5120	struct btrfs_root *root;
5121	struct btrfs_path *path;
5122	struct btrfs_key key;
5123	struct btrfs_key found_key;
 
 
5124
5125	path = alloc_path_for_send();
5126	if (!path)
5127		return -ENOMEM;
5128
5129	root = sctx->send_root;
5130
5131	key.objectid = sctx->cmp_key->objectid;
5132	key.type = BTRFS_XATTR_ITEM_KEY;
5133	key.offset = 0;
5134	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5135		if (found_key.objectid != key.objectid ||
5136		    found_key.type != key.type) {
5137			ret = 0;
5138			break;
5139		}
5140
5141		ret = iterate_dir_item(root, path, __process_new_xattr, sctx);
5142		if (ret < 0)
5143			break;
5144	}
5145	/* Catch error found during iteration */
5146	if (iter_ret < 0)
5147		ret = iter_ret;
5148
5149	btrfs_free_path(path);
5150	return ret;
5151}
5152
5153static int send_verity(struct send_ctx *sctx, struct fs_path *path,
5154		       struct fsverity_descriptor *desc)
5155{
5156	int ret;
5157
5158	ret = begin_cmd(sctx, BTRFS_SEND_C_ENABLE_VERITY);
5159	if (ret < 0)
5160		goto out;
5161
5162	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
5163	TLV_PUT_U8(sctx, BTRFS_SEND_A_VERITY_ALGORITHM,
5164			le8_to_cpu(desc->hash_algorithm));
5165	TLV_PUT_U32(sctx, BTRFS_SEND_A_VERITY_BLOCK_SIZE,
5166			1U << le8_to_cpu(desc->log_blocksize));
5167	TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SALT_DATA, desc->salt,
5168			le8_to_cpu(desc->salt_size));
5169	TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SIG_DATA, desc->signature,
5170			le32_to_cpu(desc->sig_size));
5171
5172	ret = send_cmd(sctx);
5173
5174tlv_put_failure:
5175out:
5176	return ret;
5177}
5178
5179static int process_verity(struct send_ctx *sctx)
5180{
5181	int ret = 0;
5182	struct inode *inode;
5183	struct fs_path *p;
5184
5185	inode = btrfs_iget(sctx->cur_ino, sctx->send_root);
5186	if (IS_ERR(inode))
5187		return PTR_ERR(inode);
5188
5189	ret = btrfs_get_verity_descriptor(inode, NULL, 0);
5190	if (ret < 0)
5191		goto iput;
5192
5193	if (ret > FS_VERITY_MAX_DESCRIPTOR_SIZE) {
5194		ret = -EMSGSIZE;
5195		goto iput;
5196	}
5197	if (!sctx->verity_descriptor) {
5198		sctx->verity_descriptor = kvmalloc(FS_VERITY_MAX_DESCRIPTOR_SIZE,
5199						   GFP_KERNEL);
5200		if (!sctx->verity_descriptor) {
5201			ret = -ENOMEM;
5202			goto iput;
5203		}
5204	}
5205
5206	ret = btrfs_get_verity_descriptor(inode, sctx->verity_descriptor, ret);
5207	if (ret < 0)
5208		goto iput;
5209
5210	p = fs_path_alloc();
5211	if (!p) {
5212		ret = -ENOMEM;
5213		goto iput;
5214	}
5215	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
5216	if (ret < 0)
5217		goto free_path;
5218
5219	ret = send_verity(sctx, p, sctx->verity_descriptor);
5220	if (ret < 0)
5221		goto free_path;
5222
5223free_path:
5224	fs_path_free(p);
5225iput:
5226	iput(inode);
5227	return ret;
5228}
5229
5230static inline u64 max_send_read_size(const struct send_ctx *sctx)
5231{
5232	return sctx->send_max_size - SZ_16K;
5233}
5234
5235static int put_data_header(struct send_ctx *sctx, u32 len)
5236{
5237	if (WARN_ON_ONCE(sctx->put_data))
5238		return -EINVAL;
5239	sctx->put_data = true;
5240	if (sctx->proto >= 2) {
5241		/*
5242		 * Since v2, the data attribute header doesn't include a length,
5243		 * it is implicitly to the end of the command.
5244		 */
5245		if (sctx->send_max_size - sctx->send_size < sizeof(__le16) + len)
5246			return -EOVERFLOW;
5247		put_unaligned_le16(BTRFS_SEND_A_DATA, sctx->send_buf + sctx->send_size);
5248		sctx->send_size += sizeof(__le16);
5249	} else {
5250		struct btrfs_tlv_header *hdr;
5251
5252		if (sctx->send_max_size - sctx->send_size < sizeof(*hdr) + len)
5253			return -EOVERFLOW;
5254		hdr = (struct btrfs_tlv_header *)(sctx->send_buf + sctx->send_size);
5255		put_unaligned_le16(BTRFS_SEND_A_DATA, &hdr->tlv_type);
5256		put_unaligned_le16(len, &hdr->tlv_len);
5257		sctx->send_size += sizeof(*hdr);
5258	}
5259	return 0;
5260}
5261
5262static int put_file_data(struct send_ctx *sctx, u64 offset, u32 len)
5263{
5264	struct btrfs_root *root = sctx->send_root;
5265	struct btrfs_fs_info *fs_info = root->fs_info;
5266	struct folio *folio;
 
 
 
5267	pgoff_t index = offset >> PAGE_SHIFT;
5268	pgoff_t last_index;
5269	unsigned pg_offset = offset_in_page(offset);
5270	struct address_space *mapping = sctx->cur_inode->i_mapping;
5271	int ret;
5272
5273	ret = put_data_header(sctx, len);
5274	if (ret)
5275		return ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
5276
5277	last_index = (offset + len - 1) >> PAGE_SHIFT;
5278
 
 
 
 
5279	while (index <= last_index) {
5280		unsigned cur_len = min_t(unsigned, len,
5281					 PAGE_SIZE - pg_offset);
5282
5283again:
5284		folio = filemap_lock_folio(mapping, index);
5285		if (IS_ERR(folio)) {
5286			page_cache_sync_readahead(mapping,
5287						  &sctx->ra, NULL, index,
5288						  last_index + 1 - index);
5289
5290	                folio = filemap_grab_folio(mapping, index);
5291			if (IS_ERR(folio)) {
5292				ret = PTR_ERR(folio);
5293				break;
5294			}
5295		}
5296
5297		WARN_ON(folio_order(folio));
 
 
 
5298
5299		if (folio_test_readahead(folio))
5300			page_cache_async_readahead(mapping, &sctx->ra, NULL, folio,
5301						   last_index + 1 - index);
5302
5303		if (!folio_test_uptodate(folio)) {
5304			btrfs_read_folio(NULL, folio);
5305			folio_lock(folio);
5306			if (!folio_test_uptodate(folio)) {
5307				folio_unlock(folio);
5308				btrfs_err(fs_info,
5309			"send: IO error at offset %llu for inode %llu root %llu",
5310					folio_pos(folio), sctx->cur_ino,
5311					btrfs_root_id(sctx->send_root));
5312				folio_put(folio);
5313				ret = -EIO;
5314				break;
5315			}
5316			if (folio->mapping != mapping) {
5317				folio_unlock(folio);
5318				folio_put(folio);
5319				goto again;
5320			}
5321		}
5322
5323		memcpy_from_folio(sctx->send_buf + sctx->send_size, folio,
5324				  pg_offset, cur_len);
5325		folio_unlock(folio);
5326		folio_put(folio);
 
5327		index++;
5328		pg_offset = 0;
5329		len -= cur_len;
5330		sctx->send_size += cur_len;
5331	}
5332
 
5333	return ret;
5334}
5335
5336/*
5337 * Read some bytes from the current inode/file and send a write command to
5338 * user space.
5339 */
5340static int send_write(struct send_ctx *sctx, u64 offset, u32 len)
5341{
5342	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
5343	int ret = 0;
5344	struct fs_path *p;
 
5345
5346	p = fs_path_alloc();
5347	if (!p)
5348		return -ENOMEM;
5349
5350	btrfs_debug(fs_info, "send_write offset=%llu, len=%d", offset, len);
5351
 
 
 
 
 
 
 
5352	ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE);
5353	if (ret < 0)
5354		goto out;
5355
5356	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
5357	if (ret < 0)
5358		goto out;
5359
5360	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5361	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5362	ret = put_file_data(sctx, offset, len);
5363	if (ret < 0)
5364		goto out;
5365
5366	ret = send_cmd(sctx);
5367
5368tlv_put_failure:
5369out:
5370	fs_path_free(p);
5371	return ret;
 
 
5372}
5373
5374/*
5375 * Send a clone command to user space.
5376 */
5377static int send_clone(struct send_ctx *sctx,
5378		      u64 offset, u32 len,
5379		      struct clone_root *clone_root)
5380{
5381	int ret = 0;
5382	struct fs_path *p;
5383	u64 gen;
5384
5385	btrfs_debug(sctx->send_root->fs_info,
5386		    "send_clone offset=%llu, len=%d, clone_root=%llu, clone_inode=%llu, clone_offset=%llu",
5387		    offset, len, btrfs_root_id(clone_root->root),
5388		    clone_root->ino, clone_root->offset);
5389
5390	p = fs_path_alloc();
5391	if (!p)
5392		return -ENOMEM;
5393
5394	ret = begin_cmd(sctx, BTRFS_SEND_C_CLONE);
5395	if (ret < 0)
5396		goto out;
5397
5398	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
5399	if (ret < 0)
5400		goto out;
5401
5402	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5403	TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_LEN, len);
5404	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5405
5406	if (clone_root->root == sctx->send_root) {
5407		ret = get_inode_gen(sctx->send_root, clone_root->ino, &gen);
 
5408		if (ret < 0)
5409			goto out;
5410		ret = get_cur_path(sctx, clone_root->ino, gen, p);
5411	} else {
5412		ret = get_inode_path(clone_root->root, clone_root->ino, p);
5413	}
5414	if (ret < 0)
5415		goto out;
5416
5417	/*
5418	 * If the parent we're using has a received_uuid set then use that as
5419	 * our clone source as that is what we will look for when doing a
5420	 * receive.
5421	 *
5422	 * This covers the case that we create a snapshot off of a received
5423	 * subvolume and then use that as the parent and try to receive on a
5424	 * different host.
5425	 */
5426	if (!btrfs_is_empty_uuid(clone_root->root->root_item.received_uuid))
5427		TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
5428			     clone_root->root->root_item.received_uuid);
5429	else
5430		TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
5431			     clone_root->root->root_item.uuid);
5432	TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID,
5433		    btrfs_root_ctransid(&clone_root->root->root_item));
5434	TLV_PUT_PATH(sctx, BTRFS_SEND_A_CLONE_PATH, p);
5435	TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_OFFSET,
5436			clone_root->offset);
5437
5438	ret = send_cmd(sctx);
5439
5440tlv_put_failure:
5441out:
5442	fs_path_free(p);
5443	return ret;
5444}
5445
5446/*
5447 * Send an update extent command to user space.
5448 */
5449static int send_update_extent(struct send_ctx *sctx,
5450			      u64 offset, u32 len)
5451{
5452	int ret = 0;
5453	struct fs_path *p;
5454
5455	p = fs_path_alloc();
5456	if (!p)
5457		return -ENOMEM;
5458
5459	ret = begin_cmd(sctx, BTRFS_SEND_C_UPDATE_EXTENT);
5460	if (ret < 0)
5461		goto out;
5462
5463	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
5464	if (ret < 0)
5465		goto out;
5466
5467	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5468	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5469	TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len);
5470
5471	ret = send_cmd(sctx);
5472
5473tlv_put_failure:
5474out:
5475	fs_path_free(p);
5476	return ret;
5477}
5478
5479static int send_hole(struct send_ctx *sctx, u64 end)
5480{
5481	struct fs_path *p = NULL;
5482	u64 read_size = max_send_read_size(sctx);
5483	u64 offset = sctx->cur_inode_last_extent;
 
5484	int ret = 0;
5485
5486	/*
5487	 * A hole that starts at EOF or beyond it. Since we do not yet support
5488	 * fallocate (for extent preallocation and hole punching), sending a
5489	 * write of zeroes starting at EOF or beyond would later require issuing
5490	 * a truncate operation which would undo the write and achieve nothing.
5491	 */
5492	if (offset >= sctx->cur_inode_size)
5493		return 0;
5494
5495	/*
5496	 * Don't go beyond the inode's i_size due to prealloc extents that start
5497	 * after the i_size.
5498	 */
5499	end = min_t(u64, end, sctx->cur_inode_size);
5500
5501	if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
5502		return send_update_extent(sctx, offset, end - offset);
5503
5504	p = fs_path_alloc();
5505	if (!p)
5506		return -ENOMEM;
5507	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
5508	if (ret < 0)
5509		goto tlv_put_failure;
 
5510	while (offset < end) {
5511		u64 len = min(end - offset, read_size);
5512
5513		ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE);
5514		if (ret < 0)
5515			break;
5516		TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5517		TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5518		ret = put_data_header(sctx, len);
5519		if (ret < 0)
5520			break;
5521		memset(sctx->send_buf + sctx->send_size, 0, len);
5522		sctx->send_size += len;
5523		ret = send_cmd(sctx);
5524		if (ret < 0)
5525			break;
5526		offset += len;
5527	}
5528	sctx->cur_inode_next_write_offset = offset;
5529tlv_put_failure:
5530	fs_path_free(p);
5531	return ret;
5532}
5533
5534static int send_encoded_inline_extent(struct send_ctx *sctx,
5535				      struct btrfs_path *path, u64 offset,
5536				      u64 len)
5537{
5538	struct btrfs_root *root = sctx->send_root;
5539	struct btrfs_fs_info *fs_info = root->fs_info;
5540	struct inode *inode;
5541	struct fs_path *fspath;
5542	struct extent_buffer *leaf = path->nodes[0];
5543	struct btrfs_key key;
5544	struct btrfs_file_extent_item *ei;
5545	u64 ram_bytes;
5546	size_t inline_size;
5547	int ret;
5548
5549	inode = btrfs_iget(sctx->cur_ino, root);
5550	if (IS_ERR(inode))
5551		return PTR_ERR(inode);
5552
5553	fspath = fs_path_alloc();
5554	if (!fspath) {
5555		ret = -ENOMEM;
5556		goto out;
5557	}
5558
5559	ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE);
5560	if (ret < 0)
5561		goto out;
5562
5563	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath);
5564	if (ret < 0)
5565		goto out;
5566
5567	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
5568	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
5569	ram_bytes = btrfs_file_extent_ram_bytes(leaf, ei);
5570	inline_size = btrfs_file_extent_inline_item_len(leaf, path->slots[0]);
5571
5572	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath);
5573	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5574	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN,
5575		    min(key.offset + ram_bytes - offset, len));
5576	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN, ram_bytes);
5577	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET, offset - key.offset);
5578	ret = btrfs_encoded_io_compression_from_extent(fs_info,
5579				btrfs_file_extent_compression(leaf, ei));
5580	if (ret < 0)
5581		goto out;
5582	TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret);
5583
5584	ret = put_data_header(sctx, inline_size);
5585	if (ret < 0)
5586		goto out;
5587	read_extent_buffer(leaf, sctx->send_buf + sctx->send_size,
5588			   btrfs_file_extent_inline_start(ei), inline_size);
5589	sctx->send_size += inline_size;
5590
5591	ret = send_cmd(sctx);
5592
5593tlv_put_failure:
5594out:
5595	fs_path_free(fspath);
5596	iput(inode);
5597	return ret;
5598}
5599
5600static int send_encoded_extent(struct send_ctx *sctx, struct btrfs_path *path,
5601			       u64 offset, u64 len)
5602{
5603	struct btrfs_root *root = sctx->send_root;
5604	struct btrfs_fs_info *fs_info = root->fs_info;
5605	struct inode *inode;
5606	struct fs_path *fspath;
5607	struct extent_buffer *leaf = path->nodes[0];
5608	struct btrfs_key key;
5609	struct btrfs_file_extent_item *ei;
5610	u64 disk_bytenr, disk_num_bytes;
5611	u32 data_offset;
5612	struct btrfs_cmd_header *hdr;
5613	u32 crc;
5614	int ret;
5615
5616	inode = btrfs_iget(sctx->cur_ino, root);
5617	if (IS_ERR(inode))
5618		return PTR_ERR(inode);
5619
5620	fspath = fs_path_alloc();
5621	if (!fspath) {
5622		ret = -ENOMEM;
5623		goto out;
5624	}
5625
5626	ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE);
5627	if (ret < 0)
5628		goto out;
5629
5630	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath);
5631	if (ret < 0)
5632		goto out;
5633
5634	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
5635	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
5636	disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
5637	disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, ei);
5638
5639	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath);
5640	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5641	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN,
5642		    min(key.offset + btrfs_file_extent_num_bytes(leaf, ei) - offset,
5643			len));
5644	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN,
5645		    btrfs_file_extent_ram_bytes(leaf, ei));
5646	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET,
5647		    offset - key.offset + btrfs_file_extent_offset(leaf, ei));
5648	ret = btrfs_encoded_io_compression_from_extent(fs_info,
5649				btrfs_file_extent_compression(leaf, ei));
5650	if (ret < 0)
5651		goto out;
5652	TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret);
5653	TLV_PUT_U32(sctx, BTRFS_SEND_A_ENCRYPTION, 0);
5654
5655	ret = put_data_header(sctx, disk_num_bytes);
5656	if (ret < 0)
5657		goto out;
5658
5659	/*
5660	 * We want to do I/O directly into the send buffer, so get the next page
5661	 * boundary in the send buffer. This means that there may be a gap
5662	 * between the beginning of the command and the file data.
5663	 */
5664	data_offset = PAGE_ALIGN(sctx->send_size);
5665	if (data_offset > sctx->send_max_size ||
5666	    sctx->send_max_size - data_offset < disk_num_bytes) {
5667		ret = -EOVERFLOW;
5668		goto out;
5669	}
5670
5671	/*
5672	 * Note that send_buf is a mapping of send_buf_pages, so this is really
5673	 * reading into send_buf.
5674	 */
5675	ret = btrfs_encoded_read_regular_fill_pages(BTRFS_I(inode),
5676						    disk_bytenr, disk_num_bytes,
5677						    sctx->send_buf_pages +
5678						    (data_offset >> PAGE_SHIFT),
5679						    NULL);
5680	if (ret)
5681		goto out;
5682
5683	hdr = (struct btrfs_cmd_header *)sctx->send_buf;
5684	hdr->len = cpu_to_le32(sctx->send_size + disk_num_bytes - sizeof(*hdr));
5685	hdr->crc = 0;
5686	crc = crc32c(0, sctx->send_buf, sctx->send_size);
5687	crc = crc32c(crc, sctx->send_buf + data_offset, disk_num_bytes);
5688	hdr->crc = cpu_to_le32(crc);
5689
5690	ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,
5691			&sctx->send_off);
5692	if (!ret) {
5693		ret = write_buf(sctx->send_filp, sctx->send_buf + data_offset,
5694				disk_num_bytes, &sctx->send_off);
5695	}
5696	sctx->send_size = 0;
5697	sctx->put_data = false;
5698
5699tlv_put_failure:
5700out:
5701	fs_path_free(fspath);
5702	iput(inode);
5703	return ret;
5704}
5705
5706static int send_extent_data(struct send_ctx *sctx, struct btrfs_path *path,
5707			    const u64 offset, const u64 len)
5708{
5709	const u64 end = offset + len;
5710	struct extent_buffer *leaf = path->nodes[0];
5711	struct btrfs_file_extent_item *ei;
5712	u64 read_size = max_send_read_size(sctx);
5713	u64 sent = 0;
5714
5715	if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
5716		return send_update_extent(sctx, offset, len);
5717
5718	ei = btrfs_item_ptr(leaf, path->slots[0],
5719			    struct btrfs_file_extent_item);
5720	if ((sctx->flags & BTRFS_SEND_FLAG_COMPRESSED) &&
5721	    btrfs_file_extent_compression(leaf, ei) != BTRFS_COMPRESS_NONE) {
5722		bool is_inline = (btrfs_file_extent_type(leaf, ei) ==
5723				  BTRFS_FILE_EXTENT_INLINE);
5724
5725		/*
5726		 * Send the compressed extent unless the compressed data is
5727		 * larger than the decompressed data. This can happen if we're
5728		 * not sending the entire extent, either because it has been
5729		 * partially overwritten/truncated or because this is a part of
5730		 * the extent that we couldn't clone in clone_range().
5731		 */
5732		if (is_inline &&
5733		    btrfs_file_extent_inline_item_len(leaf,
5734						      path->slots[0]) <= len) {
5735			return send_encoded_inline_extent(sctx, path, offset,
5736							  len);
5737		} else if (!is_inline &&
5738			   btrfs_file_extent_disk_num_bytes(leaf, ei) <= len) {
5739			return send_encoded_extent(sctx, path, offset, len);
5740		}
5741	}
5742
5743	if (sctx->cur_inode == NULL) {
5744		struct btrfs_root *root = sctx->send_root;
5745
5746		sctx->cur_inode = btrfs_iget(sctx->cur_ino, root);
5747		if (IS_ERR(sctx->cur_inode)) {
5748			int err = PTR_ERR(sctx->cur_inode);
5749
5750			sctx->cur_inode = NULL;
5751			return err;
5752		}
5753		memset(&sctx->ra, 0, sizeof(struct file_ra_state));
5754		file_ra_state_init(&sctx->ra, sctx->cur_inode->i_mapping);
5755
5756		/*
5757		 * It's very likely there are no pages from this inode in the page
5758		 * cache, so after reading extents and sending their data, we clean
5759		 * the page cache to avoid trashing the page cache (adding pressure
5760		 * to the page cache and forcing eviction of other data more useful
5761		 * for applications).
5762		 *
5763		 * We decide if we should clean the page cache simply by checking
5764		 * if the inode's mapping nrpages is 0 when we first open it, and
5765		 * not by using something like filemap_range_has_page() before
5766		 * reading an extent because when we ask the readahead code to
5767		 * read a given file range, it may (and almost always does) read
5768		 * pages from beyond that range (see the documentation for
5769		 * page_cache_sync_readahead()), so it would not be reliable,
5770		 * because after reading the first extent future calls to
5771		 * filemap_range_has_page() would return true because the readahead
5772		 * on the previous extent resulted in reading pages of the current
5773		 * extent as well.
5774		 */
5775		sctx->clean_page_cache = (sctx->cur_inode->i_mapping->nrpages == 0);
5776		sctx->page_cache_clear_start = round_down(offset, PAGE_SIZE);
5777	}
5778
5779	while (sent < len) {
5780		u64 size = min(len - sent, read_size);
5781		int ret;
5782
 
 
5783		ret = send_write(sctx, offset + sent, size);
5784		if (ret < 0)
5785			return ret;
5786		sent += size;
5787	}
5788
5789	if (sctx->clean_page_cache && PAGE_ALIGNED(end)) {
5790		/*
5791		 * Always operate only on ranges that are a multiple of the page
5792		 * size. This is not only to prevent zeroing parts of a page in
5793		 * the case of subpage sector size, but also to guarantee we evict
5794		 * pages, as passing a range that is smaller than page size does
5795		 * not evict the respective page (only zeroes part of its content).
5796		 *
5797		 * Always start from the end offset of the last range cleared.
5798		 * This is because the readahead code may (and very often does)
5799		 * reads pages beyond the range we request for readahead. So if
5800		 * we have an extent layout like this:
5801		 *
5802		 *            [ extent A ] [ extent B ] [ extent C ]
5803		 *
5804		 * When we ask page_cache_sync_readahead() to read extent A, it
5805		 * may also trigger reads for pages of extent B. If we are doing
5806		 * an incremental send and extent B has not changed between the
5807		 * parent and send snapshots, some or all of its pages may end
5808		 * up being read and placed in the page cache. So when truncating
5809		 * the page cache we always start from the end offset of the
5810		 * previously processed extent up to the end of the current
5811		 * extent.
5812		 */
5813		truncate_inode_pages_range(&sctx->cur_inode->i_data,
5814					   sctx->page_cache_clear_start,
5815					   end - 1);
5816		sctx->page_cache_clear_start = end;
5817	}
5818
5819	return 0;
5820}
5821
5822/*
5823 * Search for a capability xattr related to sctx->cur_ino. If the capability is
5824 * found, call send_set_xattr function to emit it.
5825 *
5826 * Return 0 if there isn't a capability, or when the capability was emitted
5827 * successfully, or < 0 if an error occurred.
5828 */
5829static int send_capabilities(struct send_ctx *sctx)
5830{
5831	struct fs_path *fspath = NULL;
5832	struct btrfs_path *path;
5833	struct btrfs_dir_item *di;
5834	struct extent_buffer *leaf;
5835	unsigned long data_ptr;
5836	char *buf = NULL;
5837	int buf_len;
5838	int ret = 0;
5839
5840	path = alloc_path_for_send();
5841	if (!path)
5842		return -ENOMEM;
5843
5844	di = btrfs_lookup_xattr(NULL, sctx->send_root, path, sctx->cur_ino,
5845				XATTR_NAME_CAPS, strlen(XATTR_NAME_CAPS), 0);
5846	if (!di) {
5847		/* There is no xattr for this inode */
5848		goto out;
5849	} else if (IS_ERR(di)) {
5850		ret = PTR_ERR(di);
5851		goto out;
5852	}
5853
5854	leaf = path->nodes[0];
5855	buf_len = btrfs_dir_data_len(leaf, di);
5856
5857	fspath = fs_path_alloc();
5858	buf = kmalloc(buf_len, GFP_KERNEL);
5859	if (!fspath || !buf) {
5860		ret = -ENOMEM;
5861		goto out;
5862	}
5863
5864	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath);
5865	if (ret < 0)
5866		goto out;
5867
5868	data_ptr = (unsigned long)(di + 1) + btrfs_dir_name_len(leaf, di);
5869	read_extent_buffer(leaf, buf, data_ptr, buf_len);
5870
5871	ret = send_set_xattr(sctx, fspath, XATTR_NAME_CAPS,
5872			strlen(XATTR_NAME_CAPS), buf, buf_len);
5873out:
5874	kfree(buf);
5875	fs_path_free(fspath);
5876	btrfs_free_path(path);
5877	return ret;
5878}
5879
5880static int clone_range(struct send_ctx *sctx, struct btrfs_path *dst_path,
5881		       struct clone_root *clone_root, const u64 disk_byte,
5882		       u64 data_offset, u64 offset, u64 len)
5883{
5884	struct btrfs_path *path;
5885	struct btrfs_key key;
5886	int ret;
5887	struct btrfs_inode_info info;
5888	u64 clone_src_i_size = 0;
5889
5890	/*
5891	 * Prevent cloning from a zero offset with a length matching the sector
5892	 * size because in some scenarios this will make the receiver fail.
5893	 *
5894	 * For example, if in the source filesystem the extent at offset 0
5895	 * has a length of sectorsize and it was written using direct IO, then
5896	 * it can never be an inline extent (even if compression is enabled).
5897	 * Then this extent can be cloned in the original filesystem to a non
5898	 * zero file offset, but it may not be possible to clone in the
5899	 * destination filesystem because it can be inlined due to compression
5900	 * on the destination filesystem (as the receiver's write operations are
5901	 * always done using buffered IO). The same happens when the original
5902	 * filesystem does not have compression enabled but the destination
5903	 * filesystem has.
5904	 */
5905	if (clone_root->offset == 0 &&
5906	    len == sctx->send_root->fs_info->sectorsize)
5907		return send_extent_data(sctx, dst_path, offset, len);
5908
5909	path = alloc_path_for_send();
5910	if (!path)
5911		return -ENOMEM;
5912
5913	/*
5914	 * There are inodes that have extents that lie behind its i_size. Don't
5915	 * accept clones from these extents.
5916	 */
5917	ret = get_inode_info(clone_root->root, clone_root->ino, &info);
 
5918	btrfs_release_path(path);
5919	if (ret < 0)
5920		goto out;
5921	clone_src_i_size = info.size;
5922
5923	/*
5924	 * We can't send a clone operation for the entire range if we find
5925	 * extent items in the respective range in the source file that
5926	 * refer to different extents or if we find holes.
5927	 * So check for that and do a mix of clone and regular write/copy
5928	 * operations if needed.
5929	 *
5930	 * Example:
5931	 *
5932	 * mkfs.btrfs -f /dev/sda
5933	 * mount /dev/sda /mnt
5934	 * xfs_io -f -c "pwrite -S 0xaa 0K 100K" /mnt/foo
5935	 * cp --reflink=always /mnt/foo /mnt/bar
5936	 * xfs_io -c "pwrite -S 0xbb 50K 50K" /mnt/foo
5937	 * btrfs subvolume snapshot -r /mnt /mnt/snap
5938	 *
5939	 * If when we send the snapshot and we are processing file bar (which
5940	 * has a higher inode number than foo) we blindly send a clone operation
5941	 * for the [0, 100K[ range from foo to bar, the receiver ends up getting
5942	 * a file bar that matches the content of file foo - iow, doesn't match
5943	 * the content from bar in the original filesystem.
5944	 */
5945	key.objectid = clone_root->ino;
5946	key.type = BTRFS_EXTENT_DATA_KEY;
5947	key.offset = clone_root->offset;
5948	ret = btrfs_search_slot(NULL, clone_root->root, &key, path, 0, 0);
5949	if (ret < 0)
5950		goto out;
5951	if (ret > 0 && path->slots[0] > 0) {
5952		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1);
5953		if (key.objectid == clone_root->ino &&
5954		    key.type == BTRFS_EXTENT_DATA_KEY)
5955			path->slots[0]--;
5956	}
5957
5958	while (true) {
5959		struct extent_buffer *leaf = path->nodes[0];
5960		int slot = path->slots[0];
5961		struct btrfs_file_extent_item *ei;
5962		u8 type;
5963		u64 ext_len;
5964		u64 clone_len;
5965		u64 clone_data_offset;
5966		bool crossed_src_i_size = false;
5967
5968		if (slot >= btrfs_header_nritems(leaf)) {
5969			ret = btrfs_next_leaf(clone_root->root, path);
5970			if (ret < 0)
5971				goto out;
5972			else if (ret > 0)
5973				break;
5974			continue;
5975		}
5976
5977		btrfs_item_key_to_cpu(leaf, &key, slot);
5978
5979		/*
5980		 * We might have an implicit trailing hole (NO_HOLES feature
5981		 * enabled). We deal with it after leaving this loop.
5982		 */
5983		if (key.objectid != clone_root->ino ||
5984		    key.type != BTRFS_EXTENT_DATA_KEY)
5985			break;
5986
5987		ei = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
5988		type = btrfs_file_extent_type(leaf, ei);
5989		if (type == BTRFS_FILE_EXTENT_INLINE) {
5990			ext_len = btrfs_file_extent_ram_bytes(leaf, ei);
5991			ext_len = PAGE_ALIGN(ext_len);
5992		} else {
5993			ext_len = btrfs_file_extent_num_bytes(leaf, ei);
5994		}
5995
5996		if (key.offset + ext_len <= clone_root->offset)
5997			goto next;
5998
5999		if (key.offset > clone_root->offset) {
6000			/* Implicit hole, NO_HOLES feature enabled. */
6001			u64 hole_len = key.offset - clone_root->offset;
6002
6003			if (hole_len > len)
6004				hole_len = len;
6005			ret = send_extent_data(sctx, dst_path, offset,
6006					       hole_len);
6007			if (ret < 0)
6008				goto out;
6009
6010			len -= hole_len;
6011			if (len == 0)
6012				break;
6013			offset += hole_len;
6014			clone_root->offset += hole_len;
6015			data_offset += hole_len;
6016		}
6017
6018		if (key.offset >= clone_root->offset + len)
6019			break;
6020
6021		if (key.offset >= clone_src_i_size)
6022			break;
6023
6024		if (key.offset + ext_len > clone_src_i_size) {
6025			ext_len = clone_src_i_size - key.offset;
6026			crossed_src_i_size = true;
6027		}
6028
6029		clone_data_offset = btrfs_file_extent_offset(leaf, ei);
6030		if (btrfs_file_extent_disk_bytenr(leaf, ei) == disk_byte) {
6031			clone_root->offset = key.offset;
6032			if (clone_data_offset < data_offset &&
6033				clone_data_offset + ext_len > data_offset) {
6034				u64 extent_offset;
6035
6036				extent_offset = data_offset - clone_data_offset;
6037				ext_len -= extent_offset;
6038				clone_data_offset += extent_offset;
6039				clone_root->offset += extent_offset;
6040			}
6041		}
6042
6043		clone_len = min_t(u64, ext_len, len);
6044
6045		if (btrfs_file_extent_disk_bytenr(leaf, ei) == disk_byte &&
6046		    clone_data_offset == data_offset) {
6047			const u64 src_end = clone_root->offset + clone_len;
6048			const u64 sectorsize = SZ_64K;
6049
6050			/*
6051			 * We can't clone the last block, when its size is not
6052			 * sector size aligned, into the middle of a file. If we
6053			 * do so, the receiver will get a failure (-EINVAL) when
6054			 * trying to clone or will silently corrupt the data in
6055			 * the destination file if it's on a kernel without the
6056			 * fix introduced by commit ac765f83f1397646
6057			 * ("Btrfs: fix data corruption due to cloning of eof
6058			 * block).
6059			 *
6060			 * So issue a clone of the aligned down range plus a
6061			 * regular write for the eof block, if we hit that case.
6062			 *
6063			 * Also, we use the maximum possible sector size, 64K,
6064			 * because we don't know what's the sector size of the
6065			 * filesystem that receives the stream, so we have to
6066			 * assume the largest possible sector size.
6067			 */
6068			if (src_end == clone_src_i_size &&
6069			    !IS_ALIGNED(src_end, sectorsize) &&
6070			    offset + clone_len < sctx->cur_inode_size) {
6071				u64 slen;
6072
6073				slen = ALIGN_DOWN(src_end - clone_root->offset,
6074						  sectorsize);
6075				if (slen > 0) {
6076					ret = send_clone(sctx, offset, slen,
6077							 clone_root);
6078					if (ret < 0)
6079						goto out;
6080				}
6081				ret = send_extent_data(sctx, dst_path,
6082						       offset + slen,
6083						       clone_len - slen);
6084			} else {
6085				ret = send_clone(sctx, offset, clone_len,
6086						 clone_root);
6087			}
6088		} else if (crossed_src_i_size && clone_len < len) {
6089			/*
6090			 * If we are at i_size of the clone source inode and we
6091			 * can not clone from it, terminate the loop. This is
6092			 * to avoid sending two write operations, one with a
6093			 * length matching clone_len and the final one after
6094			 * this loop with a length of len - clone_len.
6095			 *
6096			 * When using encoded writes (BTRFS_SEND_FLAG_COMPRESSED
6097			 * was passed to the send ioctl), this helps avoid
6098			 * sending an encoded write for an offset that is not
6099			 * sector size aligned, in case the i_size of the source
6100			 * inode is not sector size aligned. That will make the
6101			 * receiver fallback to decompression of the data and
6102			 * writing it using regular buffered IO, therefore while
6103			 * not incorrect, it's not optimal due decompression and
6104			 * possible re-compression at the receiver.
6105			 */
6106			break;
6107		} else {
6108			ret = send_extent_data(sctx, dst_path, offset,
6109					       clone_len);
6110		}
6111
6112		if (ret < 0)
6113			goto out;
6114
6115		len -= clone_len;
6116		if (len == 0)
6117			break;
6118		offset += clone_len;
6119		clone_root->offset += clone_len;
6120
6121		/*
6122		 * If we are cloning from the file we are currently processing,
6123		 * and using the send root as the clone root, we must stop once
6124		 * the current clone offset reaches the current eof of the file
6125		 * at the receiver, otherwise we would issue an invalid clone
6126		 * operation (source range going beyond eof) and cause the
6127		 * receiver to fail. So if we reach the current eof, bail out
6128		 * and fallback to a regular write.
6129		 */
6130		if (clone_root->root == sctx->send_root &&
6131		    clone_root->ino == sctx->cur_ino &&
6132		    clone_root->offset >= sctx->cur_inode_next_write_offset)
6133			break;
6134
6135		data_offset += clone_len;
6136next:
6137		path->slots[0]++;
6138	}
6139
6140	if (len > 0)
6141		ret = send_extent_data(sctx, dst_path, offset, len);
6142	else
6143		ret = 0;
6144out:
6145	btrfs_free_path(path);
6146	return ret;
6147}
6148
6149static int send_write_or_clone(struct send_ctx *sctx,
6150			       struct btrfs_path *path,
6151			       struct btrfs_key *key,
6152			       struct clone_root *clone_root)
6153{
6154	int ret = 0;
6155	u64 offset = key->offset;
6156	u64 end;
6157	u64 bs = sctx->send_root->fs_info->sectorsize;
6158	struct btrfs_file_extent_item *ei;
6159	u64 disk_byte;
6160	u64 data_offset;
6161	u64 num_bytes;
6162	struct btrfs_inode_info info = { 0 };
6163
6164	end = min_t(u64, btrfs_file_extent_end(path), sctx->cur_inode_size);
6165	if (offset >= end)
6166		return 0;
6167
6168	num_bytes = end - offset;
6169
6170	if (!clone_root)
6171		goto write_data;
6172
6173	if (IS_ALIGNED(end, bs))
6174		goto clone_data;
6175
6176	/*
6177	 * If the extent end is not aligned, we can clone if the extent ends at
6178	 * the i_size of the inode and the clone range ends at the i_size of the
6179	 * source inode, otherwise the clone operation fails with -EINVAL.
6180	 */
6181	if (end != sctx->cur_inode_size)
6182		goto write_data;
6183
6184	ret = get_inode_info(clone_root->root, clone_root->ino, &info);
6185	if (ret < 0)
6186		return ret;
6187
6188	if (clone_root->offset + num_bytes == info.size) {
 
 
 
 
6189		/*
6190		 * The final size of our file matches the end offset, but it may
6191		 * be that its current size is larger, so we have to truncate it
6192		 * to any value between the start offset of the range and the
6193		 * final i_size, otherwise the clone operation is invalid
6194		 * because it's unaligned and it ends before the current EOF.
6195		 * We do this truncate to the final i_size when we finish
6196		 * processing the inode, but it's too late by then. And here we
6197		 * truncate to the start offset of the range because it's always
6198		 * sector size aligned while if it were the final i_size it
6199		 * would result in dirtying part of a page, filling part of a
6200		 * page with zeroes and then having the clone operation at the
6201		 * receiver trigger IO and wait for it due to the dirty page.
6202		 */
6203		if (sctx->parent_root != NULL) {
6204			ret = send_truncate(sctx, sctx->cur_ino,
6205					    sctx->cur_inode_gen, offset);
6206			if (ret < 0)
6207				return ret;
6208		}
6209		goto clone_data;
6210	}
6211
6212write_data:
6213	ret = send_extent_data(sctx, path, offset, num_bytes);
6214	sctx->cur_inode_next_write_offset = end;
6215	return ret;
 
 
 
 
 
 
6216
6217clone_data:
6218	ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
6219			    struct btrfs_file_extent_item);
6220	disk_byte = btrfs_file_extent_disk_bytenr(path->nodes[0], ei);
6221	data_offset = btrfs_file_extent_offset(path->nodes[0], ei);
6222	ret = clone_range(sctx, path, clone_root, disk_byte, data_offset, offset,
6223			  num_bytes);
6224	sctx->cur_inode_next_write_offset = end;
 
 
 
 
 
6225	return ret;
6226}
6227
6228static int is_extent_unchanged(struct send_ctx *sctx,
6229			       struct btrfs_path *left_path,
6230			       struct btrfs_key *ekey)
6231{
6232	int ret = 0;
6233	struct btrfs_key key;
6234	struct btrfs_path *path = NULL;
6235	struct extent_buffer *eb;
6236	int slot;
6237	struct btrfs_key found_key;
6238	struct btrfs_file_extent_item *ei;
6239	u64 left_disknr;
6240	u64 right_disknr;
6241	u64 left_offset;
6242	u64 right_offset;
6243	u64 left_offset_fixed;
6244	u64 left_len;
6245	u64 right_len;
6246	u64 left_gen;
6247	u64 right_gen;
6248	u8 left_type;
6249	u8 right_type;
6250
6251	path = alloc_path_for_send();
6252	if (!path)
6253		return -ENOMEM;
6254
6255	eb = left_path->nodes[0];
6256	slot = left_path->slots[0];
6257	ei = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item);
6258	left_type = btrfs_file_extent_type(eb, ei);
6259
6260	if (left_type != BTRFS_FILE_EXTENT_REG) {
6261		ret = 0;
6262		goto out;
6263	}
6264	left_disknr = btrfs_file_extent_disk_bytenr(eb, ei);
6265	left_len = btrfs_file_extent_num_bytes(eb, ei);
6266	left_offset = btrfs_file_extent_offset(eb, ei);
6267	left_gen = btrfs_file_extent_generation(eb, ei);
6268
6269	/*
6270	 * Following comments will refer to these graphics. L is the left
6271	 * extents which we are checking at the moment. 1-8 are the right
6272	 * extents that we iterate.
6273	 *
6274	 *       |-----L-----|
6275	 * |-1-|-2a-|-3-|-4-|-5-|-6-|
6276	 *
6277	 *       |-----L-----|
6278	 * |--1--|-2b-|...(same as above)
6279	 *
6280	 * Alternative situation. Happens on files where extents got split.
6281	 *       |-----L-----|
6282	 * |-----------7-----------|-6-|
6283	 *
6284	 * Alternative situation. Happens on files which got larger.
6285	 *       |-----L-----|
6286	 * |-8-|
6287	 * Nothing follows after 8.
6288	 */
6289
6290	key.objectid = ekey->objectid;
6291	key.type = BTRFS_EXTENT_DATA_KEY;
6292	key.offset = ekey->offset;
6293	ret = btrfs_search_slot_for_read(sctx->parent_root, &key, path, 0, 0);
6294	if (ret < 0)
6295		goto out;
6296	if (ret) {
6297		ret = 0;
6298		goto out;
6299	}
6300
6301	/*
6302	 * Handle special case where the right side has no extents at all.
6303	 */
6304	eb = path->nodes[0];
6305	slot = path->slots[0];
6306	btrfs_item_key_to_cpu(eb, &found_key, slot);
6307	if (found_key.objectid != key.objectid ||
6308	    found_key.type != key.type) {
6309		/* If we're a hole then just pretend nothing changed */
6310		ret = (left_disknr) ? 0 : 1;
6311		goto out;
6312	}
6313
6314	/*
6315	 * We're now on 2a, 2b or 7.
6316	 */
6317	key = found_key;
6318	while (key.offset < ekey->offset + left_len) {
6319		ei = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item);
6320		right_type = btrfs_file_extent_type(eb, ei);
6321		if (right_type != BTRFS_FILE_EXTENT_REG &&
6322		    right_type != BTRFS_FILE_EXTENT_INLINE) {
6323			ret = 0;
6324			goto out;
6325		}
6326
6327		if (right_type == BTRFS_FILE_EXTENT_INLINE) {
6328			right_len = btrfs_file_extent_ram_bytes(eb, ei);
6329			right_len = PAGE_ALIGN(right_len);
6330		} else {
6331			right_len = btrfs_file_extent_num_bytes(eb, ei);
6332		}
6333
6334		/*
6335		 * Are we at extent 8? If yes, we know the extent is changed.
6336		 * This may only happen on the first iteration.
6337		 */
6338		if (found_key.offset + right_len <= ekey->offset) {
6339			/* If we're a hole just pretend nothing changed */
6340			ret = (left_disknr) ? 0 : 1;
6341			goto out;
6342		}
6343
6344		/*
6345		 * We just wanted to see if when we have an inline extent, what
6346		 * follows it is a regular extent (wanted to check the above
6347		 * condition for inline extents too). This should normally not
6348		 * happen but it's possible for example when we have an inline
6349		 * compressed extent representing data with a size matching
6350		 * the page size (currently the same as sector size).
6351		 */
6352		if (right_type == BTRFS_FILE_EXTENT_INLINE) {
6353			ret = 0;
6354			goto out;
6355		}
6356
6357		right_disknr = btrfs_file_extent_disk_bytenr(eb, ei);
6358		right_offset = btrfs_file_extent_offset(eb, ei);
6359		right_gen = btrfs_file_extent_generation(eb, ei);
6360
6361		left_offset_fixed = left_offset;
6362		if (key.offset < ekey->offset) {
6363			/* Fix the right offset for 2a and 7. */
6364			right_offset += ekey->offset - key.offset;
6365		} else {
6366			/* Fix the left offset for all behind 2a and 2b */
6367			left_offset_fixed += key.offset - ekey->offset;
6368		}
6369
6370		/*
6371		 * Check if we have the same extent.
6372		 */
6373		if (left_disknr != right_disknr ||
6374		    left_offset_fixed != right_offset ||
6375		    left_gen != right_gen) {
6376			ret = 0;
6377			goto out;
6378		}
6379
6380		/*
6381		 * Go to the next extent.
6382		 */
6383		ret = btrfs_next_item(sctx->parent_root, path);
6384		if (ret < 0)
6385			goto out;
6386		if (!ret) {
6387			eb = path->nodes[0];
6388			slot = path->slots[0];
6389			btrfs_item_key_to_cpu(eb, &found_key, slot);
6390		}
6391		if (ret || found_key.objectid != key.objectid ||
6392		    found_key.type != key.type) {
6393			key.offset += right_len;
6394			break;
6395		}
6396		if (found_key.offset != key.offset + right_len) {
6397			ret = 0;
6398			goto out;
6399		}
6400		key = found_key;
6401	}
6402
6403	/*
6404	 * We're now behind the left extent (treat as unchanged) or at the end
6405	 * of the right side (treat as changed).
6406	 */
6407	if (key.offset >= ekey->offset + left_len)
6408		ret = 1;
6409	else
6410		ret = 0;
6411
6412
6413out:
6414	btrfs_free_path(path);
6415	return ret;
6416}
6417
6418static int get_last_extent(struct send_ctx *sctx, u64 offset)
6419{
6420	struct btrfs_path *path;
6421	struct btrfs_root *root = sctx->send_root;
 
6422	struct btrfs_key key;
 
 
6423	int ret;
6424
6425	path = alloc_path_for_send();
6426	if (!path)
6427		return -ENOMEM;
6428
6429	sctx->cur_inode_last_extent = 0;
6430
6431	key.objectid = sctx->cur_ino;
6432	key.type = BTRFS_EXTENT_DATA_KEY;
6433	key.offset = offset;
6434	ret = btrfs_search_slot_for_read(root, &key, path, 0, 1);
6435	if (ret < 0)
6436		goto out;
6437	ret = 0;
6438	btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
6439	if (key.objectid != sctx->cur_ino || key.type != BTRFS_EXTENT_DATA_KEY)
6440		goto out;
6441
6442	sctx->cur_inode_last_extent = btrfs_file_extent_end(path);
 
 
 
 
 
 
 
 
 
 
 
6443out:
6444	btrfs_free_path(path);
6445	return ret;
6446}
6447
6448static int range_is_hole_in_parent(struct send_ctx *sctx,
6449				   const u64 start,
6450				   const u64 end)
6451{
6452	struct btrfs_path *path;
6453	struct btrfs_key key;
6454	struct btrfs_root *root = sctx->parent_root;
6455	u64 search_start = start;
6456	int ret;
6457
6458	path = alloc_path_for_send();
6459	if (!path)
6460		return -ENOMEM;
6461
6462	key.objectid = sctx->cur_ino;
6463	key.type = BTRFS_EXTENT_DATA_KEY;
6464	key.offset = search_start;
6465	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6466	if (ret < 0)
6467		goto out;
6468	if (ret > 0 && path->slots[0] > 0)
6469		path->slots[0]--;
6470
6471	while (search_start < end) {
6472		struct extent_buffer *leaf = path->nodes[0];
6473		int slot = path->slots[0];
6474		struct btrfs_file_extent_item *fi;
6475		u64 extent_end;
6476
6477		if (slot >= btrfs_header_nritems(leaf)) {
6478			ret = btrfs_next_leaf(root, path);
6479			if (ret < 0)
6480				goto out;
6481			else if (ret > 0)
6482				break;
6483			continue;
6484		}
6485
6486		btrfs_item_key_to_cpu(leaf, &key, slot);
6487		if (key.objectid < sctx->cur_ino ||
6488		    key.type < BTRFS_EXTENT_DATA_KEY)
6489			goto next;
6490		if (key.objectid > sctx->cur_ino ||
6491		    key.type > BTRFS_EXTENT_DATA_KEY ||
6492		    key.offset >= end)
6493			break;
6494
6495		fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
6496		extent_end = btrfs_file_extent_end(path);
 
 
 
 
 
 
 
 
 
6497		if (extent_end <= start)
6498			goto next;
6499		if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0) {
6500			search_start = extent_end;
6501			goto next;
6502		}
6503		ret = 0;
6504		goto out;
6505next:
6506		path->slots[0]++;
6507	}
6508	ret = 1;
6509out:
6510	btrfs_free_path(path);
6511	return ret;
6512}
6513
6514static int maybe_send_hole(struct send_ctx *sctx, struct btrfs_path *path,
6515			   struct btrfs_key *key)
6516{
 
 
 
6517	int ret = 0;
6518
6519	if (sctx->cur_ino != key->objectid || !need_send_hole(sctx))
6520		return 0;
6521
6522	/*
6523	 * Get last extent's end offset (exclusive) if we haven't determined it
6524	 * yet (we're processing the first file extent item that is new), or if
6525	 * we're at the first slot of a leaf and the last extent's end is less
6526	 * than the current extent's offset, because we might have skipped
6527	 * entire leaves that contained only file extent items for our current
6528	 * inode. These leaves have a generation number smaller (older) than the
6529	 * one in the current leaf and the leaf our last extent came from, and
6530	 * are located between these 2 leaves.
6531	 */
6532	if ((sctx->cur_inode_last_extent == (u64)-1) ||
6533	    (path->slots[0] == 0 && sctx->cur_inode_last_extent < key->offset)) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6534		ret = get_last_extent(sctx, key->offset - 1);
6535		if (ret)
6536			return ret;
6537	}
6538
6539	if (sctx->cur_inode_last_extent < key->offset) {
6540		ret = range_is_hole_in_parent(sctx,
6541					      sctx->cur_inode_last_extent,
6542					      key->offset);
6543		if (ret < 0)
6544			return ret;
6545		else if (ret == 0)
6546			ret = send_hole(sctx, key->offset);
6547		else
6548			ret = 0;
6549	}
6550	sctx->cur_inode_last_extent = btrfs_file_extent_end(path);
6551	return ret;
6552}
6553
6554static int process_extent(struct send_ctx *sctx,
6555			  struct btrfs_path *path,
6556			  struct btrfs_key *key)
6557{
6558	struct clone_root *found_clone = NULL;
6559	int ret = 0;
6560
6561	if (S_ISLNK(sctx->cur_inode_mode))
6562		return 0;
6563
6564	if (sctx->parent_root && !sctx->cur_inode_new) {
6565		ret = is_extent_unchanged(sctx, path, key);
6566		if (ret < 0)
6567			goto out;
6568		if (ret) {
6569			ret = 0;
6570			goto out_hole;
6571		}
6572	} else {
6573		struct btrfs_file_extent_item *ei;
6574		u8 type;
6575
6576		ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
6577				    struct btrfs_file_extent_item);
6578		type = btrfs_file_extent_type(path->nodes[0], ei);
6579		if (type == BTRFS_FILE_EXTENT_PREALLOC ||
6580		    type == BTRFS_FILE_EXTENT_REG) {
6581			/*
6582			 * The send spec does not have a prealloc command yet,
6583			 * so just leave a hole for prealloc'ed extents until
6584			 * we have enough commands queued up to justify rev'ing
6585			 * the send spec.
6586			 */
6587			if (type == BTRFS_FILE_EXTENT_PREALLOC) {
6588				ret = 0;
6589				goto out;
6590			}
6591
6592			/* Have a hole, just skip it. */
6593			if (btrfs_file_extent_disk_bytenr(path->nodes[0], ei) == 0) {
6594				ret = 0;
6595				goto out;
6596			}
6597		}
6598	}
6599
6600	ret = find_extent_clone(sctx, path, key->objectid, key->offset,
6601			sctx->cur_inode_size, &found_clone);
6602	if (ret != -ENOENT && ret < 0)
6603		goto out;
6604
6605	ret = send_write_or_clone(sctx, path, key, found_clone);
6606	if (ret)
6607		goto out;
6608out_hole:
6609	ret = maybe_send_hole(sctx, path, key);
6610out:
6611	return ret;
6612}
6613
6614static int process_all_extents(struct send_ctx *sctx)
6615{
6616	int ret = 0;
6617	int iter_ret = 0;
6618	struct btrfs_root *root;
6619	struct btrfs_path *path;
6620	struct btrfs_key key;
6621	struct btrfs_key found_key;
 
 
6622
6623	root = sctx->send_root;
6624	path = alloc_path_for_send();
6625	if (!path)
6626		return -ENOMEM;
6627
6628	key.objectid = sctx->cmp_key->objectid;
6629	key.type = BTRFS_EXTENT_DATA_KEY;
6630	key.offset = 0;
6631	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6632		if (found_key.objectid != key.objectid ||
6633		    found_key.type != key.type) {
6634			ret = 0;
6635			break;
6636		}
6637
6638		ret = process_extent(sctx, path, &found_key);
6639		if (ret < 0)
6640			break;
 
 
6641	}
6642	/* Catch error found during iteration */
6643	if (iter_ret < 0)
6644		ret = iter_ret;
6645
 
6646	btrfs_free_path(path);
6647	return ret;
6648}
6649
6650static int process_recorded_refs_if_needed(struct send_ctx *sctx, int at_end,
6651					   int *pending_move,
6652					   int *refs_processed)
6653{
6654	int ret = 0;
6655
6656	if (sctx->cur_ino == 0)
6657		goto out;
6658	if (!at_end && sctx->cur_ino == sctx->cmp_key->objectid &&
6659	    sctx->cmp_key->type <= BTRFS_INODE_EXTREF_KEY)
6660		goto out;
6661	if (list_empty(&sctx->new_refs) && list_empty(&sctx->deleted_refs))
6662		goto out;
6663
6664	ret = process_recorded_refs(sctx, pending_move);
6665	if (ret < 0)
6666		goto out;
6667
6668	*refs_processed = 1;
6669out:
6670	return ret;
6671}
6672
6673static int finish_inode_if_needed(struct send_ctx *sctx, int at_end)
6674{
6675	int ret = 0;
6676	struct btrfs_inode_info info;
6677	u64 left_mode;
6678	u64 left_uid;
6679	u64 left_gid;
6680	u64 left_fileattr;
6681	u64 right_mode;
6682	u64 right_uid;
6683	u64 right_gid;
6684	u64 right_fileattr;
6685	int need_chmod = 0;
6686	int need_chown = 0;
6687	bool need_fileattr = false;
6688	int need_truncate = 1;
6689	int pending_move = 0;
6690	int refs_processed = 0;
6691
6692	if (sctx->ignore_cur_inode)
6693		return 0;
6694
6695	ret = process_recorded_refs_if_needed(sctx, at_end, &pending_move,
6696					      &refs_processed);
6697	if (ret < 0)
6698		goto out;
6699
6700	/*
6701	 * We have processed the refs and thus need to advance send_progress.
6702	 * Now, calls to get_cur_xxx will take the updated refs of the current
6703	 * inode into account.
6704	 *
6705	 * On the other hand, if our current inode is a directory and couldn't
6706	 * be moved/renamed because its parent was renamed/moved too and it has
6707	 * a higher inode number, we can only move/rename our current inode
6708	 * after we moved/renamed its parent. Therefore in this case operate on
6709	 * the old path (pre move/rename) of our current inode, and the
6710	 * move/rename will be performed later.
6711	 */
6712	if (refs_processed && !pending_move)
6713		sctx->send_progress = sctx->cur_ino + 1;
6714
6715	if (sctx->cur_ino == 0 || sctx->cur_inode_deleted)
6716		goto out;
6717	if (!at_end && sctx->cmp_key->objectid == sctx->cur_ino)
6718		goto out;
6719	ret = get_inode_info(sctx->send_root, sctx->cur_ino, &info);
 
 
6720	if (ret < 0)
6721		goto out;
6722	left_mode = info.mode;
6723	left_uid = info.uid;
6724	left_gid = info.gid;
6725	left_fileattr = info.fileattr;
6726
6727	if (!sctx->parent_root || sctx->cur_inode_new) {
6728		need_chown = 1;
6729		if (!S_ISLNK(sctx->cur_inode_mode))
6730			need_chmod = 1;
6731		if (sctx->cur_inode_next_write_offset == sctx->cur_inode_size)
6732			need_truncate = 0;
6733	} else {
6734		u64 old_size;
6735
6736		ret = get_inode_info(sctx->parent_root, sctx->cur_ino, &info);
 
 
6737		if (ret < 0)
6738			goto out;
6739		old_size = info.size;
6740		right_mode = info.mode;
6741		right_uid = info.uid;
6742		right_gid = info.gid;
6743		right_fileattr = info.fileattr;
6744
6745		if (left_uid != right_uid || left_gid != right_gid)
6746			need_chown = 1;
6747		if (!S_ISLNK(sctx->cur_inode_mode) && left_mode != right_mode)
6748			need_chmod = 1;
6749		if (!S_ISLNK(sctx->cur_inode_mode) && left_fileattr != right_fileattr)
6750			need_fileattr = true;
6751		if ((old_size == sctx->cur_inode_size) ||
6752		    (sctx->cur_inode_size > old_size &&
6753		     sctx->cur_inode_next_write_offset == sctx->cur_inode_size))
6754			need_truncate = 0;
6755	}
6756
6757	if (S_ISREG(sctx->cur_inode_mode)) {
6758		if (need_send_hole(sctx)) {
6759			if (sctx->cur_inode_last_extent == (u64)-1 ||
6760			    sctx->cur_inode_last_extent <
6761			    sctx->cur_inode_size) {
6762				ret = get_last_extent(sctx, (u64)-1);
6763				if (ret)
6764					goto out;
6765			}
6766			if (sctx->cur_inode_last_extent < sctx->cur_inode_size) {
6767				ret = range_is_hole_in_parent(sctx,
6768						      sctx->cur_inode_last_extent,
6769						      sctx->cur_inode_size);
6770				if (ret < 0) {
6771					goto out;
6772				} else if (ret == 0) {
6773					ret = send_hole(sctx, sctx->cur_inode_size);
6774					if (ret < 0)
6775						goto out;
6776				} else {
6777					/* Range is already a hole, skip. */
6778					ret = 0;
6779				}
6780			}
6781		}
6782		if (need_truncate) {
6783			ret = send_truncate(sctx, sctx->cur_ino,
6784					    sctx->cur_inode_gen,
6785					    sctx->cur_inode_size);
6786			if (ret < 0)
6787				goto out;
6788		}
6789	}
6790
6791	if (need_chown) {
6792		ret = send_chown(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6793				left_uid, left_gid);
6794		if (ret < 0)
6795			goto out;
6796	}
6797	if (need_chmod) {
6798		ret = send_chmod(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6799				left_mode);
6800		if (ret < 0)
6801			goto out;
6802	}
6803	if (need_fileattr) {
6804		ret = send_fileattr(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6805				    left_fileattr);
6806		if (ret < 0)
6807			goto out;
6808	}
6809
6810	if (proto_cmd_ok(sctx, BTRFS_SEND_C_ENABLE_VERITY)
6811	    && sctx->cur_inode_needs_verity) {
6812		ret = process_verity(sctx);
6813		if (ret < 0)
6814			goto out;
6815	}
6816
6817	ret = send_capabilities(sctx);
6818	if (ret < 0)
6819		goto out;
6820
6821	/*
6822	 * If other directory inodes depended on our current directory
6823	 * inode's move/rename, now do their move/rename operations.
6824	 */
6825	if (!is_waiting_for_move(sctx, sctx->cur_ino)) {
6826		ret = apply_children_dir_moves(sctx);
6827		if (ret)
6828			goto out;
6829		/*
6830		 * Need to send that every time, no matter if it actually
6831		 * changed between the two trees as we have done changes to
6832		 * the inode before. If our inode is a directory and it's
6833		 * waiting to be moved/renamed, we will send its utimes when
6834		 * it's moved/renamed, therefore we don't need to do it here.
6835		 */
6836		sctx->send_progress = sctx->cur_ino + 1;
6837
6838		/*
6839		 * If the current inode is a non-empty directory, delay issuing
6840		 * the utimes command for it, as it's very likely we have inodes
6841		 * with an higher number inside it. We want to issue the utimes
6842		 * command only after adding all dentries to it.
6843		 */
6844		if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_size > 0)
6845			ret = cache_dir_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen);
6846		else
6847			ret = send_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen);
6848
6849		if (ret < 0)
6850			goto out;
6851	}
6852
6853out:
6854	if (!ret)
6855		ret = trim_dir_utimes_cache(sctx);
6856
6857	return ret;
6858}
6859
6860static void close_current_inode(struct send_ctx *sctx)
 
 
 
 
 
 
6861{
6862	u64 i_size;
6863
6864	if (sctx->cur_inode == NULL)
6865		return;
 
6866
6867	i_size = i_size_read(sctx->cur_inode);
 
 
 
 
 
 
 
 
 
 
6868
6869	/*
6870	 * If we are doing an incremental send, we may have extents between the
6871	 * last processed extent and the i_size that have not been processed
6872	 * because they haven't changed but we may have read some of their pages
6873	 * through readahead, see the comments at send_extent_data().
6874	 */
6875	if (sctx->clean_page_cache && sctx->page_cache_clear_start < i_size)
6876		truncate_inode_pages_range(&sctx->cur_inode->i_data,
6877					   sctx->page_cache_clear_start,
6878					   round_up(i_size, PAGE_SIZE) - 1);
 
 
 
6879
6880	iput(sctx->cur_inode);
6881	sctx->cur_inode = NULL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6882}
6883
6884static int changed_inode(struct send_ctx *sctx,
6885			 enum btrfs_compare_tree_result result)
6886{
6887	int ret = 0;
6888	struct btrfs_key *key = sctx->cmp_key;
6889	struct btrfs_inode_item *left_ii = NULL;
6890	struct btrfs_inode_item *right_ii = NULL;
6891	u64 left_gen = 0;
6892	u64 right_gen = 0;
6893
6894	close_current_inode(sctx);
6895
6896	sctx->cur_ino = key->objectid;
6897	sctx->cur_inode_new_gen = false;
6898	sctx->cur_inode_last_extent = (u64)-1;
6899	sctx->cur_inode_next_write_offset = 0;
6900	sctx->ignore_cur_inode = false;
6901
6902	/*
6903	 * Set send_progress to current inode. This will tell all get_cur_xxx
6904	 * functions that the current inode's refs are not updated yet. Later,
6905	 * when process_recorded_refs is finished, it is set to cur_ino + 1.
6906	 */
6907	sctx->send_progress = sctx->cur_ino;
6908
6909	if (result == BTRFS_COMPARE_TREE_NEW ||
6910	    result == BTRFS_COMPARE_TREE_CHANGED) {
6911		left_ii = btrfs_item_ptr(sctx->left_path->nodes[0],
6912				sctx->left_path->slots[0],
6913				struct btrfs_inode_item);
6914		left_gen = btrfs_inode_generation(sctx->left_path->nodes[0],
6915				left_ii);
6916	} else {
6917		right_ii = btrfs_item_ptr(sctx->right_path->nodes[0],
6918				sctx->right_path->slots[0],
6919				struct btrfs_inode_item);
6920		right_gen = btrfs_inode_generation(sctx->right_path->nodes[0],
6921				right_ii);
6922	}
6923	if (result == BTRFS_COMPARE_TREE_CHANGED) {
6924		right_ii = btrfs_item_ptr(sctx->right_path->nodes[0],
6925				sctx->right_path->slots[0],
6926				struct btrfs_inode_item);
6927
6928		right_gen = btrfs_inode_generation(sctx->right_path->nodes[0],
6929				right_ii);
6930
6931		/*
6932		 * The cur_ino = root dir case is special here. We can't treat
6933		 * the inode as deleted+reused because it would generate a
6934		 * stream that tries to delete/mkdir the root dir.
6935		 */
6936		if (left_gen != right_gen &&
6937		    sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)
6938			sctx->cur_inode_new_gen = true;
6939	}
6940
6941	/*
6942	 * Normally we do not find inodes with a link count of zero (orphans)
6943	 * because the most common case is to create a snapshot and use it
6944	 * for a send operation. However other less common use cases involve
6945	 * using a subvolume and send it after turning it to RO mode just
6946	 * after deleting all hard links of a file while holding an open
6947	 * file descriptor against it or turning a RO snapshot into RW mode,
6948	 * keep an open file descriptor against a file, delete it and then
6949	 * turn the snapshot back to RO mode before using it for a send
6950	 * operation. The former is what the receiver operation does.
6951	 * Therefore, if we want to send these snapshots soon after they're
6952	 * received, we need to handle orphan inodes as well. Moreover, orphans
6953	 * can appear not only in the send snapshot but also in the parent
6954	 * snapshot. Here are several cases:
6955	 *
6956	 * Case 1: BTRFS_COMPARE_TREE_NEW
6957	 *       |  send snapshot  | action
6958	 * --------------------------------
6959	 * nlink |        0        | ignore
6960	 *
6961	 * Case 2: BTRFS_COMPARE_TREE_DELETED
6962	 *       | parent snapshot | action
6963	 * ----------------------------------
6964	 * nlink |        0        | as usual
6965	 * Note: No unlinks will be sent because there're no paths for it.
6966	 *
6967	 * Case 3: BTRFS_COMPARE_TREE_CHANGED
6968	 *           |       | parent snapshot | send snapshot | action
6969	 * -----------------------------------------------------------------------
6970	 * subcase 1 | nlink |        0        |       0       | ignore
6971	 * subcase 2 | nlink |       >0        |       0       | new_gen(deletion)
6972	 * subcase 3 | nlink |        0        |      >0       | new_gen(creation)
6973	 *
6974	 */
6975	if (result == BTRFS_COMPARE_TREE_NEW) {
6976		if (btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii) == 0) {
 
 
 
 
6977			sctx->ignore_cur_inode = true;
 
 
6978			goto out;
6979		}
 
 
 
6980		sctx->cur_inode_gen = left_gen;
6981		sctx->cur_inode_new = true;
6982		sctx->cur_inode_deleted = false;
6983		sctx->cur_inode_size = btrfs_inode_size(
6984				sctx->left_path->nodes[0], left_ii);
6985		sctx->cur_inode_mode = btrfs_inode_mode(
6986				sctx->left_path->nodes[0], left_ii);
6987		sctx->cur_inode_rdev = btrfs_inode_rdev(
6988				sctx->left_path->nodes[0], left_ii);
6989		if (sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)
6990			ret = send_create_inode_if_needed(sctx);
6991	} else if (result == BTRFS_COMPARE_TREE_DELETED) {
6992		sctx->cur_inode_gen = right_gen;
6993		sctx->cur_inode_new = false;
6994		sctx->cur_inode_deleted = true;
6995		sctx->cur_inode_size = btrfs_inode_size(
6996				sctx->right_path->nodes[0], right_ii);
6997		sctx->cur_inode_mode = btrfs_inode_mode(
6998				sctx->right_path->nodes[0], right_ii);
6999	} else if (result == BTRFS_COMPARE_TREE_CHANGED) {
7000		u32 new_nlinks, old_nlinks;
7001
7002		new_nlinks = btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii);
7003		old_nlinks = btrfs_inode_nlink(sctx->right_path->nodes[0], right_ii);
7004		if (new_nlinks == 0 && old_nlinks == 0) {
7005			sctx->ignore_cur_inode = true;
7006			goto out;
7007		} else if (new_nlinks == 0 || old_nlinks == 0) {
7008			sctx->cur_inode_new_gen = 1;
7009		}
7010		/*
7011		 * We need to do some special handling in case the inode was
7012		 * reported as changed with a changed generation number. This
7013		 * means that the original inode was deleted and new inode
7014		 * reused the same inum. So we have to treat the old inode as
7015		 * deleted and the new one as new.
7016		 */
7017		if (sctx->cur_inode_new_gen) {
7018			/*
7019			 * First, process the inode as if it was deleted.
7020			 */
7021			if (old_nlinks > 0) {
7022				sctx->cur_inode_gen = right_gen;
7023				sctx->cur_inode_new = false;
7024				sctx->cur_inode_deleted = true;
7025				sctx->cur_inode_size = btrfs_inode_size(
7026						sctx->right_path->nodes[0], right_ii);
7027				sctx->cur_inode_mode = btrfs_inode_mode(
7028						sctx->right_path->nodes[0], right_ii);
7029				ret = process_all_refs(sctx,
7030						BTRFS_COMPARE_TREE_DELETED);
7031				if (ret < 0)
7032					goto out;
7033			}
7034
7035			/*
7036			 * Now process the inode as if it was new.
7037			 */
7038			if (new_nlinks > 0) {
7039				sctx->cur_inode_gen = left_gen;
7040				sctx->cur_inode_new = true;
7041				sctx->cur_inode_deleted = false;
7042				sctx->cur_inode_size = btrfs_inode_size(
7043						sctx->left_path->nodes[0],
7044						left_ii);
7045				sctx->cur_inode_mode = btrfs_inode_mode(
7046						sctx->left_path->nodes[0],
7047						left_ii);
7048				sctx->cur_inode_rdev = btrfs_inode_rdev(
7049						sctx->left_path->nodes[0],
7050						left_ii);
7051				ret = send_create_inode_if_needed(sctx);
7052				if (ret < 0)
7053					goto out;
7054
7055				ret = process_all_refs(sctx, BTRFS_COMPARE_TREE_NEW);
7056				if (ret < 0)
7057					goto out;
7058				/*
7059				 * Advance send_progress now as we did not get
7060				 * into process_recorded_refs_if_needed in the
7061				 * new_gen case.
7062				 */
7063				sctx->send_progress = sctx->cur_ino + 1;
7064
7065				/*
7066				 * Now process all extents and xattrs of the
7067				 * inode as if they were all new.
7068				 */
7069				ret = process_all_extents(sctx);
7070				if (ret < 0)
7071					goto out;
7072				ret = process_all_new_xattrs(sctx);
7073				if (ret < 0)
7074					goto out;
7075			}
7076		} else {
7077			sctx->cur_inode_gen = left_gen;
7078			sctx->cur_inode_new = false;
7079			sctx->cur_inode_new_gen = false;
7080			sctx->cur_inode_deleted = false;
7081			sctx->cur_inode_size = btrfs_inode_size(
7082					sctx->left_path->nodes[0], left_ii);
7083			sctx->cur_inode_mode = btrfs_inode_mode(
7084					sctx->left_path->nodes[0], left_ii);
7085		}
7086	}
7087
7088out:
7089	return ret;
7090}
7091
7092/*
7093 * We have to process new refs before deleted refs, but compare_trees gives us
7094 * the new and deleted refs mixed. To fix this, we record the new/deleted refs
7095 * first and later process them in process_recorded_refs.
7096 * For the cur_inode_new_gen case, we skip recording completely because
7097 * changed_inode did already initiate processing of refs. The reason for this is
7098 * that in this case, compare_tree actually compares the refs of 2 different
7099 * inodes. To fix this, process_all_refs is used in changed_inode to handle all
7100 * refs of the right tree as deleted and all refs of the left tree as new.
7101 */
7102static int changed_ref(struct send_ctx *sctx,
7103		       enum btrfs_compare_tree_result result)
7104{
7105	int ret = 0;
7106
7107	if (sctx->cur_ino != sctx->cmp_key->objectid) {
7108		inconsistent_snapshot_error(sctx, result, "reference");
7109		return -EIO;
7110	}
7111
7112	if (!sctx->cur_inode_new_gen &&
7113	    sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) {
7114		if (result == BTRFS_COMPARE_TREE_NEW)
7115			ret = record_new_ref(sctx);
7116		else if (result == BTRFS_COMPARE_TREE_DELETED)
7117			ret = record_deleted_ref(sctx);
7118		else if (result == BTRFS_COMPARE_TREE_CHANGED)
7119			ret = record_changed_ref(sctx);
7120	}
7121
7122	return ret;
7123}
7124
7125/*
7126 * Process new/deleted/changed xattrs. We skip processing in the
7127 * cur_inode_new_gen case because changed_inode did already initiate processing
7128 * of xattrs. The reason is the same as in changed_ref
7129 */
7130static int changed_xattr(struct send_ctx *sctx,
7131			 enum btrfs_compare_tree_result result)
7132{
7133	int ret = 0;
7134
7135	if (sctx->cur_ino != sctx->cmp_key->objectid) {
7136		inconsistent_snapshot_error(sctx, result, "xattr");
7137		return -EIO;
7138	}
7139
7140	if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
7141		if (result == BTRFS_COMPARE_TREE_NEW)
7142			ret = process_new_xattr(sctx);
7143		else if (result == BTRFS_COMPARE_TREE_DELETED)
7144			ret = process_deleted_xattr(sctx);
7145		else if (result == BTRFS_COMPARE_TREE_CHANGED)
7146			ret = process_changed_xattr(sctx);
7147	}
7148
7149	return ret;
7150}
7151
7152/*
7153 * Process new/deleted/changed extents. We skip processing in the
7154 * cur_inode_new_gen case because changed_inode did already initiate processing
7155 * of extents. The reason is the same as in changed_ref
7156 */
7157static int changed_extent(struct send_ctx *sctx,
7158			  enum btrfs_compare_tree_result result)
7159{
7160	int ret = 0;
7161
7162	/*
7163	 * We have found an extent item that changed without the inode item
7164	 * having changed. This can happen either after relocation (where the
7165	 * disk_bytenr of an extent item is replaced at
7166	 * relocation.c:replace_file_extents()) or after deduplication into a
7167	 * file in both the parent and send snapshots (where an extent item can
7168	 * get modified or replaced with a new one). Note that deduplication
7169	 * updates the inode item, but it only changes the iversion (sequence
7170	 * field in the inode item) of the inode, so if a file is deduplicated
7171	 * the same amount of times in both the parent and send snapshots, its
7172	 * iversion becomes the same in both snapshots, whence the inode item is
7173	 * the same on both snapshots.
7174	 */
7175	if (sctx->cur_ino != sctx->cmp_key->objectid)
7176		return 0;
7177
7178	if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
7179		if (result != BTRFS_COMPARE_TREE_DELETED)
7180			ret = process_extent(sctx, sctx->left_path,
7181					sctx->cmp_key);
7182	}
7183
7184	return ret;
7185}
7186
7187static int changed_verity(struct send_ctx *sctx, enum btrfs_compare_tree_result result)
7188{
7189	if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
7190		if (result == BTRFS_COMPARE_TREE_NEW)
7191			sctx->cur_inode_needs_verity = true;
7192	}
7193	return 0;
7194}
7195
7196static int dir_changed(struct send_ctx *sctx, u64 dir)
7197{
7198	u64 orig_gen, new_gen;
7199	int ret;
7200
7201	ret = get_inode_gen(sctx->send_root, dir, &new_gen);
 
7202	if (ret)
7203		return ret;
7204
7205	ret = get_inode_gen(sctx->parent_root, dir, &orig_gen);
 
7206	if (ret)
7207		return ret;
7208
7209	return (orig_gen != new_gen) ? 1 : 0;
7210}
7211
7212static int compare_refs(struct send_ctx *sctx, struct btrfs_path *path,
7213			struct btrfs_key *key)
7214{
7215	struct btrfs_inode_extref *extref;
7216	struct extent_buffer *leaf;
7217	u64 dirid = 0, last_dirid = 0;
7218	unsigned long ptr;
7219	u32 item_size;
7220	u32 cur_offset = 0;
7221	int ref_name_len;
7222	int ret = 0;
7223
7224	/* Easy case, just check this one dirid */
7225	if (key->type == BTRFS_INODE_REF_KEY) {
7226		dirid = key->offset;
7227
7228		ret = dir_changed(sctx, dirid);
7229		goto out;
7230	}
7231
7232	leaf = path->nodes[0];
7233	item_size = btrfs_item_size(leaf, path->slots[0]);
7234	ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
7235	while (cur_offset < item_size) {
7236		extref = (struct btrfs_inode_extref *)(ptr +
7237						       cur_offset);
7238		dirid = btrfs_inode_extref_parent(leaf, extref);
7239		ref_name_len = btrfs_inode_extref_name_len(leaf, extref);
7240		cur_offset += ref_name_len + sizeof(*extref);
7241		if (dirid == last_dirid)
7242			continue;
7243		ret = dir_changed(sctx, dirid);
7244		if (ret)
7245			break;
7246		last_dirid = dirid;
7247	}
7248out:
7249	return ret;
7250}
7251
7252/*
7253 * Updates compare related fields in sctx and simply forwards to the actual
7254 * changed_xxx functions.
7255 */
7256static int changed_cb(struct btrfs_path *left_path,
7257		      struct btrfs_path *right_path,
7258		      struct btrfs_key *key,
7259		      enum btrfs_compare_tree_result result,
7260		      struct send_ctx *sctx)
7261{
7262	int ret = 0;
7263
7264	/*
7265	 * We can not hold the commit root semaphore here. This is because in
7266	 * the case of sending and receiving to the same filesystem, using a
7267	 * pipe, could result in a deadlock:
7268	 *
7269	 * 1) The task running send blocks on the pipe because it's full;
7270	 *
7271	 * 2) The task running receive, which is the only consumer of the pipe,
7272	 *    is waiting for a transaction commit (for example due to a space
7273	 *    reservation when doing a write or triggering a transaction commit
7274	 *    when creating a subvolume);
7275	 *
7276	 * 3) The transaction is waiting to write lock the commit root semaphore,
7277	 *    but can not acquire it since it's being held at 1).
7278	 *
7279	 * Down this call chain we write to the pipe through kernel_write().
7280	 * The same type of problem can also happen when sending to a file that
7281	 * is stored in the same filesystem - when reserving space for a write
7282	 * into the file, we can trigger a transaction commit.
7283	 *
7284	 * Our caller has supplied us with clones of leaves from the send and
7285	 * parent roots, so we're safe here from a concurrent relocation and
7286	 * further reallocation of metadata extents while we are here. Below we
7287	 * also assert that the leaves are clones.
7288	 */
7289	lockdep_assert_not_held(&sctx->send_root->fs_info->commit_root_sem);
7290
7291	/*
7292	 * We always have a send root, so left_path is never NULL. We will not
7293	 * have a leaf when we have reached the end of the send root but have
7294	 * not yet reached the end of the parent root.
7295	 */
7296	if (left_path->nodes[0])
7297		ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED,
7298				&left_path->nodes[0]->bflags));
7299	/*
7300	 * When doing a full send we don't have a parent root, so right_path is
7301	 * NULL. When doing an incremental send, we may have reached the end of
7302	 * the parent root already, so we don't have a leaf at right_path.
7303	 */
7304	if (right_path && right_path->nodes[0])
7305		ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED,
7306				&right_path->nodes[0]->bflags));
7307
7308	if (result == BTRFS_COMPARE_TREE_SAME) {
7309		if (key->type == BTRFS_INODE_REF_KEY ||
7310		    key->type == BTRFS_INODE_EXTREF_KEY) {
7311			ret = compare_refs(sctx, left_path, key);
7312			if (!ret)
7313				return 0;
7314			if (ret < 0)
7315				return ret;
7316		} else if (key->type == BTRFS_EXTENT_DATA_KEY) {
7317			return maybe_send_hole(sctx, left_path, key);
7318		} else {
7319			return 0;
7320		}
7321		result = BTRFS_COMPARE_TREE_CHANGED;
7322		ret = 0;
7323	}
7324
7325	sctx->left_path = left_path;
7326	sctx->right_path = right_path;
7327	sctx->cmp_key = key;
7328
7329	ret = finish_inode_if_needed(sctx, 0);
7330	if (ret < 0)
7331		goto out;
7332
7333	/* Ignore non-FS objects */
7334	if (key->objectid == BTRFS_FREE_INO_OBJECTID ||
7335	    key->objectid == BTRFS_FREE_SPACE_OBJECTID)
7336		goto out;
7337
7338	if (key->type == BTRFS_INODE_ITEM_KEY) {
7339		ret = changed_inode(sctx, result);
7340	} else if (!sctx->ignore_cur_inode) {
7341		if (key->type == BTRFS_INODE_REF_KEY ||
7342		    key->type == BTRFS_INODE_EXTREF_KEY)
7343			ret = changed_ref(sctx, result);
7344		else if (key->type == BTRFS_XATTR_ITEM_KEY)
7345			ret = changed_xattr(sctx, result);
7346		else if (key->type == BTRFS_EXTENT_DATA_KEY)
7347			ret = changed_extent(sctx, result);
7348		else if (key->type == BTRFS_VERITY_DESC_ITEM_KEY &&
7349			 key->offset == 0)
7350			ret = changed_verity(sctx, result);
7351	}
7352
7353out:
7354	return ret;
7355}
7356
7357static int search_key_again(const struct send_ctx *sctx,
7358			    struct btrfs_root *root,
7359			    struct btrfs_path *path,
7360			    const struct btrfs_key *key)
7361{
7362	int ret;
7363
7364	if (!path->need_commit_sem)
7365		lockdep_assert_held_read(&root->fs_info->commit_root_sem);
7366
7367	/*
7368	 * Roots used for send operations are readonly and no one can add,
7369	 * update or remove keys from them, so we should be able to find our
7370	 * key again. The only exception is deduplication, which can operate on
7371	 * readonly roots and add, update or remove keys to/from them - but at
7372	 * the moment we don't allow it to run in parallel with send.
7373	 */
7374	ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
7375	ASSERT(ret <= 0);
7376	if (ret > 0) {
7377		btrfs_print_tree(path->nodes[path->lowest_level], false);
7378		btrfs_err(root->fs_info,
7379"send: key (%llu %u %llu) not found in %s root %llu, lowest_level %d, slot %d",
7380			  key->objectid, key->type, key->offset,
7381			  (root == sctx->parent_root ? "parent" : "send"),
7382			  btrfs_root_id(root), path->lowest_level,
7383			  path->slots[path->lowest_level]);
7384		return -EUCLEAN;
7385	}
7386
7387	return ret;
7388}
7389
7390static int full_send_tree(struct send_ctx *sctx)
7391{
7392	int ret;
7393	struct btrfs_root *send_root = sctx->send_root;
7394	struct btrfs_key key;
7395	struct btrfs_fs_info *fs_info = send_root->fs_info;
7396	struct btrfs_path *path;
 
 
7397
7398	path = alloc_path_for_send();
7399	if (!path)
7400		return -ENOMEM;
7401	path->reada = READA_FORWARD_ALWAYS;
7402
7403	key.objectid = BTRFS_FIRST_FREE_OBJECTID;
7404	key.type = BTRFS_INODE_ITEM_KEY;
7405	key.offset = 0;
7406
7407	down_read(&fs_info->commit_root_sem);
7408	sctx->last_reloc_trans = fs_info->last_reloc_trans;
7409	up_read(&fs_info->commit_root_sem);
7410
7411	ret = btrfs_search_slot_for_read(send_root, &key, path, 1, 0);
7412	if (ret < 0)
7413		goto out;
7414	if (ret)
7415		goto out_finish;
7416
7417	while (1) {
7418		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
 
 
7419
7420		ret = changed_cb(path, NULL, &key,
7421				 BTRFS_COMPARE_TREE_NEW, sctx);
7422		if (ret < 0)
7423			goto out;
7424
7425		down_read(&fs_info->commit_root_sem);
7426		if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
7427			sctx->last_reloc_trans = fs_info->last_reloc_trans;
7428			up_read(&fs_info->commit_root_sem);
7429			/*
7430			 * A transaction used for relocating a block group was
7431			 * committed or is about to finish its commit. Release
7432			 * our path (leaf) and restart the search, so that we
7433			 * avoid operating on any file extent items that are
7434			 * stale, with a disk_bytenr that reflects a pre
7435			 * relocation value. This way we avoid as much as
7436			 * possible to fallback to regular writes when checking
7437			 * if we can clone file ranges.
7438			 */
7439			btrfs_release_path(path);
7440			ret = search_key_again(sctx, send_root, path, &key);
7441			if (ret < 0)
7442				goto out;
7443		} else {
7444			up_read(&fs_info->commit_root_sem);
7445		}
7446
7447		ret = btrfs_next_item(send_root, path);
7448		if (ret < 0)
7449			goto out;
7450		if (ret) {
7451			ret  = 0;
7452			break;
7453		}
7454	}
7455
7456out_finish:
7457	ret = finish_inode_if_needed(sctx, 1);
7458
7459out:
7460	btrfs_free_path(path);
7461	return ret;
7462}
7463
7464static int replace_node_with_clone(struct btrfs_path *path, int level)
7465{
7466	struct extent_buffer *clone;
7467
7468	clone = btrfs_clone_extent_buffer(path->nodes[level]);
7469	if (!clone)
7470		return -ENOMEM;
7471
7472	free_extent_buffer(path->nodes[level]);
7473	path->nodes[level] = clone;
7474
7475	return 0;
7476}
7477
7478static int tree_move_down(struct btrfs_path *path, int *level, u64 reada_min_gen)
7479{
7480	struct extent_buffer *eb;
7481	struct extent_buffer *parent = path->nodes[*level];
7482	int slot = path->slots[*level];
7483	const int nritems = btrfs_header_nritems(parent);
7484	u64 reada_max;
7485	u64 reada_done = 0;
7486
7487	lockdep_assert_held_read(&parent->fs_info->commit_root_sem);
7488	ASSERT(*level != 0);
7489
7490	eb = btrfs_read_node_slot(parent, slot);
7491	if (IS_ERR(eb))
7492		return PTR_ERR(eb);
7493
7494	/*
7495	 * Trigger readahead for the next leaves we will process, so that it is
7496	 * very likely that when we need them they are already in memory and we
7497	 * will not block on disk IO. For nodes we only do readahead for one,
7498	 * since the time window between processing nodes is typically larger.
7499	 */
7500	reada_max = (*level == 1 ? SZ_128K : eb->fs_info->nodesize);
7501
7502	for (slot++; slot < nritems && reada_done < reada_max; slot++) {
7503		if (btrfs_node_ptr_generation(parent, slot) > reada_min_gen) {
7504			btrfs_readahead_node_child(parent, slot);
7505			reada_done += eb->fs_info->nodesize;
7506		}
7507	}
7508
7509	path->nodes[*level - 1] = eb;
7510	path->slots[*level - 1] = 0;
7511	(*level)--;
7512
7513	if (*level == 0)
7514		return replace_node_with_clone(path, 0);
7515
7516	return 0;
7517}
7518
7519static int tree_move_next_or_upnext(struct btrfs_path *path,
7520				    int *level, int root_level)
7521{
7522	int ret = 0;
7523	int nritems;
7524	nritems = btrfs_header_nritems(path->nodes[*level]);
7525
7526	path->slots[*level]++;
7527
7528	while (path->slots[*level] >= nritems) {
7529		if (*level == root_level) {
7530			path->slots[*level] = nritems - 1;
7531			return -1;
7532		}
7533
7534		/* move upnext */
7535		path->slots[*level] = 0;
7536		free_extent_buffer(path->nodes[*level]);
7537		path->nodes[*level] = NULL;
7538		(*level)++;
7539		path->slots[*level]++;
7540
7541		nritems = btrfs_header_nritems(path->nodes[*level]);
7542		ret = 1;
7543	}
7544	return ret;
7545}
7546
7547/*
7548 * Returns 1 if it had to move up and next. 0 is returned if it moved only next
7549 * or down.
7550 */
7551static int tree_advance(struct btrfs_path *path,
7552			int *level, int root_level,
7553			int allow_down,
7554			struct btrfs_key *key,
7555			u64 reada_min_gen)
7556{
7557	int ret;
7558
7559	if (*level == 0 || !allow_down) {
7560		ret = tree_move_next_or_upnext(path, level, root_level);
7561	} else {
7562		ret = tree_move_down(path, level, reada_min_gen);
 
 
 
 
 
 
 
 
7563	}
7564
7565	/*
7566	 * Even if we have reached the end of a tree, ret is -1, update the key
7567	 * anyway, so that in case we need to restart due to a block group
7568	 * relocation, we can assert that the last key of the root node still
7569	 * exists in the tree.
7570	 */
7571	if (*level == 0)
7572		btrfs_item_key_to_cpu(path->nodes[*level], key,
7573				      path->slots[*level]);
7574	else
7575		btrfs_node_key_to_cpu(path->nodes[*level], key,
7576				      path->slots[*level]);
7577
7578	return ret;
7579}
7580
7581static int tree_compare_item(struct btrfs_path *left_path,
7582			     struct btrfs_path *right_path,
7583			     char *tmp_buf)
7584{
7585	int cmp;
7586	int len1, len2;
7587	unsigned long off1, off2;
7588
7589	len1 = btrfs_item_size(left_path->nodes[0], left_path->slots[0]);
7590	len2 = btrfs_item_size(right_path->nodes[0], right_path->slots[0]);
7591	if (len1 != len2)
7592		return 1;
7593
7594	off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]);
7595	off2 = btrfs_item_ptr_offset(right_path->nodes[0],
7596				right_path->slots[0]);
7597
7598	read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1);
7599
7600	cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1);
7601	if (cmp)
7602		return 1;
7603	return 0;
7604}
7605
7606/*
7607 * A transaction used for relocating a block group was committed or is about to
7608 * finish its commit. Release our paths and restart the search, so that we are
7609 * not using stale extent buffers:
7610 *
7611 * 1) For levels > 0, we are only holding references of extent buffers, without
7612 *    any locks on them, which does not prevent them from having been relocated
7613 *    and reallocated after the last time we released the commit root semaphore.
7614 *    The exception are the root nodes, for which we always have a clone, see
7615 *    the comment at btrfs_compare_trees();
7616 *
7617 * 2) For leaves, level 0, we are holding copies (clones) of extent buffers, so
7618 *    we are safe from the concurrent relocation and reallocation. However they
7619 *    can have file extent items with a pre relocation disk_bytenr value, so we
7620 *    restart the start from the current commit roots and clone the new leaves so
7621 *    that we get the post relocation disk_bytenr values. Not doing so, could
7622 *    make us clone the wrong data in case there are new extents using the old
7623 *    disk_bytenr that happen to be shared.
7624 */
7625static int restart_after_relocation(struct btrfs_path *left_path,
7626				    struct btrfs_path *right_path,
7627				    const struct btrfs_key *left_key,
7628				    const struct btrfs_key *right_key,
7629				    int left_level,
7630				    int right_level,
7631				    const struct send_ctx *sctx)
7632{
7633	int root_level;
7634	int ret;
7635
7636	lockdep_assert_held_read(&sctx->send_root->fs_info->commit_root_sem);
7637
7638	btrfs_release_path(left_path);
7639	btrfs_release_path(right_path);
7640
7641	/*
7642	 * Since keys can not be added or removed to/from our roots because they
7643	 * are readonly and we do not allow deduplication to run in parallel
7644	 * (which can add, remove or change keys), the layout of the trees should
7645	 * not change.
7646	 */
7647	left_path->lowest_level = left_level;
7648	ret = search_key_again(sctx, sctx->send_root, left_path, left_key);
7649	if (ret < 0)
7650		return ret;
7651
7652	right_path->lowest_level = right_level;
7653	ret = search_key_again(sctx, sctx->parent_root, right_path, right_key);
7654	if (ret < 0)
7655		return ret;
7656
7657	/*
7658	 * If the lowest level nodes are leaves, clone them so that they can be
7659	 * safely used by changed_cb() while not under the protection of the
7660	 * commit root semaphore, even if relocation and reallocation happens in
7661	 * parallel.
7662	 */
7663	if (left_level == 0) {
7664		ret = replace_node_with_clone(left_path, 0);
7665		if (ret < 0)
7666			return ret;
7667	}
7668
7669	if (right_level == 0) {
7670		ret = replace_node_with_clone(right_path, 0);
7671		if (ret < 0)
7672			return ret;
7673	}
7674
7675	/*
7676	 * Now clone the root nodes (unless they happen to be the leaves we have
7677	 * already cloned). This is to protect against concurrent snapshotting of
7678	 * the send and parent roots (see the comment at btrfs_compare_trees()).
7679	 */
7680	root_level = btrfs_header_level(sctx->send_root->commit_root);
7681	if (root_level > 0) {
7682		ret = replace_node_with_clone(left_path, root_level);
7683		if (ret < 0)
7684			return ret;
7685	}
7686
7687	root_level = btrfs_header_level(sctx->parent_root->commit_root);
7688	if (root_level > 0) {
7689		ret = replace_node_with_clone(right_path, root_level);
7690		if (ret < 0)
7691			return ret;
7692	}
7693
7694	return 0;
7695}
7696
7697/*
7698 * This function compares two trees and calls the provided callback for
7699 * every changed/new/deleted item it finds.
7700 * If shared tree blocks are encountered, whole subtrees are skipped, making
7701 * the compare pretty fast on snapshotted subvolumes.
7702 *
7703 * This currently works on commit roots only. As commit roots are read only,
7704 * we don't do any locking. The commit roots are protected with transactions.
7705 * Transactions are ended and rejoined when a commit is tried in between.
7706 *
7707 * This function checks for modifications done to the trees while comparing.
7708 * If it detects a change, it aborts immediately.
7709 */
7710static int btrfs_compare_trees(struct btrfs_root *left_root,
7711			struct btrfs_root *right_root, struct send_ctx *sctx)
 
7712{
7713	struct btrfs_fs_info *fs_info = left_root->fs_info;
7714	int ret;
7715	int cmp;
7716	struct btrfs_path *left_path = NULL;
7717	struct btrfs_path *right_path = NULL;
7718	struct btrfs_key left_key;
7719	struct btrfs_key right_key;
7720	char *tmp_buf = NULL;
7721	int left_root_level;
7722	int right_root_level;
7723	int left_level;
7724	int right_level;
7725	int left_end_reached = 0;
7726	int right_end_reached = 0;
7727	int advance_left = 0;
7728	int advance_right = 0;
7729	u64 left_blockptr;
7730	u64 right_blockptr;
7731	u64 left_gen;
7732	u64 right_gen;
7733	u64 reada_min_gen;
7734
7735	left_path = btrfs_alloc_path();
7736	if (!left_path) {
7737		ret = -ENOMEM;
7738		goto out;
7739	}
7740	right_path = btrfs_alloc_path();
7741	if (!right_path) {
7742		ret = -ENOMEM;
7743		goto out;
7744	}
7745
7746	tmp_buf = kvmalloc(fs_info->nodesize, GFP_KERNEL);
7747	if (!tmp_buf) {
7748		ret = -ENOMEM;
7749		goto out;
7750	}
7751
7752	left_path->search_commit_root = 1;
7753	left_path->skip_locking = 1;
7754	right_path->search_commit_root = 1;
7755	right_path->skip_locking = 1;
7756
7757	/*
7758	 * Strategy: Go to the first items of both trees. Then do
7759	 *
7760	 * If both trees are at level 0
7761	 *   Compare keys of current items
7762	 *     If left < right treat left item as new, advance left tree
7763	 *       and repeat
7764	 *     If left > right treat right item as deleted, advance right tree
7765	 *       and repeat
7766	 *     If left == right do deep compare of items, treat as changed if
7767	 *       needed, advance both trees and repeat
7768	 * If both trees are at the same level but not at level 0
7769	 *   Compare keys of current nodes/leafs
7770	 *     If left < right advance left tree and repeat
7771	 *     If left > right advance right tree and repeat
7772	 *     If left == right compare blockptrs of the next nodes/leafs
7773	 *       If they match advance both trees but stay at the same level
7774	 *         and repeat
7775	 *       If they don't match advance both trees while allowing to go
7776	 *         deeper and repeat
7777	 * If tree levels are different
7778	 *   Advance the tree that needs it and repeat
7779	 *
7780	 * Advancing a tree means:
7781	 *   If we are at level 0, try to go to the next slot. If that's not
7782	 *   possible, go one level up and repeat. Stop when we found a level
7783	 *   where we could go to the next slot. We may at this point be on a
7784	 *   node or a leaf.
7785	 *
7786	 *   If we are not at level 0 and not on shared tree blocks, go one
7787	 *   level deeper.
7788	 *
7789	 *   If we are not at level 0 and on shared tree blocks, go one slot to
7790	 *   the right if possible or go up and right.
7791	 */
7792
7793	down_read(&fs_info->commit_root_sem);
7794	left_level = btrfs_header_level(left_root->commit_root);
7795	left_root_level = left_level;
7796	/*
7797	 * We clone the root node of the send and parent roots to prevent races
7798	 * with snapshot creation of these roots. Snapshot creation COWs the
7799	 * root node of a tree, so after the transaction is committed the old
7800	 * extent can be reallocated while this send operation is still ongoing.
7801	 * So we clone them, under the commit root semaphore, to be race free.
7802	 */
7803	left_path->nodes[left_level] =
7804			btrfs_clone_extent_buffer(left_root->commit_root);
7805	if (!left_path->nodes[left_level]) {
 
7806		ret = -ENOMEM;
7807		goto out_unlock;
7808	}
7809
7810	right_level = btrfs_header_level(right_root->commit_root);
7811	right_root_level = right_level;
7812	right_path->nodes[right_level] =
7813			btrfs_clone_extent_buffer(right_root->commit_root);
7814	if (!right_path->nodes[right_level]) {
 
7815		ret = -ENOMEM;
7816		goto out_unlock;
7817	}
7818	/*
7819	 * Our right root is the parent root, while the left root is the "send"
7820	 * root. We know that all new nodes/leaves in the left root must have
7821	 * a generation greater than the right root's generation, so we trigger
7822	 * readahead for those nodes and leaves of the left root, as we know we
7823	 * will need to read them at some point.
7824	 */
7825	reada_min_gen = btrfs_header_generation(right_root->commit_root);
7826
7827	if (left_level == 0)
7828		btrfs_item_key_to_cpu(left_path->nodes[left_level],
7829				&left_key, left_path->slots[left_level]);
7830	else
7831		btrfs_node_key_to_cpu(left_path->nodes[left_level],
7832				&left_key, left_path->slots[left_level]);
7833	if (right_level == 0)
7834		btrfs_item_key_to_cpu(right_path->nodes[right_level],
7835				&right_key, right_path->slots[right_level]);
7836	else
7837		btrfs_node_key_to_cpu(right_path->nodes[right_level],
7838				&right_key, right_path->slots[right_level]);
7839
7840	sctx->last_reloc_trans = fs_info->last_reloc_trans;
 
7841
7842	while (1) {
7843		if (need_resched() ||
7844		    rwsem_is_contended(&fs_info->commit_root_sem)) {
7845			up_read(&fs_info->commit_root_sem);
7846			cond_resched();
7847			down_read(&fs_info->commit_root_sem);
7848		}
7849
7850		if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
7851			ret = restart_after_relocation(left_path, right_path,
7852						       &left_key, &right_key,
7853						       left_level, right_level,
7854						       sctx);
7855			if (ret < 0)
7856				goto out_unlock;
7857			sctx->last_reloc_trans = fs_info->last_reloc_trans;
7858		}
7859
7860		if (advance_left && !left_end_reached) {
7861			ret = tree_advance(left_path, &left_level,
7862					left_root_level,
7863					advance_left != ADVANCE_ONLY_NEXT,
7864					&left_key, reada_min_gen);
7865			if (ret == -1)
7866				left_end_reached = ADVANCE;
7867			else if (ret < 0)
7868				goto out_unlock;
7869			advance_left = 0;
7870		}
7871		if (advance_right && !right_end_reached) {
7872			ret = tree_advance(right_path, &right_level,
7873					right_root_level,
7874					advance_right != ADVANCE_ONLY_NEXT,
7875					&right_key, reada_min_gen);
7876			if (ret == -1)
7877				right_end_reached = ADVANCE;
7878			else if (ret < 0)
7879				goto out_unlock;
7880			advance_right = 0;
7881		}
7882
7883		if (left_end_reached && right_end_reached) {
7884			ret = 0;
7885			goto out_unlock;
7886		} else if (left_end_reached) {
7887			if (right_level == 0) {
7888				up_read(&fs_info->commit_root_sem);
7889				ret = changed_cb(left_path, right_path,
7890						&right_key,
7891						BTRFS_COMPARE_TREE_DELETED,
7892						sctx);
7893				if (ret < 0)
7894					goto out;
7895				down_read(&fs_info->commit_root_sem);
7896			}
7897			advance_right = ADVANCE;
7898			continue;
7899		} else if (right_end_reached) {
7900			if (left_level == 0) {
7901				up_read(&fs_info->commit_root_sem);
7902				ret = changed_cb(left_path, right_path,
7903						&left_key,
7904						BTRFS_COMPARE_TREE_NEW,
7905						sctx);
7906				if (ret < 0)
7907					goto out;
7908				down_read(&fs_info->commit_root_sem);
7909			}
7910			advance_left = ADVANCE;
7911			continue;
7912		}
7913
7914		if (left_level == 0 && right_level == 0) {
7915			up_read(&fs_info->commit_root_sem);
7916			cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
7917			if (cmp < 0) {
7918				ret = changed_cb(left_path, right_path,
7919						&left_key,
7920						BTRFS_COMPARE_TREE_NEW,
7921						sctx);
 
 
7922				advance_left = ADVANCE;
7923			} else if (cmp > 0) {
7924				ret = changed_cb(left_path, right_path,
7925						&right_key,
7926						BTRFS_COMPARE_TREE_DELETED,
7927						sctx);
 
 
7928				advance_right = ADVANCE;
7929			} else {
7930				enum btrfs_compare_tree_result result;
7931
7932				WARN_ON(!extent_buffer_uptodate(left_path->nodes[0]));
7933				ret = tree_compare_item(left_path, right_path,
7934							tmp_buf);
7935				if (ret)
7936					result = BTRFS_COMPARE_TREE_CHANGED;
7937				else
7938					result = BTRFS_COMPARE_TREE_SAME;
7939				ret = changed_cb(left_path, right_path,
7940						 &left_key, result, sctx);
 
 
7941				advance_left = ADVANCE;
7942				advance_right = ADVANCE;
7943			}
7944
7945			if (ret < 0)
7946				goto out;
7947			down_read(&fs_info->commit_root_sem);
7948		} else if (left_level == right_level) {
7949			cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
7950			if (cmp < 0) {
7951				advance_left = ADVANCE;
7952			} else if (cmp > 0) {
7953				advance_right = ADVANCE;
7954			} else {
7955				left_blockptr = btrfs_node_blockptr(
7956						left_path->nodes[left_level],
7957						left_path->slots[left_level]);
7958				right_blockptr = btrfs_node_blockptr(
7959						right_path->nodes[right_level],
7960						right_path->slots[right_level]);
7961				left_gen = btrfs_node_ptr_generation(
7962						left_path->nodes[left_level],
7963						left_path->slots[left_level]);
7964				right_gen = btrfs_node_ptr_generation(
7965						right_path->nodes[right_level],
7966						right_path->slots[right_level]);
7967				if (left_blockptr == right_blockptr &&
7968				    left_gen == right_gen) {
7969					/*
7970					 * As we're on a shared block, don't
7971					 * allow to go deeper.
7972					 */
7973					advance_left = ADVANCE_ONLY_NEXT;
7974					advance_right = ADVANCE_ONLY_NEXT;
7975				} else {
7976					advance_left = ADVANCE;
7977					advance_right = ADVANCE;
7978				}
7979			}
7980		} else if (left_level < right_level) {
7981			advance_right = ADVANCE;
7982		} else {
7983			advance_left = ADVANCE;
7984		}
7985	}
7986
7987out_unlock:
7988	up_read(&fs_info->commit_root_sem);
7989out:
7990	btrfs_free_path(left_path);
7991	btrfs_free_path(right_path);
7992	kvfree(tmp_buf);
7993	return ret;
7994}
7995
7996static int send_subvol(struct send_ctx *sctx)
7997{
7998	int ret;
7999
8000	if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_STREAM_HEADER)) {
8001		ret = send_header(sctx);
8002		if (ret < 0)
8003			goto out;
8004	}
8005
8006	ret = send_subvol_begin(sctx);
8007	if (ret < 0)
8008		goto out;
8009
8010	if (sctx->parent_root) {
8011		ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root, sctx);
 
8012		if (ret < 0)
8013			goto out;
8014		ret = finish_inode_if_needed(sctx, 1);
8015		if (ret < 0)
8016			goto out;
8017	} else {
8018		ret = full_send_tree(sctx);
8019		if (ret < 0)
8020			goto out;
8021	}
8022
8023out:
8024	free_recorded_refs(sctx);
8025	return ret;
8026}
8027
8028/*
8029 * If orphan cleanup did remove any orphans from a root, it means the tree
8030 * was modified and therefore the commit root is not the same as the current
8031 * root anymore. This is a problem, because send uses the commit root and
8032 * therefore can see inode items that don't exist in the current root anymore,
8033 * and for example make calls to btrfs_iget, which will do tree lookups based
8034 * on the current root and not on the commit root. Those lookups will fail,
8035 * returning a -ESTALE error, and making send fail with that error. So make
8036 * sure a send does not see any orphans we have just removed, and that it will
8037 * see the same inodes regardless of whether a transaction commit happened
8038 * before it started (meaning that the commit root will be the same as the
8039 * current root) or not.
8040 */
8041static int ensure_commit_roots_uptodate(struct send_ctx *sctx)
8042{
8043	struct btrfs_root *root = sctx->parent_root;
 
8044
8045	if (root && root->node != root->commit_root)
8046		return btrfs_commit_current_transaction(root);
 
 
 
 
 
 
 
8047
8048	for (int i = 0; i < sctx->clone_roots_cnt; i++) {
8049		root = sctx->clone_roots[i].root;
8050		if (root->node != root->commit_root)
8051			return btrfs_commit_current_transaction(root);
8052	}
8053
8054	return 0;
 
 
 
 
 
 
 
 
 
 
 
8055}
8056
8057/*
8058 * Make sure any existing dellaloc is flushed for any root used by a send
8059 * operation so that we do not miss any data and we do not race with writeback
8060 * finishing and changing a tree while send is using the tree. This could
8061 * happen if a subvolume is in RW mode, has delalloc, is turned to RO mode and
8062 * a send operation then uses the subvolume.
8063 * After flushing delalloc ensure_commit_roots_uptodate() must be called.
8064 */
8065static int flush_delalloc_roots(struct send_ctx *sctx)
8066{
8067	struct btrfs_root *root = sctx->parent_root;
8068	int ret;
8069	int i;
8070
8071	if (root) {
8072		ret = btrfs_start_delalloc_snapshot(root, false);
8073		if (ret)
8074			return ret;
8075		btrfs_wait_ordered_extents(root, U64_MAX, NULL);
8076	}
8077
8078	for (i = 0; i < sctx->clone_roots_cnt; i++) {
8079		root = sctx->clone_roots[i].root;
8080		ret = btrfs_start_delalloc_snapshot(root, false);
8081		if (ret)
8082			return ret;
8083		btrfs_wait_ordered_extents(root, U64_MAX, NULL);
8084	}
8085
8086	return 0;
8087}
8088
8089static void btrfs_root_dec_send_in_progress(struct btrfs_root* root)
8090{
8091	spin_lock(&root->root_item_lock);
8092	root->send_in_progress--;
8093	/*
8094	 * Not much left to do, we don't know why it's unbalanced and
8095	 * can't blindly reset it to 0.
8096	 */
8097	if (root->send_in_progress < 0)
8098		btrfs_err(root->fs_info,
8099			  "send_in_progress unbalanced %d root %llu",
8100			  root->send_in_progress, btrfs_root_id(root));
8101	spin_unlock(&root->root_item_lock);
8102}
8103
8104static void dedupe_in_progress_warn(const struct btrfs_root *root)
8105{
8106	btrfs_warn_rl(root->fs_info,
8107"cannot use root %llu for send while deduplications on it are in progress (%d in progress)",
8108		      btrfs_root_id(root), root->dedupe_in_progress);
8109}
8110
8111long btrfs_ioctl_send(struct btrfs_inode *inode, const struct btrfs_ioctl_send_args *arg)
8112{
8113	int ret = 0;
8114	struct btrfs_root *send_root = inode->root;
8115	struct btrfs_fs_info *fs_info = send_root->fs_info;
8116	struct btrfs_root *clone_root;
 
8117	struct send_ctx *sctx = NULL;
8118	u32 i;
8119	u64 *clone_sources_tmp = NULL;
8120	int clone_sources_to_rollback = 0;
8121	size_t alloc_size;
8122	int sort_clone_roots = 0;
8123	struct btrfs_lru_cache_entry *entry;
8124	struct btrfs_lru_cache_entry *tmp;
8125
8126	if (!capable(CAP_SYS_ADMIN))
8127		return -EPERM;
8128
8129	/*
8130	 * The subvolume must remain read-only during send, protect against
8131	 * making it RW. This also protects against deletion.
8132	 */
8133	spin_lock(&send_root->root_item_lock);
8134	/*
8135	 * Unlikely but possible, if the subvolume is marked for deletion but
8136	 * is slow to remove the directory entry, send can still be started.
8137	 */
8138	if (btrfs_root_dead(send_root)) {
8139		spin_unlock(&send_root->root_item_lock);
8140		return -EPERM;
8141	}
8142	/* Userspace tools do the checks and warn the user if it's not RO. */
8143	if (!btrfs_root_readonly(send_root)) {
8144		spin_unlock(&send_root->root_item_lock);
8145		return -EPERM;
8146	}
8147	if (send_root->dedupe_in_progress) {
8148		dedupe_in_progress_warn(send_root);
8149		spin_unlock(&send_root->root_item_lock);
8150		return -EAGAIN;
8151	}
8152	send_root->send_in_progress++;
8153	spin_unlock(&send_root->root_item_lock);
8154
8155	/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8156	 * Check that we don't overflow at later allocations, we request
8157	 * clone_sources_count + 1 items, and compare to unsigned long inside
8158	 * access_ok. Also set an upper limit for allocation size so this can't
8159	 * easily exhaust memory. Max number of clone sources is about 200K.
8160	 */
8161	if (arg->clone_sources_count > SZ_8M / sizeof(struct clone_root)) {
 
8162		ret = -EINVAL;
8163		goto out;
8164	}
8165
 
 
 
 
 
 
 
8166	if (arg->flags & ~BTRFS_SEND_FLAG_MASK) {
8167		ret = -EOPNOTSUPP;
8168		goto out;
8169	}
8170
8171	sctx = kzalloc(sizeof(struct send_ctx), GFP_KERNEL);
8172	if (!sctx) {
8173		ret = -ENOMEM;
8174		goto out;
8175	}
8176
8177	INIT_LIST_HEAD(&sctx->new_refs);
8178	INIT_LIST_HEAD(&sctx->deleted_refs);
8179
8180	btrfs_lru_cache_init(&sctx->name_cache, SEND_MAX_NAME_CACHE_SIZE);
8181	btrfs_lru_cache_init(&sctx->backref_cache, SEND_MAX_BACKREF_CACHE_SIZE);
8182	btrfs_lru_cache_init(&sctx->dir_created_cache,
8183			     SEND_MAX_DIR_CREATED_CACHE_SIZE);
8184	/*
8185	 * This cache is periodically trimmed to a fixed size elsewhere, see
8186	 * cache_dir_utimes() and trim_dir_utimes_cache().
8187	 */
8188	btrfs_lru_cache_init(&sctx->dir_utimes_cache, 0);
8189
8190	sctx->pending_dir_moves = RB_ROOT;
8191	sctx->waiting_dir_moves = RB_ROOT;
8192	sctx->orphan_dirs = RB_ROOT;
8193	sctx->rbtree_new_refs = RB_ROOT;
8194	sctx->rbtree_deleted_refs = RB_ROOT;
8195
8196	sctx->flags = arg->flags;
8197
8198	if (arg->flags & BTRFS_SEND_FLAG_VERSION) {
8199		if (arg->version > BTRFS_SEND_STREAM_VERSION) {
8200			ret = -EPROTO;
8201			goto out;
8202		}
8203		/* Zero means "use the highest version" */
8204		sctx->proto = arg->version ?: BTRFS_SEND_STREAM_VERSION;
8205	} else {
8206		sctx->proto = 1;
8207	}
8208	if ((arg->flags & BTRFS_SEND_FLAG_COMPRESSED) && sctx->proto < 2) {
8209		ret = -EINVAL;
8210		goto out;
8211	}
8212
8213	sctx->send_filp = fget(arg->send_fd);
8214	if (!sctx->send_filp || !(sctx->send_filp->f_mode & FMODE_WRITE)) {
8215		ret = -EBADF;
8216		goto out;
8217	}
8218
8219	sctx->send_root = send_root;
8220	sctx->clone_roots_cnt = arg->clone_sources_count;
 
 
 
 
 
 
 
8221
8222	if (sctx->proto >= 2) {
8223		u32 send_buf_num_pages;
8224
8225		sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V2;
8226		sctx->send_buf = vmalloc(sctx->send_max_size);
8227		if (!sctx->send_buf) {
8228			ret = -ENOMEM;
8229			goto out;
8230		}
8231		send_buf_num_pages = sctx->send_max_size >> PAGE_SHIFT;
8232		sctx->send_buf_pages = kcalloc(send_buf_num_pages,
8233					       sizeof(*sctx->send_buf_pages),
8234					       GFP_KERNEL);
8235		if (!sctx->send_buf_pages) {
8236			ret = -ENOMEM;
8237			goto out;
8238		}
8239		for (i = 0; i < send_buf_num_pages; i++) {
8240			sctx->send_buf_pages[i] =
8241				vmalloc_to_page(sctx->send_buf + (i << PAGE_SHIFT));
8242		}
8243	} else {
8244		sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V1;
8245		sctx->send_buf = kvmalloc(sctx->send_max_size, GFP_KERNEL);
8246	}
8247	if (!sctx->send_buf) {
8248		ret = -ENOMEM;
8249		goto out;
8250	}
8251
8252	sctx->clone_roots = kvcalloc(arg->clone_sources_count + 1,
8253				     sizeof(*sctx->clone_roots),
8254				     GFP_KERNEL);
 
 
 
 
 
 
 
 
 
 
8255	if (!sctx->clone_roots) {
8256		ret = -ENOMEM;
8257		goto out;
8258	}
8259
8260	alloc_size = array_size(sizeof(*arg->clone_sources),
8261				arg->clone_sources_count);
8262
8263	if (arg->clone_sources_count) {
8264		clone_sources_tmp = kvmalloc(alloc_size, GFP_KERNEL);
8265		if (!clone_sources_tmp) {
8266			ret = -ENOMEM;
8267			goto out;
8268		}
8269
8270		ret = copy_from_user(clone_sources_tmp, arg->clone_sources,
8271				alloc_size);
8272		if (ret) {
8273			ret = -EFAULT;
8274			goto out;
8275		}
8276
8277		for (i = 0; i < arg->clone_sources_count; i++) {
8278			clone_root = btrfs_get_fs_root(fs_info,
8279						clone_sources_tmp[i], true);
 
 
 
 
 
8280			if (IS_ERR(clone_root)) {
 
8281				ret = PTR_ERR(clone_root);
8282				goto out;
8283			}
8284			spin_lock(&clone_root->root_item_lock);
8285			if (!btrfs_root_readonly(clone_root) ||
8286			    btrfs_root_dead(clone_root)) {
8287				spin_unlock(&clone_root->root_item_lock);
8288				btrfs_put_root(clone_root);
8289				ret = -EPERM;
8290				goto out;
8291			}
8292			if (clone_root->dedupe_in_progress) {
8293				dedupe_in_progress_warn(clone_root);
8294				spin_unlock(&clone_root->root_item_lock);
8295				btrfs_put_root(clone_root);
8296				ret = -EAGAIN;
8297				goto out;
8298			}
8299			clone_root->send_in_progress++;
8300			spin_unlock(&clone_root->root_item_lock);
 
8301
8302			sctx->clone_roots[i].root = clone_root;
8303			clone_sources_to_rollback = i + 1;
8304		}
8305		kvfree(clone_sources_tmp);
8306		clone_sources_tmp = NULL;
8307	}
8308
8309	if (arg->parent_root) {
8310		sctx->parent_root = btrfs_get_fs_root(fs_info, arg->parent_root,
8311						      true);
 
 
 
 
 
8312		if (IS_ERR(sctx->parent_root)) {
 
8313			ret = PTR_ERR(sctx->parent_root);
8314			goto out;
8315		}
8316
8317		spin_lock(&sctx->parent_root->root_item_lock);
8318		sctx->parent_root->send_in_progress++;
8319		if (!btrfs_root_readonly(sctx->parent_root) ||
8320				btrfs_root_dead(sctx->parent_root)) {
8321			spin_unlock(&sctx->parent_root->root_item_lock);
 
8322			ret = -EPERM;
8323			goto out;
8324		}
8325		if (sctx->parent_root->dedupe_in_progress) {
8326			dedupe_in_progress_warn(sctx->parent_root);
8327			spin_unlock(&sctx->parent_root->root_item_lock);
 
8328			ret = -EAGAIN;
8329			goto out;
8330		}
8331		spin_unlock(&sctx->parent_root->root_item_lock);
 
 
8332	}
8333
8334	/*
8335	 * Clones from send_root are allowed, but only if the clone source
8336	 * is behind the current send position. This is checked while searching
8337	 * for possible clone sources.
8338	 */
8339	sctx->clone_roots[sctx->clone_roots_cnt++].root =
8340		btrfs_grab_root(sctx->send_root);
8341
8342	/* We do a bsearch later */
8343	sort(sctx->clone_roots, sctx->clone_roots_cnt,
8344			sizeof(*sctx->clone_roots), __clone_root_cmp_sort,
8345			NULL);
8346	sort_clone_roots = 1;
8347
8348	ret = flush_delalloc_roots(sctx);
8349	if (ret)
8350		goto out;
8351
8352	ret = ensure_commit_roots_uptodate(sctx);
8353	if (ret)
8354		goto out;
8355
 
 
 
 
 
 
 
 
 
 
 
 
8356	ret = send_subvol(sctx);
 
 
 
 
8357	if (ret < 0)
8358		goto out;
8359
8360	btrfs_lru_cache_for_each_entry_safe(&sctx->dir_utimes_cache, entry, tmp) {
8361		ret = send_utimes(sctx, entry->key, entry->gen);
8362		if (ret < 0)
8363			goto out;
8364		btrfs_lru_cache_remove(&sctx->dir_utimes_cache, entry);
8365	}
8366
8367	if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_END_CMD)) {
8368		ret = begin_cmd(sctx, BTRFS_SEND_C_END);
8369		if (ret < 0)
8370			goto out;
8371		ret = send_cmd(sctx);
8372		if (ret < 0)
8373			goto out;
8374	}
8375
8376out:
8377	WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->pending_dir_moves));
8378	while (sctx && !RB_EMPTY_ROOT(&sctx->pending_dir_moves)) {
8379		struct rb_node *n;
8380		struct pending_dir_move *pm;
8381
8382		n = rb_first(&sctx->pending_dir_moves);
8383		pm = rb_entry(n, struct pending_dir_move, node);
8384		while (!list_empty(&pm->list)) {
8385			struct pending_dir_move *pm2;
8386
8387			pm2 = list_first_entry(&pm->list,
8388					       struct pending_dir_move, list);
8389			free_pending_move(sctx, pm2);
8390		}
8391		free_pending_move(sctx, pm);
8392	}
8393
8394	WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves));
8395	while (sctx && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves)) {
8396		struct rb_node *n;
8397		struct waiting_dir_move *dm;
8398
8399		n = rb_first(&sctx->waiting_dir_moves);
8400		dm = rb_entry(n, struct waiting_dir_move, node);
8401		rb_erase(&dm->node, &sctx->waiting_dir_moves);
8402		kfree(dm);
8403	}
8404
8405	WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->orphan_dirs));
8406	while (sctx && !RB_EMPTY_ROOT(&sctx->orphan_dirs)) {
8407		struct rb_node *n;
8408		struct orphan_dir_info *odi;
8409
8410		n = rb_first(&sctx->orphan_dirs);
8411		odi = rb_entry(n, struct orphan_dir_info, node);
8412		free_orphan_dir_info(sctx, odi);
8413	}
8414
8415	if (sort_clone_roots) {
8416		for (i = 0; i < sctx->clone_roots_cnt; i++) {
8417			btrfs_root_dec_send_in_progress(
8418					sctx->clone_roots[i].root);
8419			btrfs_put_root(sctx->clone_roots[i].root);
8420		}
8421	} else {
8422		for (i = 0; sctx && i < clone_sources_to_rollback; i++) {
8423			btrfs_root_dec_send_in_progress(
8424					sctx->clone_roots[i].root);
8425			btrfs_put_root(sctx->clone_roots[i].root);
8426		}
8427
8428		btrfs_root_dec_send_in_progress(send_root);
8429	}
8430	if (sctx && !IS_ERR_OR_NULL(sctx->parent_root)) {
8431		btrfs_root_dec_send_in_progress(sctx->parent_root);
8432		btrfs_put_root(sctx->parent_root);
8433	}
8434
8435	kvfree(clone_sources_tmp);
8436
8437	if (sctx) {
8438		if (sctx->send_filp)
8439			fput(sctx->send_filp);
8440
8441		kvfree(sctx->clone_roots);
8442		kfree(sctx->send_buf_pages);
8443		kvfree(sctx->send_buf);
8444		kvfree(sctx->verity_descriptor);
8445
8446		close_current_inode(sctx);
8447
8448		btrfs_lru_cache_clear(&sctx->name_cache);
8449		btrfs_lru_cache_clear(&sctx->backref_cache);
8450		btrfs_lru_cache_clear(&sctx->dir_created_cache);
8451		btrfs_lru_cache_clear(&sctx->dir_utimes_cache);
8452
8453		kfree(sctx);
8454	}
8455
8456	return ret;
8457}