1// SPDX-License-Identifier: GPL-2.0
   2
   3/*
   4 * fs/ext4/fast_commit.c
   5 *
   6 * Written by Harshad Shirwadkar <harshadshirwadkar@gmail.com>
   7 *
   8 * Ext4 fast commits routines.
   9 */
  10#include "ext4.h"
  11#include "ext4_jbd2.h"
  12#include "ext4_extents.h"
  13#include "mballoc.h"
  14
  15/*
  16 * Ext4 Fast Commits
  17 * -----------------
  18 *
  19 * Ext4 fast commits implement fine grained journalling for Ext4.
  20 *
  21 * Fast commits are organized as a log of tag-length-value (TLV) structs. (See
  22 * struct ext4_fc_tl). Each TLV contains some delta that is replayed TLV by
  23 * TLV during the recovery phase. For the scenarios for which we currently
  24 * don't have replay code, fast commit falls back to full commits.
  25 * Fast commits record delta in one of the following three categories.
  26 *
  27 * (A) Directory entry updates:
  28 *
  29 * - EXT4_FC_TAG_UNLINK		- records directory entry unlink
  30 * - EXT4_FC_TAG_LINK		- records directory entry link
  31 * - EXT4_FC_TAG_CREAT		- records inode and directory entry creation
  32 *
  33 * (B) File specific data range updates:
  34 *
  35 * - EXT4_FC_TAG_ADD_RANGE	- records addition of new blocks to an inode
  36 * - EXT4_FC_TAG_DEL_RANGE	- records deletion of blocks from an inode
  37 *
  38 * (C) Inode metadata (mtime / ctime etc):
  39 *
  40 * - EXT4_FC_TAG_INODE		- record the inode that should be replayed
  41 *				  during recovery. Note that iblocks field is
  42 *				  not replayed and instead derived during
  43 *				  replay.
  44 * Commit Operation
  45 * ----------------
  46 * With fast commits, we maintain all the directory entry operations in the
  47 * order in which they are issued in an in-memory queue. This queue is flushed
  48 * to disk during the commit operation. We also maintain a list of inodes
  49 * that need to be committed during a fast commit in another in memory queue of
  50 * inodes. During the commit operation, we commit in the following order:
  51 *
  52 * [1] Lock inodes for any further data updates by setting COMMITTING state
  53 * [2] Submit data buffers of all the inodes
  54 * [3] Wait for [2] to complete
  55 * [4] Commit all the directory entry updates in the fast commit space
  56 * [5] Commit all the changed inode structures
  57 * [6] Write tail tag (this tag ensures the atomicity, please read the following
  58 *     section for more details).
  59 * [7] Wait for [4], [5] and [6] to complete.
  60 *
  61 * All the inode updates must call ext4_fc_start_update() before starting an
  62 * update. If such an ongoing update is present, fast commit waits for it to
  63 * complete. The completion of such an update is marked by
  64 * ext4_fc_stop_update().
  65 *
  66 * Fast Commit Ineligibility
  67 * -------------------------
  68 *
  69 * Not all operations are supported by fast commits today (e.g extended
  70 * attributes). Fast commit ineligibility is marked by calling
  71 * ext4_fc_mark_ineligible(): This makes next fast commit operation to fall back
  72 * to full commit.
 
 
 
 
 
 
 
 
 
 
 
  73 *
  74 * Atomicity of commits
  75 * --------------------
  76 * In order to guarantee atomicity during the commit operation, fast commit
  77 * uses "EXT4_FC_TAG_TAIL" tag that marks a fast commit as complete. Tail
  78 * tag contains CRC of the contents and TID of the transaction after which
  79 * this fast commit should be applied. Recovery code replays fast commit
  80 * logs only if there's at least 1 valid tail present. For every fast commit
  81 * operation, there is 1 tail. This means, we may end up with multiple tails
  82 * in the fast commit space. Here's an example:
  83 *
  84 * - Create a new file A and remove existing file B
  85 * - fsync()
  86 * - Append contents to file A
  87 * - Truncate file A
  88 * - fsync()
  89 *
  90 * The fast commit space at the end of above operations would look like this:
  91 *      [HEAD] [CREAT A] [UNLINK B] [TAIL] [ADD_RANGE A] [DEL_RANGE A] [TAIL]
  92 *             |<---  Fast Commit 1   --->|<---      Fast Commit 2     ---->|
  93 *
  94 * Replay code should thus check for all the valid tails in the FC area.
  95 *
  96 * Fast Commit Replay Idempotence
  97 * ------------------------------
  98 *
  99 * Fast commits tags are idempotent in nature provided the recovery code follows
 100 * certain rules. The guiding principle that the commit path follows while
 101 * committing is that it stores the result of a particular operation instead of
 102 * storing the procedure.
 103 *
 104 * Let's consider this rename operation: 'mv /a /b'. Let's assume dirent '/a'
 105 * was associated with inode 10. During fast commit, instead of storing this
 106 * operation as a procedure "rename a to b", we store the resulting file system
 107 * state as a "series" of outcomes:
 108 *
 109 * - Link dirent b to inode 10
 110 * - Unlink dirent a
 111 * - Inode <10> with valid refcount
 112 *
 113 * Now when recovery code runs, it needs "enforce" this state on the file
 114 * system. This is what guarantees idempotence of fast commit replay.
 115 *
 116 * Let's take an example of a procedure that is not idempotent and see how fast
 117 * commits make it idempotent. Consider following sequence of operations:
 118 *
 119 *     rm A;    mv B A;    read A
 120 *  (x)     (y)        (z)
 121 *
 122 * (x), (y) and (z) are the points at which we can crash. If we store this
 123 * sequence of operations as is then the replay is not idempotent. Let's say
 124 * while in replay, we crash at (z). During the second replay, file A (which was
 125 * actually created as a result of "mv B A" operation) would get deleted. Thus,
 126 * file named A would be absent when we try to read A. So, this sequence of
 127 * operations is not idempotent. However, as mentioned above, instead of storing
 128 * the procedure fast commits store the outcome of each procedure. Thus the fast
 129 * commit log for above procedure would be as follows:
 130 *
 131 * (Let's assume dirent A was linked to inode 10 and dirent B was linked to
 132 * inode 11 before the replay)
 133 *
 134 *    [Unlink A]   [Link A to inode 11]   [Unlink B]   [Inode 11]
 135 * (w)          (x)                    (y)          (z)
 136 *
 137 * If we crash at (z), we will have file A linked to inode 11. During the second
 138 * replay, we will remove file A (inode 11). But we will create it back and make
 139 * it point to inode 11. We won't find B, so we'll just skip that step. At this
 140 * point, the refcount for inode 11 is not reliable, but that gets fixed by the
 141 * replay of last inode 11 tag. Crashes at points (w), (x) and (y) get handled
 142 * similarly. Thus, by converting a non-idempotent procedure into a series of
 143 * idempotent outcomes, fast commits ensured idempotence during the replay.
 144 *
 145 * TODOs
 146 * -----
 147 *
 148 * 0) Fast commit replay path hardening: Fast commit replay code should use
 149 *    journal handles to make sure all the updates it does during the replay
 150 *    path are atomic. With that if we crash during fast commit replay, after
 151 *    trying to do recovery again, we will find a file system where fast commit
 152 *    area is invalid (because new full commit would be found). In order to deal
 153 *    with that, fast commit replay code should ensure that the "FC_REPLAY"
 154 *    superblock state is persisted before starting the replay, so that after
 155 *    the crash, fast commit recovery code can look at that flag and perform
 156 *    fast commit recovery even if that area is invalidated by later full
 157 *    commits.
 158 *
 159 * 1) Fast commit's commit path locks the entire file system during fast
 160 *    commit. This has significant performance penalty. Instead of that, we
 161 *    should use ext4_fc_start/stop_update functions to start inode level
 162 *    updates from ext4_journal_start/stop. Once we do that we can drop file
 163 *    system locking during commit path.
 
 
 164 *
 165 * 2) Handle more ineligible cases.
 166 */
 167
 168#include <trace/events/ext4.h>
 169static struct kmem_cache *ext4_fc_dentry_cachep;
 170
 171static void ext4_end_buffer_io_sync(struct buffer_head *bh, int uptodate)
 172{
 173	BUFFER_TRACE(bh, "");
 174	if (uptodate) {
 175		ext4_debug("%s: Block %lld up-to-date",
 176			   __func__, bh->b_blocknr);
 177		set_buffer_uptodate(bh);
 178	} else {
 179		ext4_debug("%s: Block %lld not up-to-date",
 180			   __func__, bh->b_blocknr);
 181		clear_buffer_uptodate(bh);
 182	}
 183
 184	unlock_buffer(bh);
 185}
 186
 187static inline void ext4_fc_reset_inode(struct inode *inode)
 188{
 189	struct ext4_inode_info *ei = EXT4_I(inode);
 190
 191	ei->i_fc_lblk_start = 0;
 192	ei->i_fc_lblk_len = 0;
 193}
 194
 195void ext4_fc_init_inode(struct inode *inode)
 196{
 197	struct ext4_inode_info *ei = EXT4_I(inode);
 198
 199	ext4_fc_reset_inode(inode);
 200	ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING);
 201	INIT_LIST_HEAD(&ei->i_fc_list);
 202	INIT_LIST_HEAD(&ei->i_fc_dilist);
 203	init_waitqueue_head(&ei->i_fc_wait);
 204	atomic_set(&ei->i_fc_updates, 0);
 205}
 206
 207/* This function must be called with sbi->s_fc_lock held. */
 208static void ext4_fc_wait_committing_inode(struct inode *inode)
 209__releases(&EXT4_SB(inode->i_sb)->s_fc_lock)
 210{
 211	wait_queue_head_t *wq;
 212	struct ext4_inode_info *ei = EXT4_I(inode);
 213
 214#if (BITS_PER_LONG < 64)
 215	DEFINE_WAIT_BIT(wait, &ei->i_state_flags,
 216			EXT4_STATE_FC_COMMITTING);
 217	wq = bit_waitqueue(&ei->i_state_flags,
 218				EXT4_STATE_FC_COMMITTING);
 219#else
 220	DEFINE_WAIT_BIT(wait, &ei->i_flags,
 221			EXT4_STATE_FC_COMMITTING);
 222	wq = bit_waitqueue(&ei->i_flags,
 223				EXT4_STATE_FC_COMMITTING);
 224#endif
 225	lockdep_assert_held(&EXT4_SB(inode->i_sb)->s_fc_lock);
 226	prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
 227	spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 228	schedule();
 229	finish_wait(wq, &wait.wq_entry);
 230}
 231
 232static bool ext4_fc_disabled(struct super_block *sb)
 233{
 234	return (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
 235		(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY));
 236}
 237
 238/*
 239 * Inform Ext4's fast about start of an inode update
 240 *
 241 * This function is called by the high level call VFS callbacks before
 242 * performing any inode update. This function blocks if there's an ongoing
 243 * fast commit on the inode in question.
 244 */
 245void ext4_fc_start_update(struct inode *inode)
 246{
 247	struct ext4_inode_info *ei = EXT4_I(inode);
 248
 249	if (ext4_fc_disabled(inode->i_sb))
 
 250		return;
 251
 252restart:
 253	spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 254	if (list_empty(&ei->i_fc_list))
 255		goto out;
 256
 257	if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {
 258		ext4_fc_wait_committing_inode(inode);
 259		goto restart;
 260	}
 261out:
 262	atomic_inc(&ei->i_fc_updates);
 263	spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 264}
 265
 266/*
 267 * Stop inode update and wake up waiting fast commits if any.
 268 */
 269void ext4_fc_stop_update(struct inode *inode)
 270{
 271	struct ext4_inode_info *ei = EXT4_I(inode);
 272
 273	if (ext4_fc_disabled(inode->i_sb))
 
 274		return;
 275
 276	if (atomic_dec_and_test(&ei->i_fc_updates))
 277		wake_up_all(&ei->i_fc_wait);
 278}
 279
 280/*
 281 * Remove inode from fast commit list. If the inode is being committed
 282 * we wait until inode commit is done.
 283 */
 284void ext4_fc_del(struct inode *inode)
 285{
 286	struct ext4_inode_info *ei = EXT4_I(inode);
 287	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
 288	struct ext4_fc_dentry_update *fc_dentry;
 289
 290	if (ext4_fc_disabled(inode->i_sb))
 
 291		return;
 292
 293restart:
 294	spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 295	if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) {
 296		spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 297		return;
 298	}
 299
 300	if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {
 301		ext4_fc_wait_committing_inode(inode);
 302		goto restart;
 303	}
 
 
 
 304
 305	if (!list_empty(&ei->i_fc_list))
 306		list_del_init(&ei->i_fc_list);
 
 
 
 
 
 307
 308	/*
 309	 * Since this inode is getting removed, let's also remove all FC
 310	 * dentry create references, since it is not needed to log it anyways.
 311	 */
 312	if (list_empty(&ei->i_fc_dilist)) {
 313		spin_unlock(&sbi->s_fc_lock);
 314		return;
 315	}
 316
 317	fc_dentry = list_first_entry(&ei->i_fc_dilist, struct ext4_fc_dentry_update, fcd_dilist);
 318	WARN_ON(fc_dentry->fcd_op != EXT4_FC_TAG_CREAT);
 319	list_del_init(&fc_dentry->fcd_list);
 320	list_del_init(&fc_dentry->fcd_dilist);
 321
 322	WARN_ON(!list_empty(&ei->i_fc_dilist));
 323	spin_unlock(&sbi->s_fc_lock);
 324
 325	if (fc_dentry->fcd_name.name &&
 326		fc_dentry->fcd_name.len > DNAME_INLINE_LEN)
 327		kfree(fc_dentry->fcd_name.name);
 328	kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry);
 329
 330	return;
 331}
 332
 333/*
 334 * Mark file system as fast commit ineligible, and record latest
 335 * ineligible transaction tid. This means until the recorded
 336 * transaction, commit operation would result in a full jbd2 commit.
 337 */
 338void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handle)
 339{
 340	struct ext4_sb_info *sbi = EXT4_SB(sb);
 341	tid_t tid;
 342
 343	if (ext4_fc_disabled(sb))
 
 344		return;
 345
 346	ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
 347	if (handle && !IS_ERR(handle))
 348		tid = handle->h_transaction->t_tid;
 349	else {
 350		read_lock(&sbi->s_journal->j_state_lock);
 351		tid = sbi->s_journal->j_running_transaction ?
 352				sbi->s_journal->j_running_transaction->t_tid : 0;
 353		read_unlock(&sbi->s_journal->j_state_lock);
 354	}
 355	spin_lock(&sbi->s_fc_lock);
 356	if (sbi->s_fc_ineligible_tid < tid)
 357		sbi->s_fc_ineligible_tid = tid;
 358	spin_unlock(&sbi->s_fc_lock);
 359	WARN_ON(reason >= EXT4_FC_REASON_MAX);
 360	sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 361}
 362
 363/*
 364 * Generic fast commit tracking function. If this is the first time this we are
 365 * called after a full commit, we initialize fast commit fields and then call
 366 * __fc_track_fn() with update = 0. If we have already been called after a full
 367 * commit, we pass update = 1. Based on that, the track function can determine
 368 * if it needs to track a field for the first time or if it needs to just
 369 * update the previously tracked value.
 370 *
 371 * If enqueue is set, this function enqueues the inode in fast commit list.
 372 */
 373static int ext4_fc_track_template(
 374	handle_t *handle, struct inode *inode,
 375	int (*__fc_track_fn)(struct inode *, void *, bool),
 376	void *args, int enqueue)
 377{
 378	bool update = false;
 379	struct ext4_inode_info *ei = EXT4_I(inode);
 380	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
 381	tid_t tid = 0;
 382	int ret;
 383
 
 
 
 
 
 
 
 384	tid = handle->h_transaction->t_tid;
 385	mutex_lock(&ei->i_fc_lock);
 386	if (tid == ei->i_sync_tid) {
 387		update = true;
 388	} else {
 389		ext4_fc_reset_inode(inode);
 390		ei->i_sync_tid = tid;
 391	}
 392	ret = __fc_track_fn(inode, args, update);
 393	mutex_unlock(&ei->i_fc_lock);
 394
 395	if (!enqueue)
 396		return ret;
 397
 398	spin_lock(&sbi->s_fc_lock);
 399	if (list_empty(&EXT4_I(inode)->i_fc_list))
 400		list_add_tail(&EXT4_I(inode)->i_fc_list,
 401				(sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||
 402				 sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ?
 403				&sbi->s_fc_q[FC_Q_STAGING] :
 404				&sbi->s_fc_q[FC_Q_MAIN]);
 405	spin_unlock(&sbi->s_fc_lock);
 406
 407	return ret;
 408}
 409
 410struct __track_dentry_update_args {
 411	struct dentry *dentry;
 412	int op;
 413};
 414
 415/* __track_fn for directory entry updates. Called with ei->i_fc_lock. */
 416static int __track_dentry_update(struct inode *inode, void *arg, bool update)
 417{
 418	struct ext4_fc_dentry_update *node;
 419	struct ext4_inode_info *ei = EXT4_I(inode);
 420	struct __track_dentry_update_args *dentry_update =
 421		(struct __track_dentry_update_args *)arg;
 422	struct dentry *dentry = dentry_update->dentry;
 423	struct inode *dir = dentry->d_parent->d_inode;
 424	struct super_block *sb = inode->i_sb;
 425	struct ext4_sb_info *sbi = EXT4_SB(sb);
 426
 427	mutex_unlock(&ei->i_fc_lock);
 428
 429	if (IS_ENCRYPTED(dir)) {
 430		ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_ENCRYPTED_FILENAME,
 431					NULL);
 432		mutex_lock(&ei->i_fc_lock);
 433		return -EOPNOTSUPP;
 434	}
 435
 436	node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS);
 437	if (!node) {
 438		ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM, NULL);
 439		mutex_lock(&ei->i_fc_lock);
 440		return -ENOMEM;
 441	}
 442
 443	node->fcd_op = dentry_update->op;
 444	node->fcd_parent = dir->i_ino;
 445	node->fcd_ino = inode->i_ino;
 446	if (dentry->d_name.len > DNAME_INLINE_LEN) {
 447		node->fcd_name.name = kmalloc(dentry->d_name.len, GFP_NOFS);
 448		if (!node->fcd_name.name) {
 449			kmem_cache_free(ext4_fc_dentry_cachep, node);
 450			ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM, NULL);
 
 451			mutex_lock(&ei->i_fc_lock);
 452			return -ENOMEM;
 453		}
 454		memcpy((u8 *)node->fcd_name.name, dentry->d_name.name,
 455			dentry->d_name.len);
 456	} else {
 457		memcpy(node->fcd_iname, dentry->d_name.name,
 458			dentry->d_name.len);
 459		node->fcd_name.name = node->fcd_iname;
 460	}
 461	node->fcd_name.len = dentry->d_name.len;
 462	INIT_LIST_HEAD(&node->fcd_dilist);
 463	spin_lock(&sbi->s_fc_lock);
 464	if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||
 465		sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING)
 466		list_add_tail(&node->fcd_list,
 467				&sbi->s_fc_dentry_q[FC_Q_STAGING]);
 468	else
 469		list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]);
 470
 471	/*
 472	 * This helps us keep a track of all fc_dentry updates which is part of
 473	 * this ext4 inode. So in case the inode is getting unlinked, before
 474	 * even we get a chance to fsync, we could remove all fc_dentry
 475	 * references while evicting the inode in ext4_fc_del().
 476	 * Also with this, we don't need to loop over all the inodes in
 477	 * sbi->s_fc_q to get the corresponding inode in
 478	 * ext4_fc_commit_dentry_updates().
 479	 */
 480	if (dentry_update->op == EXT4_FC_TAG_CREAT) {
 481		WARN_ON(!list_empty(&ei->i_fc_dilist));
 482		list_add_tail(&node->fcd_dilist, &ei->i_fc_dilist);
 483	}
 484	spin_unlock(&sbi->s_fc_lock);
 485	mutex_lock(&ei->i_fc_lock);
 486
 487	return 0;
 488}
 489
 490void __ext4_fc_track_unlink(handle_t *handle,
 491		struct inode *inode, struct dentry *dentry)
 492{
 493	struct __track_dentry_update_args args;
 494	int ret;
 495
 496	args.dentry = dentry;
 497	args.op = EXT4_FC_TAG_UNLINK;
 498
 499	ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
 500					(void *)&args, 0);
 501	trace_ext4_fc_track_unlink(handle, inode, dentry, ret);
 502}
 503
 504void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry)
 505{
 506	struct inode *inode = d_inode(dentry);
 507
 508	if (ext4_fc_disabled(inode->i_sb))
 509		return;
 510
 511	if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))
 512		return;
 513
 514	__ext4_fc_track_unlink(handle, inode, dentry);
 515}
 516
 517void __ext4_fc_track_link(handle_t *handle,
 518	struct inode *inode, struct dentry *dentry)
 519{
 520	struct __track_dentry_update_args args;
 521	int ret;
 522
 523	args.dentry = dentry;
 524	args.op = EXT4_FC_TAG_LINK;
 525
 526	ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
 527					(void *)&args, 0);
 528	trace_ext4_fc_track_link(handle, inode, dentry, ret);
 529}
 530
 531void ext4_fc_track_link(handle_t *handle, struct dentry *dentry)
 532{
 533	struct inode *inode = d_inode(dentry);
 534
 535	if (ext4_fc_disabled(inode->i_sb))
 536		return;
 537
 538	if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))
 539		return;
 540
 541	__ext4_fc_track_link(handle, inode, dentry);
 542}
 543
 544void __ext4_fc_track_create(handle_t *handle, struct inode *inode,
 545			  struct dentry *dentry)
 546{
 547	struct __track_dentry_update_args args;
 548	int ret;
 549
 550	args.dentry = dentry;
 551	args.op = EXT4_FC_TAG_CREAT;
 552
 553	ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
 554					(void *)&args, 0);
 555	trace_ext4_fc_track_create(handle, inode, dentry, ret);
 556}
 557
 558void ext4_fc_track_create(handle_t *handle, struct dentry *dentry)
 559{
 560	struct inode *inode = d_inode(dentry);
 561
 562	if (ext4_fc_disabled(inode->i_sb))
 563		return;
 564
 565	if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))
 566		return;
 567
 568	__ext4_fc_track_create(handle, inode, dentry);
 569}
 570
 571/* __track_fn for inode tracking */
 572static int __track_inode(struct inode *inode, void *arg, bool update)
 573{
 574	if (update)
 575		return -EEXIST;
 576
 577	EXT4_I(inode)->i_fc_lblk_len = 0;
 578
 579	return 0;
 580}
 581
 582void ext4_fc_track_inode(handle_t *handle, struct inode *inode)
 583{
 584	int ret;
 585
 586	if (S_ISDIR(inode->i_mode))
 587		return;
 588
 589	if (ext4_fc_disabled(inode->i_sb))
 590		return;
 591
 592	if (ext4_should_journal_data(inode)) {
 593		ext4_fc_mark_ineligible(inode->i_sb,
 594					EXT4_FC_REASON_INODE_JOURNAL_DATA, handle);
 595		return;
 596	}
 597
 598	if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))
 599		return;
 600
 601	ret = ext4_fc_track_template(handle, inode, __track_inode, NULL, 1);
 602	trace_ext4_fc_track_inode(handle, inode, ret);
 603}
 604
 605struct __track_range_args {
 606	ext4_lblk_t start, end;
 607};
 608
 609/* __track_fn for tracking data updates */
 610static int __track_range(struct inode *inode, void *arg, bool update)
 611{
 612	struct ext4_inode_info *ei = EXT4_I(inode);
 613	ext4_lblk_t oldstart;
 614	struct __track_range_args *__arg =
 615		(struct __track_range_args *)arg;
 616
 617	if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) {
 618		ext4_debug("Special inode %ld being modified\n", inode->i_ino);
 619		return -ECANCELED;
 620	}
 621
 622	oldstart = ei->i_fc_lblk_start;
 623
 624	if (update && ei->i_fc_lblk_len > 0) {
 625		ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start);
 626		ei->i_fc_lblk_len =
 627			max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) -
 628				ei->i_fc_lblk_start + 1;
 629	} else {
 630		ei->i_fc_lblk_start = __arg->start;
 631		ei->i_fc_lblk_len = __arg->end - __arg->start + 1;
 632	}
 633
 634	return 0;
 635}
 636
 637void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start,
 638			 ext4_lblk_t end)
 639{
 640	struct __track_range_args args;
 641	int ret;
 642
 643	if (S_ISDIR(inode->i_mode))
 644		return;
 645
 646	if (ext4_fc_disabled(inode->i_sb))
 647		return;
 648
 649	if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))
 650		return;
 651
 652	args.start = start;
 653	args.end = end;
 654
 655	ret = ext4_fc_track_template(handle, inode,  __track_range, &args, 1);
 656
 657	trace_ext4_fc_track_range(handle, inode, start, end, ret);
 658}
 659
 660static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail)
 661{
 662	blk_opf_t write_flags = REQ_SYNC;
 663	struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh;
 664
 665	/* Add REQ_FUA | REQ_PREFLUSH only its tail */
 666	if (test_opt(sb, BARRIER) && is_tail)
 667		write_flags |= REQ_FUA | REQ_PREFLUSH;
 668	lock_buffer(bh);
 669	set_buffer_dirty(bh);
 670	set_buffer_uptodate(bh);
 671	bh->b_end_io = ext4_end_buffer_io_sync;
 672	submit_bh(REQ_OP_WRITE | write_flags, bh);
 673	EXT4_SB(sb)->s_fc_bh = NULL;
 674}
 675
 676/* Ext4 commit path routines */
 677
 
 
 
 
 
 
 
 
 
 
 
 
 678/*
 679 * Allocate len bytes on a fast commit buffer.
 680 *
 681 * During the commit time this function is used to manage fast commit
 682 * block space. We don't split a fast commit log onto different
 683 * blocks. So this function makes sure that if there's not enough space
 684 * on the current block, the remaining space in the current block is
 685 * marked as unused by adding EXT4_FC_TAG_PAD tag. In that case,
 686 * new block is from jbd2 and CRC is updated to reflect the padding
 687 * we added.
 688 */
 689static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc)
 690{
 691	struct ext4_fc_tl tl;
 692	struct ext4_sb_info *sbi = EXT4_SB(sb);
 693	struct buffer_head *bh;
 694	int bsize = sbi->s_journal->j_blocksize;
 695	int ret, off = sbi->s_fc_bytes % bsize;
 696	int remaining;
 697	u8 *dst;
 698
 699	/*
 700	 * If 'len' is too long to fit in any block alongside a PAD tlv, then we
 701	 * cannot fulfill the request.
 702	 */
 703	if (len > bsize - EXT4_FC_TAG_BASE_LEN)
 704		return NULL;
 705
 706	if (!sbi->s_fc_bh) {
 707		ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
 708		if (ret)
 709			return NULL;
 710		sbi->s_fc_bh = bh;
 711	}
 712	dst = sbi->s_fc_bh->b_data + off;
 713
 714	/*
 715	 * Allocate the bytes in the current block if we can do so while still
 716	 * leaving enough space for a PAD tlv.
 717	 */
 718	remaining = bsize - EXT4_FC_TAG_BASE_LEN - off;
 719	if (len <= remaining) {
 720		sbi->s_fc_bytes += len;
 721		return dst;
 722	}
 723
 724	/*
 725	 * Else, terminate the current block with a PAD tlv, then allocate a new
 726	 * block and allocate the bytes at the start of that new block.
 727	 */
 728
 729	tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD);
 730	tl.fc_len = cpu_to_le16(remaining);
 731	memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);
 732	memset(dst + EXT4_FC_TAG_BASE_LEN, 0, remaining);
 733	*crc = ext4_chksum(sbi, *crc, sbi->s_fc_bh->b_data, bsize);
 734
 735	ext4_fc_submit_bh(sb, false);
 736
 737	ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
 738	if (ret)
 739		return NULL;
 740	sbi->s_fc_bh = bh;
 741	sbi->s_fc_bytes += bsize - off + len;
 742	return sbi->s_fc_bh->b_data;
 743}
 744
 
 
 
 
 
 
 
 
 
 745/*
 746 * Complete a fast commit by writing tail tag.
 747 *
 748 * Writing tail tag marks the end of a fast commit. In order to guarantee
 749 * atomicity, after writing tail tag, even if there's space remaining
 750 * in the block, next commit shouldn't use it. That's why tail tag
 751 * has the length as that of the remaining space on the block.
 752 */
 753static int ext4_fc_write_tail(struct super_block *sb, u32 crc)
 754{
 755	struct ext4_sb_info *sbi = EXT4_SB(sb);
 756	struct ext4_fc_tl tl;
 757	struct ext4_fc_tail tail;
 758	int off, bsize = sbi->s_journal->j_blocksize;
 759	u8 *dst;
 760
 761	/*
 762	 * ext4_fc_reserve_space takes care of allocating an extra block if
 763	 * there's no enough space on this block for accommodating this tail.
 764	 */
 765	dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + sizeof(tail), &crc);
 766	if (!dst)
 767		return -ENOSPC;
 768
 769	off = sbi->s_fc_bytes % bsize;
 770
 771	tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_TAIL);
 772	tl.fc_len = cpu_to_le16(bsize - off + sizeof(struct ext4_fc_tail));
 773	sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize);
 774
 775	memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);
 776	dst += EXT4_FC_TAG_BASE_LEN;
 777	tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid);
 778	memcpy(dst, &tail.fc_tid, sizeof(tail.fc_tid));
 779	dst += sizeof(tail.fc_tid);
 780	crc = ext4_chksum(sbi, crc, sbi->s_fc_bh->b_data,
 781			  dst - (u8 *)sbi->s_fc_bh->b_data);
 782	tail.fc_crc = cpu_to_le32(crc);
 783	memcpy(dst, &tail.fc_crc, sizeof(tail.fc_crc));
 784	dst += sizeof(tail.fc_crc);
 785	memset(dst, 0, bsize - off); /* Don't leak uninitialized memory. */
 786
 787	ext4_fc_submit_bh(sb, true);
 788
 789	return 0;
 790}
 791
 792/*
 793 * Adds tag, length, value and updates CRC. Returns true if tlv was added.
 794 * Returns false if there's not enough space.
 795 */
 796static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val,
 797			   u32 *crc)
 798{
 799	struct ext4_fc_tl tl;
 800	u8 *dst;
 801
 802	dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + len, crc);
 803	if (!dst)
 804		return false;
 805
 806	tl.fc_tag = cpu_to_le16(tag);
 807	tl.fc_len = cpu_to_le16(len);
 808
 809	memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);
 810	memcpy(dst + EXT4_FC_TAG_BASE_LEN, val, len);
 811
 812	return true;
 813}
 814
 815/* Same as above, but adds dentry tlv. */
 816static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc,
 817				   struct ext4_fc_dentry_update *fc_dentry)
 
 
 818{
 819	struct ext4_fc_dentry_info fcd;
 820	struct ext4_fc_tl tl;
 821	int dlen = fc_dentry->fcd_name.len;
 822	u8 *dst = ext4_fc_reserve_space(sb,
 823			EXT4_FC_TAG_BASE_LEN + sizeof(fcd) + dlen, crc);
 824
 825	if (!dst)
 826		return false;
 827
 828	fcd.fc_parent_ino = cpu_to_le32(fc_dentry->fcd_parent);
 829	fcd.fc_ino = cpu_to_le32(fc_dentry->fcd_ino);
 830	tl.fc_tag = cpu_to_le16(fc_dentry->fcd_op);
 831	tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen);
 832	memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);
 833	dst += EXT4_FC_TAG_BASE_LEN;
 834	memcpy(dst, &fcd, sizeof(fcd));
 835	dst += sizeof(fcd);
 836	memcpy(dst, fc_dentry->fcd_name.name, dlen);
 
 837
 838	return true;
 839}
 840
 841/*
 842 * Writes inode in the fast commit space under TLV with tag @tag.
 843 * Returns 0 on success, error on failure.
 844 */
 845static int ext4_fc_write_inode(struct inode *inode, u32 *crc)
 846{
 847	struct ext4_inode_info *ei = EXT4_I(inode);
 848	int inode_len = EXT4_GOOD_OLD_INODE_SIZE;
 849	int ret;
 850	struct ext4_iloc iloc;
 851	struct ext4_fc_inode fc_inode;
 852	struct ext4_fc_tl tl;
 853	u8 *dst;
 854
 855	ret = ext4_get_inode_loc(inode, &iloc);
 856	if (ret)
 857		return ret;
 858
 859	if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA))
 860		inode_len = EXT4_INODE_SIZE(inode->i_sb);
 861	else if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE)
 862		inode_len += ei->i_extra_isize;
 863
 864	fc_inode.fc_ino = cpu_to_le32(inode->i_ino);
 865	tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE);
 866	tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino));
 867
 868	ret = -ECANCELED;
 869	dst = ext4_fc_reserve_space(inode->i_sb,
 870		EXT4_FC_TAG_BASE_LEN + inode_len + sizeof(fc_inode.fc_ino), crc);
 871	if (!dst)
 872		goto err;
 873
 874	memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);
 875	dst += EXT4_FC_TAG_BASE_LEN;
 876	memcpy(dst, &fc_inode, sizeof(fc_inode));
 
 
 877	dst += sizeof(fc_inode);
 878	memcpy(dst, (u8 *)ext4_raw_inode(&iloc), inode_len);
 879	ret = 0;
 880err:
 881	brelse(iloc.bh);
 882	return ret;
 883}
 884
 885/*
 886 * Writes updated data ranges for the inode in question. Updates CRC.
 887 * Returns 0 on success, error otherwise.
 888 */
 889static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc)
 890{
 891	ext4_lblk_t old_blk_size, cur_lblk_off, new_blk_size;
 892	struct ext4_inode_info *ei = EXT4_I(inode);
 893	struct ext4_map_blocks map;
 894	struct ext4_fc_add_range fc_ext;
 895	struct ext4_fc_del_range lrange;
 896	struct ext4_extent *ex;
 897	int ret;
 898
 899	mutex_lock(&ei->i_fc_lock);
 900	if (ei->i_fc_lblk_len == 0) {
 901		mutex_unlock(&ei->i_fc_lock);
 902		return 0;
 903	}
 904	old_blk_size = ei->i_fc_lblk_start;
 905	new_blk_size = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1;
 906	ei->i_fc_lblk_len = 0;
 907	mutex_unlock(&ei->i_fc_lock);
 908
 909	cur_lblk_off = old_blk_size;
 910	ext4_debug("will try writing %d to %d for inode %ld\n",
 911		   cur_lblk_off, new_blk_size, inode->i_ino);
 912
 913	while (cur_lblk_off <= new_blk_size) {
 914		map.m_lblk = cur_lblk_off;
 915		map.m_len = new_blk_size - cur_lblk_off + 1;
 916		ret = ext4_map_blocks(NULL, inode, &map, 0);
 917		if (ret < 0)
 918			return -ECANCELED;
 919
 920		if (map.m_len == 0) {
 921			cur_lblk_off++;
 922			continue;
 923		}
 924
 925		if (ret == 0) {
 926			lrange.fc_ino = cpu_to_le32(inode->i_ino);
 927			lrange.fc_lblk = cpu_to_le32(map.m_lblk);
 928			lrange.fc_len = cpu_to_le32(map.m_len);
 929			if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE,
 930					    sizeof(lrange), (u8 *)&lrange, crc))
 931				return -ENOSPC;
 932		} else {
 933			unsigned int max = (map.m_flags & EXT4_MAP_UNWRITTEN) ?
 934				EXT_UNWRITTEN_MAX_LEN : EXT_INIT_MAX_LEN;
 935
 936			/* Limit the number of blocks in one extent */
 937			map.m_len = min(max, map.m_len);
 938
 939			fc_ext.fc_ino = cpu_to_le32(inode->i_ino);
 940			ex = (struct ext4_extent *)&fc_ext.fc_ex;
 941			ex->ee_block = cpu_to_le32(map.m_lblk);
 942			ex->ee_len = cpu_to_le16(map.m_len);
 943			ext4_ext_store_pblock(ex, map.m_pblk);
 944			if (map.m_flags & EXT4_MAP_UNWRITTEN)
 945				ext4_ext_mark_unwritten(ex);
 946			else
 947				ext4_ext_mark_initialized(ex);
 948			if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE,
 949					    sizeof(fc_ext), (u8 *)&fc_ext, crc))
 950				return -ENOSPC;
 951		}
 952
 953		cur_lblk_off += map.m_len;
 954	}
 955
 956	return 0;
 957}
 958
 959
 960/* Submit data for all the fast commit inodes */
 961static int ext4_fc_submit_inode_data_all(journal_t *journal)
 962{
 963	struct super_block *sb = journal->j_private;
 964	struct ext4_sb_info *sbi = EXT4_SB(sb);
 965	struct ext4_inode_info *ei;
 966	int ret = 0;
 967
 968	spin_lock(&sbi->s_fc_lock);
 
 969	list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
 970		ext4_set_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING);
 971		while (atomic_read(&ei->i_fc_updates)) {
 972			DEFINE_WAIT(wait);
 973
 974			prepare_to_wait(&ei->i_fc_wait, &wait,
 975						TASK_UNINTERRUPTIBLE);
 976			if (atomic_read(&ei->i_fc_updates)) {
 977				spin_unlock(&sbi->s_fc_lock);
 978				schedule();
 979				spin_lock(&sbi->s_fc_lock);
 980			}
 981			finish_wait(&ei->i_fc_wait, &wait);
 982		}
 983		spin_unlock(&sbi->s_fc_lock);
 984		ret = jbd2_submit_inode_data(journal, ei->jinode);
 985		if (ret)
 986			return ret;
 987		spin_lock(&sbi->s_fc_lock);
 988	}
 989	spin_unlock(&sbi->s_fc_lock);
 990
 991	return ret;
 992}
 993
 994/* Wait for completion of data for all the fast commit inodes */
 995static int ext4_fc_wait_inode_data_all(journal_t *journal)
 996{
 997	struct super_block *sb = journal->j_private;
 998	struct ext4_sb_info *sbi = EXT4_SB(sb);
 999	struct ext4_inode_info *pos, *n;
1000	int ret = 0;
1001
1002	spin_lock(&sbi->s_fc_lock);
1003	list_for_each_entry_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
1004		if (!ext4_test_inode_state(&pos->vfs_inode,
1005					   EXT4_STATE_FC_COMMITTING))
1006			continue;
1007		spin_unlock(&sbi->s_fc_lock);
1008
1009		ret = jbd2_wait_inode_data(journal, pos->jinode);
1010		if (ret)
1011			return ret;
1012		spin_lock(&sbi->s_fc_lock);
1013	}
1014	spin_unlock(&sbi->s_fc_lock);
1015
1016	return 0;
1017}
1018
1019/* Commit all the directory entry updates */
1020static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc)
1021__acquires(&sbi->s_fc_lock)
1022__releases(&sbi->s_fc_lock)
1023{
1024	struct super_block *sb = journal->j_private;
1025	struct ext4_sb_info *sbi = EXT4_SB(sb);
1026	struct ext4_fc_dentry_update *fc_dentry, *fc_dentry_n;
1027	struct inode *inode;
1028	struct ext4_inode_info *ei;
1029	int ret;
1030
1031	if (list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN]))
1032		return 0;
1033	list_for_each_entry_safe(fc_dentry, fc_dentry_n,
1034				 &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) {
1035		if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) {
1036			spin_unlock(&sbi->s_fc_lock);
1037			if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry)) {
 
 
 
 
1038				ret = -ENOSPC;
1039				goto lock_and_exit;
1040			}
1041			spin_lock(&sbi->s_fc_lock);
1042			continue;
1043		}
 
 
 
 
 
 
 
 
 
1044		/*
1045		 * With fcd_dilist we need not loop in sbi->s_fc_q to get the
1046		 * corresponding inode pointer
1047		 */
1048		WARN_ON(list_empty(&fc_dentry->fcd_dilist));
1049		ei = list_first_entry(&fc_dentry->fcd_dilist,
1050				struct ext4_inode_info, i_fc_dilist);
1051		inode = &ei->vfs_inode;
1052		WARN_ON(inode->i_ino != fc_dentry->fcd_ino);
1053
1054		spin_unlock(&sbi->s_fc_lock);
1055
1056		/*
1057		 * We first write the inode and then the create dirent. This
1058		 * allows the recovery code to create an unnamed inode first
1059		 * and then link it to a directory entry. This allows us
1060		 * to use namei.c routines almost as is and simplifies
1061		 * the recovery code.
1062		 */
1063		ret = ext4_fc_write_inode(inode, crc);
1064		if (ret)
1065			goto lock_and_exit;
1066
1067		ret = ext4_fc_write_inode_data(inode, crc);
1068		if (ret)
1069			goto lock_and_exit;
1070
1071		if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry)) {
 
 
 
 
1072			ret = -ENOSPC;
1073			goto lock_and_exit;
1074		}
1075
1076		spin_lock(&sbi->s_fc_lock);
1077	}
1078	return 0;
1079lock_and_exit:
1080	spin_lock(&sbi->s_fc_lock);
1081	return ret;
1082}
1083
1084static int ext4_fc_perform_commit(journal_t *journal)
1085{
1086	struct super_block *sb = journal->j_private;
1087	struct ext4_sb_info *sbi = EXT4_SB(sb);
1088	struct ext4_inode_info *iter;
1089	struct ext4_fc_head head;
1090	struct inode *inode;
1091	struct blk_plug plug;
1092	int ret = 0;
1093	u32 crc = 0;
1094
1095	ret = ext4_fc_submit_inode_data_all(journal);
1096	if (ret)
1097		return ret;
1098
1099	ret = ext4_fc_wait_inode_data_all(journal);
1100	if (ret)
1101		return ret;
1102
1103	/*
1104	 * If file system device is different from journal device, issue a cache
1105	 * flush before we start writing fast commit blocks.
1106	 */
1107	if (journal->j_fs_dev != journal->j_dev)
1108		blkdev_issue_flush(journal->j_fs_dev);
1109
1110	blk_start_plug(&plug);
1111	if (sbi->s_fc_bytes == 0) {
1112		/*
1113		 * Add a head tag only if this is the first fast commit
1114		 * in this TID.
1115		 */
1116		head.fc_features = cpu_to_le32(EXT4_FC_SUPPORTED_FEATURES);
1117		head.fc_tid = cpu_to_le32(
1118			sbi->s_journal->j_running_transaction->t_tid);
1119		if (!ext4_fc_add_tlv(sb, EXT4_FC_TAG_HEAD, sizeof(head),
1120			(u8 *)&head, &crc)) {
1121			ret = -ENOSPC;
1122			goto out;
1123		}
1124	}
1125
1126	spin_lock(&sbi->s_fc_lock);
1127	ret = ext4_fc_commit_dentry_updates(journal, &crc);
1128	if (ret) {
1129		spin_unlock(&sbi->s_fc_lock);
1130		goto out;
1131	}
1132
1133	list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
1134		inode = &iter->vfs_inode;
1135		if (!ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING))
1136			continue;
1137
1138		spin_unlock(&sbi->s_fc_lock);
1139		ret = ext4_fc_write_inode_data(inode, &crc);
1140		if (ret)
1141			goto out;
1142		ret = ext4_fc_write_inode(inode, &crc);
1143		if (ret)
1144			goto out;
1145		spin_lock(&sbi->s_fc_lock);
1146	}
1147	spin_unlock(&sbi->s_fc_lock);
1148
1149	ret = ext4_fc_write_tail(sb, crc);
1150
1151out:
1152	blk_finish_plug(&plug);
1153	return ret;
1154}
1155
1156static void ext4_fc_update_stats(struct super_block *sb, int status,
1157				 u64 commit_time, int nblks, tid_t commit_tid)
1158{
1159	struct ext4_fc_stats *stats = &EXT4_SB(sb)->s_fc_stats;
1160
1161	ext4_debug("Fast commit ended with status = %d for tid %u",
1162			status, commit_tid);
1163	if (status == EXT4_FC_STATUS_OK) {
1164		stats->fc_num_commits++;
1165		stats->fc_numblks += nblks;
1166		if (likely(stats->s_fc_avg_commit_time))
1167			stats->s_fc_avg_commit_time =
1168				(commit_time +
1169				 stats->s_fc_avg_commit_time * 3) / 4;
1170		else
1171			stats->s_fc_avg_commit_time = commit_time;
1172	} else if (status == EXT4_FC_STATUS_FAILED ||
1173		   status == EXT4_FC_STATUS_INELIGIBLE) {
1174		if (status == EXT4_FC_STATUS_FAILED)
1175			stats->fc_failed_commits++;
1176		stats->fc_ineligible_commits++;
1177	} else {
1178		stats->fc_skipped_commits++;
1179	}
1180	trace_ext4_fc_commit_stop(sb, nblks, status, commit_tid);
1181}
1182
1183/*
1184 * The main commit entry point. Performs a fast commit for transaction
1185 * commit_tid if needed. If it's not possible to perform a fast commit
1186 * due to various reasons, we fall back to full commit. Returns 0
1187 * on success, error otherwise.
1188 */
1189int ext4_fc_commit(journal_t *journal, tid_t commit_tid)
1190{
1191	struct super_block *sb = journal->j_private;
1192	struct ext4_sb_info *sbi = EXT4_SB(sb);
1193	int nblks = 0, ret, bsize = journal->j_blocksize;
1194	int subtid = atomic_read(&sbi->s_fc_subtid);
1195	int status = EXT4_FC_STATUS_OK, fc_bufs_before = 0;
1196	ktime_t start_time, commit_time;
1197
1198	if (!test_opt2(sb, JOURNAL_FAST_COMMIT))
1199		return jbd2_complete_transaction(journal, commit_tid);
1200
1201	trace_ext4_fc_commit_start(sb, commit_tid);
1202
1203	start_time = ktime_get();
1204
 
 
 
 
 
 
1205restart_fc:
1206	ret = jbd2_fc_begin_commit(journal, commit_tid);
1207	if (ret == -EALREADY) {
1208		/* There was an ongoing commit, check if we need to restart */
1209		if (atomic_read(&sbi->s_fc_subtid) <= subtid &&
1210			commit_tid > journal->j_commit_sequence)
1211			goto restart_fc;
1212		ext4_fc_update_stats(sb, EXT4_FC_STATUS_SKIPPED, 0, 0,
1213				commit_tid);
1214		return 0;
1215	} else if (ret) {
1216		/*
1217		 * Commit couldn't start. Just update stats and perform a
1218		 * full commit.
1219		 */
1220		ext4_fc_update_stats(sb, EXT4_FC_STATUS_FAILED, 0, 0,
1221				commit_tid);
1222		return jbd2_complete_transaction(journal, commit_tid);
1223	}
1224
1225	/*
1226	 * After establishing journal barrier via jbd2_fc_begin_commit(), check
1227	 * if we are fast commit ineligible.
1228	 */
1229	if (ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE)) {
1230		status = EXT4_FC_STATUS_INELIGIBLE;
1231		goto fallback;
1232	}
1233
1234	fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize;
1235	ret = ext4_fc_perform_commit(journal);
1236	if (ret < 0) {
1237		status = EXT4_FC_STATUS_FAILED;
1238		goto fallback;
 
1239	}
1240	nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before;
1241	ret = jbd2_fc_wait_bufs(journal, nblks);
1242	if (ret < 0) {
1243		status = EXT4_FC_STATUS_FAILED;
1244		goto fallback;
 
1245	}
1246	atomic_inc(&sbi->s_fc_subtid);
1247	ret = jbd2_fc_end_commit(journal);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1248	/*
1249	 * weight the commit time higher than the average time so we
1250	 * don't react too strongly to vast changes in the commit time
1251	 */
1252	commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time));
1253	ext4_fc_update_stats(sb, status, commit_time, nblks, commit_tid);
1254	return ret;
1255
1256fallback:
1257	ret = jbd2_fc_end_commit_fallback(journal);
1258	ext4_fc_update_stats(sb, status, 0, 0, commit_tid);
1259	return ret;
 
 
 
 
 
 
1260}
1261
1262/*
1263 * Fast commit cleanup routine. This is called after every fast commit and
1264 * full commit. full is true if we are called after a full commit.
1265 */
1266static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid)
1267{
1268	struct super_block *sb = journal->j_private;
1269	struct ext4_sb_info *sbi = EXT4_SB(sb);
1270	struct ext4_inode_info *iter, *iter_n;
1271	struct ext4_fc_dentry_update *fc_dentry;
1272
1273	if (full && sbi->s_fc_bh)
1274		sbi->s_fc_bh = NULL;
1275
1276	trace_ext4_fc_cleanup(journal, full, tid);
1277	jbd2_fc_release_bufs(journal);
1278
1279	spin_lock(&sbi->s_fc_lock);
1280	list_for_each_entry_safe(iter, iter_n, &sbi->s_fc_q[FC_Q_MAIN],
1281				 i_fc_list) {
1282		list_del_init(&iter->i_fc_list);
1283		ext4_clear_inode_state(&iter->vfs_inode,
1284				       EXT4_STATE_FC_COMMITTING);
1285		if (iter->i_sync_tid <= tid)
1286			ext4_fc_reset_inode(&iter->vfs_inode);
1287		/* Make sure EXT4_STATE_FC_COMMITTING bit is clear */
1288		smp_mb();
1289#if (BITS_PER_LONG < 64)
1290		wake_up_bit(&iter->i_state_flags, EXT4_STATE_FC_COMMITTING);
1291#else
1292		wake_up_bit(&iter->i_flags, EXT4_STATE_FC_COMMITTING);
1293#endif
1294	}
1295
1296	while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) {
1297		fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN],
1298					     struct ext4_fc_dentry_update,
1299					     fcd_list);
1300		list_del_init(&fc_dentry->fcd_list);
1301		list_del_init(&fc_dentry->fcd_dilist);
1302		spin_unlock(&sbi->s_fc_lock);
1303
1304		if (fc_dentry->fcd_name.name &&
1305			fc_dentry->fcd_name.len > DNAME_INLINE_LEN)
1306			kfree(fc_dentry->fcd_name.name);
1307		kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry);
1308		spin_lock(&sbi->s_fc_lock);
1309	}
1310
1311	list_splice_init(&sbi->s_fc_dentry_q[FC_Q_STAGING],
1312				&sbi->s_fc_dentry_q[FC_Q_MAIN]);
1313	list_splice_init(&sbi->s_fc_q[FC_Q_STAGING],
1314				&sbi->s_fc_q[FC_Q_MAIN]);
1315
1316	if (tid >= sbi->s_fc_ineligible_tid) {
1317		sbi->s_fc_ineligible_tid = 0;
1318		ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
1319	}
1320
1321	if (full)
1322		sbi->s_fc_bytes = 0;
1323	spin_unlock(&sbi->s_fc_lock);
1324	trace_ext4_fc_stats(sb);
1325}
1326
1327/* Ext4 Replay Path Routines */
1328
1329/* Helper struct for dentry replay routines */
1330struct dentry_info_args {
1331	int parent_ino, dname_len, ino, inode_len;
1332	char *dname;
1333};
1334
1335/* Same as struct ext4_fc_tl, but uses native endianness fields */
1336struct ext4_fc_tl_mem {
1337	u16 fc_tag;
1338	u16 fc_len;
1339};
1340
1341static inline void tl_to_darg(struct dentry_info_args *darg,
1342			      struct ext4_fc_tl_mem *tl, u8 *val)
1343{
1344	struct ext4_fc_dentry_info fcd;
1345
1346	memcpy(&fcd, val, sizeof(fcd));
1347
1348	darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino);
1349	darg->ino = le32_to_cpu(fcd.fc_ino);
1350	darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname);
1351	darg->dname_len = tl->fc_len - sizeof(struct ext4_fc_dentry_info);
1352}
1353
1354static inline void ext4_fc_get_tl(struct ext4_fc_tl_mem *tl, u8 *val)
1355{
1356	struct ext4_fc_tl tl_disk;
1357
1358	memcpy(&tl_disk, val, EXT4_FC_TAG_BASE_LEN);
1359	tl->fc_len = le16_to_cpu(tl_disk.fc_len);
1360	tl->fc_tag = le16_to_cpu(tl_disk.fc_tag);
1361}
1362
1363/* Unlink replay function */
1364static int ext4_fc_replay_unlink(struct super_block *sb,
1365				 struct ext4_fc_tl_mem *tl, u8 *val)
1366{
1367	struct inode *inode, *old_parent;
1368	struct qstr entry;
1369	struct dentry_info_args darg;
1370	int ret = 0;
1371
1372	tl_to_darg(&darg, tl, val);
1373
1374	trace_ext4_fc_replay(sb, EXT4_FC_TAG_UNLINK, darg.ino,
1375			darg.parent_ino, darg.dname_len);
1376
1377	entry.name = darg.dname;
1378	entry.len = darg.dname_len;
1379	inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1380
1381	if (IS_ERR(inode)) {
1382		ext4_debug("Inode %d not found", darg.ino);
1383		return 0;
1384	}
1385
1386	old_parent = ext4_iget(sb, darg.parent_ino,
1387				EXT4_IGET_NORMAL);
1388	if (IS_ERR(old_parent)) {
1389		ext4_debug("Dir with inode %d not found", darg.parent_ino);
1390		iput(inode);
1391		return 0;
1392	}
1393
1394	ret = __ext4_unlink(old_parent, &entry, inode, NULL);
1395	/* -ENOENT ok coz it might not exist anymore. */
1396	if (ret == -ENOENT)
1397		ret = 0;
1398	iput(old_parent);
1399	iput(inode);
1400	return ret;
1401}
1402
1403static int ext4_fc_replay_link_internal(struct super_block *sb,
1404				struct dentry_info_args *darg,
1405				struct inode *inode)
1406{
1407	struct inode *dir = NULL;
1408	struct dentry *dentry_dir = NULL, *dentry_inode = NULL;
1409	struct qstr qstr_dname = QSTR_INIT(darg->dname, darg->dname_len);
1410	int ret = 0;
1411
1412	dir = ext4_iget(sb, darg->parent_ino, EXT4_IGET_NORMAL);
1413	if (IS_ERR(dir)) {
1414		ext4_debug("Dir with inode %d not found.", darg->parent_ino);
1415		dir = NULL;
1416		goto out;
1417	}
1418
1419	dentry_dir = d_obtain_alias(dir);
1420	if (IS_ERR(dentry_dir)) {
1421		ext4_debug("Failed to obtain dentry");
1422		dentry_dir = NULL;
1423		goto out;
1424	}
1425
1426	dentry_inode = d_alloc(dentry_dir, &qstr_dname);
1427	if (!dentry_inode) {
1428		ext4_debug("Inode dentry not created.");
1429		ret = -ENOMEM;
1430		goto out;
1431	}
1432
1433	ret = __ext4_link(dir, inode, dentry_inode);
1434	/*
1435	 * It's possible that link already existed since data blocks
1436	 * for the dir in question got persisted before we crashed OR
1437	 * we replayed this tag and crashed before the entire replay
1438	 * could complete.
1439	 */
1440	if (ret && ret != -EEXIST) {
1441		ext4_debug("Failed to link\n");
1442		goto out;
1443	}
1444
1445	ret = 0;
1446out:
1447	if (dentry_dir) {
1448		d_drop(dentry_dir);
1449		dput(dentry_dir);
1450	} else if (dir) {
1451		iput(dir);
1452	}
1453	if (dentry_inode) {
1454		d_drop(dentry_inode);
1455		dput(dentry_inode);
1456	}
1457
1458	return ret;
1459}
1460
1461/* Link replay function */
1462static int ext4_fc_replay_link(struct super_block *sb,
1463			       struct ext4_fc_tl_mem *tl, u8 *val)
1464{
1465	struct inode *inode;
1466	struct dentry_info_args darg;
1467	int ret = 0;
1468
1469	tl_to_darg(&darg, tl, val);
1470	trace_ext4_fc_replay(sb, EXT4_FC_TAG_LINK, darg.ino,
1471			darg.parent_ino, darg.dname_len);
1472
1473	inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1474	if (IS_ERR(inode)) {
1475		ext4_debug("Inode not found.");
1476		return 0;
1477	}
1478
1479	ret = ext4_fc_replay_link_internal(sb, &darg, inode);
1480	iput(inode);
1481	return ret;
1482}
1483
1484/*
1485 * Record all the modified inodes during replay. We use this later to setup
1486 * block bitmaps correctly.
1487 */
1488static int ext4_fc_record_modified_inode(struct super_block *sb, int ino)
1489{
1490	struct ext4_fc_replay_state *state;
1491	int i;
1492
1493	state = &EXT4_SB(sb)->s_fc_replay_state;
1494	for (i = 0; i < state->fc_modified_inodes_used; i++)
1495		if (state->fc_modified_inodes[i] == ino)
1496			return 0;
1497	if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) {
1498		int *fc_modified_inodes;
1499
1500		fc_modified_inodes = krealloc(state->fc_modified_inodes,
1501				sizeof(int) * (state->fc_modified_inodes_size +
1502				EXT4_FC_REPLAY_REALLOC_INCREMENT),
1503				GFP_KERNEL);
1504		if (!fc_modified_inodes)
1505			return -ENOMEM;
1506		state->fc_modified_inodes = fc_modified_inodes;
1507		state->fc_modified_inodes_size +=
1508			EXT4_FC_REPLAY_REALLOC_INCREMENT;
 
 
 
 
 
 
1509	}
1510	state->fc_modified_inodes[state->fc_modified_inodes_used++] = ino;
1511	return 0;
1512}
1513
1514/*
1515 * Inode replay function
1516 */
1517static int ext4_fc_replay_inode(struct super_block *sb,
1518				struct ext4_fc_tl_mem *tl, u8 *val)
1519{
1520	struct ext4_fc_inode fc_inode;
1521	struct ext4_inode *raw_inode;
1522	struct ext4_inode *raw_fc_inode;
1523	struct inode *inode = NULL;
1524	struct ext4_iloc iloc;
1525	int inode_len, ino, ret, tag = tl->fc_tag;
1526	struct ext4_extent_header *eh;
1527	size_t off_gen = offsetof(struct ext4_inode, i_generation);
1528
1529	memcpy(&fc_inode, val, sizeof(fc_inode));
1530
1531	ino = le32_to_cpu(fc_inode.fc_ino);
1532	trace_ext4_fc_replay(sb, tag, ino, 0, 0);
1533
1534	inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
1535	if (!IS_ERR(inode)) {
1536		ext4_ext_clear_bb(inode);
1537		iput(inode);
1538	}
1539	inode = NULL;
1540
1541	ret = ext4_fc_record_modified_inode(sb, ino);
1542	if (ret)
1543		goto out;
1544
1545	raw_fc_inode = (struct ext4_inode *)
1546		(val + offsetof(struct ext4_fc_inode, fc_raw_inode));
1547	ret = ext4_get_fc_inode_loc(sb, ino, &iloc);
1548	if (ret)
1549		goto out;
1550
1551	inode_len = tl->fc_len - sizeof(struct ext4_fc_inode);
1552	raw_inode = ext4_raw_inode(&iloc);
1553
1554	memcpy(raw_inode, raw_fc_inode, offsetof(struct ext4_inode, i_block));
1555	memcpy((u8 *)raw_inode + off_gen, (u8 *)raw_fc_inode + off_gen,
1556	       inode_len - off_gen);
1557	if (le32_to_cpu(raw_inode->i_flags) & EXT4_EXTENTS_FL) {
1558		eh = (struct ext4_extent_header *)(&raw_inode->i_block[0]);
1559		if (eh->eh_magic != EXT4_EXT_MAGIC) {
1560			memset(eh, 0, sizeof(*eh));
1561			eh->eh_magic = EXT4_EXT_MAGIC;
1562			eh->eh_max = cpu_to_le16(
1563				(sizeof(raw_inode->i_block) -
1564				 sizeof(struct ext4_extent_header))
1565				 / sizeof(struct ext4_extent));
1566		}
1567	} else if (le32_to_cpu(raw_inode->i_flags) & EXT4_INLINE_DATA_FL) {
1568		memcpy(raw_inode->i_block, raw_fc_inode->i_block,
1569			sizeof(raw_inode->i_block));
1570	}
1571
1572	/* Immediately update the inode on disk. */
1573	ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
1574	if (ret)
1575		goto out;
1576	ret = sync_dirty_buffer(iloc.bh);
1577	if (ret)
1578		goto out;
1579	ret = ext4_mark_inode_used(sb, ino);
1580	if (ret)
1581		goto out;
1582
1583	/* Given that we just wrote the inode on disk, this SHOULD succeed. */
1584	inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
1585	if (IS_ERR(inode)) {
1586		ext4_debug("Inode not found.");
1587		return -EFSCORRUPTED;
1588	}
1589
1590	/*
1591	 * Our allocator could have made different decisions than before
1592	 * crashing. This should be fixed but until then, we calculate
1593	 * the number of blocks the inode.
1594	 */
1595	if (!ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA))
1596		ext4_ext_replay_set_iblocks(inode);
1597
1598	inode->i_generation = le32_to_cpu(ext4_raw_inode(&iloc)->i_generation);
1599	ext4_reset_inode_seed(inode);
1600
1601	ext4_inode_csum_set(inode, ext4_raw_inode(&iloc), EXT4_I(inode));
1602	ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
1603	sync_dirty_buffer(iloc.bh);
1604	brelse(iloc.bh);
1605out:
1606	iput(inode);
1607	if (!ret)
1608		blkdev_issue_flush(sb->s_bdev);
1609
1610	return 0;
1611}
1612
1613/*
1614 * Dentry create replay function.
1615 *
1616 * EXT4_FC_TAG_CREAT is preceded by EXT4_FC_TAG_INODE_FULL. Which means, the
1617 * inode for which we are trying to create a dentry here, should already have
1618 * been replayed before we start here.
1619 */
1620static int ext4_fc_replay_create(struct super_block *sb,
1621				 struct ext4_fc_tl_mem *tl, u8 *val)
1622{
1623	int ret = 0;
1624	struct inode *inode = NULL;
1625	struct inode *dir = NULL;
1626	struct dentry_info_args darg;
1627
1628	tl_to_darg(&darg, tl, val);
1629
1630	trace_ext4_fc_replay(sb, EXT4_FC_TAG_CREAT, darg.ino,
1631			darg.parent_ino, darg.dname_len);
1632
1633	/* This takes care of update group descriptor and other metadata */
1634	ret = ext4_mark_inode_used(sb, darg.ino);
1635	if (ret)
1636		goto out;
1637
1638	inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1639	if (IS_ERR(inode)) {
1640		ext4_debug("inode %d not found.", darg.ino);
1641		inode = NULL;
1642		ret = -EINVAL;
1643		goto out;
1644	}
1645
1646	if (S_ISDIR(inode->i_mode)) {
1647		/*
1648		 * If we are creating a directory, we need to make sure that the
1649		 * dot and dot dot dirents are setup properly.
1650		 */
1651		dir = ext4_iget(sb, darg.parent_ino, EXT4_IGET_NORMAL);
1652		if (IS_ERR(dir)) {
1653			ext4_debug("Dir %d not found.", darg.ino);
1654			goto out;
1655		}
1656		ret = ext4_init_new_dir(NULL, dir, inode);
1657		iput(dir);
1658		if (ret) {
1659			ret = 0;
1660			goto out;
1661		}
1662	}
1663	ret = ext4_fc_replay_link_internal(sb, &darg, inode);
1664	if (ret)
1665		goto out;
1666	set_nlink(inode, 1);
1667	ext4_mark_inode_dirty(NULL, inode);
1668out:
1669	iput(inode);
 
1670	return ret;
1671}
1672
1673/*
1674 * Record physical disk regions which are in use as per fast commit area,
1675 * and used by inodes during replay phase. Our simple replay phase
1676 * allocator excludes these regions from allocation.
1677 */
1678int ext4_fc_record_regions(struct super_block *sb, int ino,
1679		ext4_lblk_t lblk, ext4_fsblk_t pblk, int len, int replay)
1680{
1681	struct ext4_fc_replay_state *state;
1682	struct ext4_fc_alloc_region *region;
1683
1684	state = &EXT4_SB(sb)->s_fc_replay_state;
1685	/*
1686	 * during replay phase, the fc_regions_valid may not same as
1687	 * fc_regions_used, update it when do new additions.
1688	 */
1689	if (replay && state->fc_regions_used != state->fc_regions_valid)
1690		state->fc_regions_used = state->fc_regions_valid;
1691	if (state->fc_regions_used == state->fc_regions_size) {
1692		struct ext4_fc_alloc_region *fc_regions;
1693
1694		fc_regions = krealloc(state->fc_regions,
1695				      sizeof(struct ext4_fc_alloc_region) *
1696				      (state->fc_regions_size +
1697				       EXT4_FC_REPLAY_REALLOC_INCREMENT),
1698				      GFP_KERNEL);
1699		if (!fc_regions)
1700			return -ENOMEM;
1701		state->fc_regions_size +=
1702			EXT4_FC_REPLAY_REALLOC_INCREMENT;
1703		state->fc_regions = fc_regions;
 
 
 
 
 
 
1704	}
1705	region = &state->fc_regions[state->fc_regions_used++];
1706	region->ino = ino;
1707	region->lblk = lblk;
1708	region->pblk = pblk;
1709	region->len = len;
1710
1711	if (replay)
1712		state->fc_regions_valid++;
1713
1714	return 0;
1715}
1716
1717/* Replay add range tag */
1718static int ext4_fc_replay_add_range(struct super_block *sb,
1719				    struct ext4_fc_tl_mem *tl, u8 *val)
1720{
1721	struct ext4_fc_add_range fc_add_ex;
1722	struct ext4_extent newex, *ex;
1723	struct inode *inode;
1724	ext4_lblk_t start, cur;
1725	int remaining, len;
1726	ext4_fsblk_t start_pblk;
1727	struct ext4_map_blocks map;
1728	struct ext4_ext_path *path = NULL;
1729	int ret;
1730
1731	memcpy(&fc_add_ex, val, sizeof(fc_add_ex));
1732	ex = (struct ext4_extent *)&fc_add_ex.fc_ex;
1733
1734	trace_ext4_fc_replay(sb, EXT4_FC_TAG_ADD_RANGE,
1735		le32_to_cpu(fc_add_ex.fc_ino), le32_to_cpu(ex->ee_block),
1736		ext4_ext_get_actual_len(ex));
1737
1738	inode = ext4_iget(sb, le32_to_cpu(fc_add_ex.fc_ino), EXT4_IGET_NORMAL);
1739	if (IS_ERR(inode)) {
1740		ext4_debug("Inode not found.");
1741		return 0;
1742	}
1743
1744	ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
1745	if (ret)
1746		goto out;
1747
1748	start = le32_to_cpu(ex->ee_block);
1749	start_pblk = ext4_ext_pblock(ex);
1750	len = ext4_ext_get_actual_len(ex);
1751
1752	cur = start;
1753	remaining = len;
1754	ext4_debug("ADD_RANGE, lblk %d, pblk %lld, len %d, unwritten %d, inode %ld\n",
1755		  start, start_pblk, len, ext4_ext_is_unwritten(ex),
1756		  inode->i_ino);
1757
1758	while (remaining > 0) {
1759		map.m_lblk = cur;
1760		map.m_len = remaining;
1761		map.m_pblk = 0;
1762		ret = ext4_map_blocks(NULL, inode, &map, 0);
1763
1764		if (ret < 0)
1765			goto out;
 
 
1766
1767		if (ret == 0) {
1768			/* Range is not mapped */
1769			path = ext4_find_extent(inode, cur, NULL, 0);
1770			if (IS_ERR(path))
1771				goto out;
 
 
1772			memset(&newex, 0, sizeof(newex));
1773			newex.ee_block = cpu_to_le32(cur);
1774			ext4_ext_store_pblock(
1775				&newex, start_pblk + cur - start);
1776			newex.ee_len = cpu_to_le16(map.m_len);
1777			if (ext4_ext_is_unwritten(ex))
1778				ext4_ext_mark_unwritten(&newex);
1779			down_write(&EXT4_I(inode)->i_data_sem);
1780			ret = ext4_ext_insert_extent(
1781				NULL, inode, &path, &newex, 0);
1782			up_write((&EXT4_I(inode)->i_data_sem));
1783			ext4_free_ext_path(path);
1784			if (ret)
1785				goto out;
 
 
 
1786			goto next;
1787		}
1788
1789		if (start_pblk + cur - start != map.m_pblk) {
1790			/*
1791			 * Logical to physical mapping changed. This can happen
1792			 * if this range was removed and then reallocated to
1793			 * map to new physical blocks during a fast commit.
1794			 */
1795			ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
1796					ext4_ext_is_unwritten(ex),
1797					start_pblk + cur - start);
1798			if (ret)
1799				goto out;
 
 
1800			/*
1801			 * Mark the old blocks as free since they aren't used
1802			 * anymore. We maintain an array of all the modified
1803			 * inodes. In case these blocks are still used at either
1804			 * a different logical range in the same inode or in
1805			 * some different inode, we will mark them as allocated
1806			 * at the end of the FC replay using our array of
1807			 * modified inodes.
1808			 */
1809			ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, false);
1810			goto next;
1811		}
1812
1813		/* Range is mapped and needs a state change */
1814		ext4_debug("Converting from %ld to %d %lld",
1815				map.m_flags & EXT4_MAP_UNWRITTEN,
1816			ext4_ext_is_unwritten(ex), map.m_pblk);
1817		ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
1818					ext4_ext_is_unwritten(ex), map.m_pblk);
1819		if (ret)
1820			goto out;
 
 
1821		/*
1822		 * We may have split the extent tree while toggling the state.
1823		 * Try to shrink the extent tree now.
1824		 */
1825		ext4_ext_replay_shrink_inode(inode, start + len);
1826next:
1827		cur += map.m_len;
1828		remaining -= map.m_len;
1829	}
1830	ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >>
1831					sb->s_blocksize_bits);
1832out:
1833	iput(inode);
1834	return 0;
1835}
1836
1837/* Replay DEL_RANGE tag */
1838static int
1839ext4_fc_replay_del_range(struct super_block *sb,
1840			 struct ext4_fc_tl_mem *tl, u8 *val)
1841{
1842	struct inode *inode;
1843	struct ext4_fc_del_range lrange;
1844	struct ext4_map_blocks map;
1845	ext4_lblk_t cur, remaining;
1846	int ret;
1847
1848	memcpy(&lrange, val, sizeof(lrange));
1849	cur = le32_to_cpu(lrange.fc_lblk);
1850	remaining = le32_to_cpu(lrange.fc_len);
1851
1852	trace_ext4_fc_replay(sb, EXT4_FC_TAG_DEL_RANGE,
1853		le32_to_cpu(lrange.fc_ino), cur, remaining);
1854
1855	inode = ext4_iget(sb, le32_to_cpu(lrange.fc_ino), EXT4_IGET_NORMAL);
1856	if (IS_ERR(inode)) {
1857		ext4_debug("Inode %d not found", le32_to_cpu(lrange.fc_ino));
1858		return 0;
1859	}
1860
1861	ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
1862	if (ret)
1863		goto out;
1864
1865	ext4_debug("DEL_RANGE, inode %ld, lblk %d, len %d\n",
1866			inode->i_ino, le32_to_cpu(lrange.fc_lblk),
1867			le32_to_cpu(lrange.fc_len));
1868	while (remaining > 0) {
1869		map.m_lblk = cur;
1870		map.m_len = remaining;
1871
1872		ret = ext4_map_blocks(NULL, inode, &map, 0);
1873		if (ret < 0)
1874			goto out;
 
 
1875		if (ret > 0) {
1876			remaining -= ret;
1877			cur += ret;
1878			ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, false);
1879		} else {
1880			remaining -= map.m_len;
1881			cur += map.m_len;
1882		}
1883	}
1884
1885	down_write(&EXT4_I(inode)->i_data_sem);
1886	ret = ext4_ext_remove_space(inode, le32_to_cpu(lrange.fc_lblk),
1887				le32_to_cpu(lrange.fc_lblk) +
1888				le32_to_cpu(lrange.fc_len) - 1);
1889	up_write(&EXT4_I(inode)->i_data_sem);
1890	if (ret)
1891		goto out;
1892	ext4_ext_replay_shrink_inode(inode,
1893		i_size_read(inode) >> sb->s_blocksize_bits);
1894	ext4_mark_inode_dirty(NULL, inode);
1895out:
1896	iput(inode);
 
1897	return 0;
1898}
1899
1900static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb)
1901{
1902	struct ext4_fc_replay_state *state;
1903	struct inode *inode;
1904	struct ext4_ext_path *path = NULL;
1905	struct ext4_map_blocks map;
1906	int i, ret, j;
1907	ext4_lblk_t cur, end;
1908
1909	state = &EXT4_SB(sb)->s_fc_replay_state;
1910	for (i = 0; i < state->fc_modified_inodes_used; i++) {
1911		inode = ext4_iget(sb, state->fc_modified_inodes[i],
1912			EXT4_IGET_NORMAL);
1913		if (IS_ERR(inode)) {
1914			ext4_debug("Inode %d not found.",
1915				state->fc_modified_inodes[i]);
1916			continue;
1917		}
1918		cur = 0;
1919		end = EXT_MAX_BLOCKS;
1920		if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) {
1921			iput(inode);
1922			continue;
1923		}
1924		while (cur < end) {
1925			map.m_lblk = cur;
1926			map.m_len = end - cur;
1927
1928			ret = ext4_map_blocks(NULL, inode, &map, 0);
1929			if (ret < 0)
1930				break;
1931
1932			if (ret > 0) {
1933				path = ext4_find_extent(inode, map.m_lblk, NULL, 0);
1934				if (!IS_ERR(path)) {
1935					for (j = 0; j < path->p_depth; j++)
1936						ext4_mb_mark_bb(inode->i_sb,
1937							path[j].p_block, 1, true);
1938					ext4_free_ext_path(path);
 
1939				}
1940				cur += ret;
1941				ext4_mb_mark_bb(inode->i_sb, map.m_pblk,
1942							map.m_len, true);
1943			} else {
1944				cur = cur + (map.m_len ? map.m_len : 1);
1945			}
1946		}
1947		iput(inode);
1948	}
1949}
1950
1951/*
1952 * Check if block is in excluded regions for block allocation. The simple
1953 * allocator that runs during replay phase is calls this function to see
1954 * if it is okay to use a block.
1955 */
1956bool ext4_fc_replay_check_excluded(struct super_block *sb, ext4_fsblk_t blk)
1957{
1958	int i;
1959	struct ext4_fc_replay_state *state;
1960
1961	state = &EXT4_SB(sb)->s_fc_replay_state;
1962	for (i = 0; i < state->fc_regions_valid; i++) {
1963		if (state->fc_regions[i].ino == 0 ||
1964			state->fc_regions[i].len == 0)
1965			continue;
1966		if (in_range(blk, state->fc_regions[i].pblk,
1967					state->fc_regions[i].len))
1968			return true;
1969	}
1970	return false;
1971}
1972
1973/* Cleanup function called after replay */
1974void ext4_fc_replay_cleanup(struct super_block *sb)
1975{
1976	struct ext4_sb_info *sbi = EXT4_SB(sb);
1977
1978	sbi->s_mount_state &= ~EXT4_FC_REPLAY;
1979	kfree(sbi->s_fc_replay_state.fc_regions);
1980	kfree(sbi->s_fc_replay_state.fc_modified_inodes);
1981}
1982
1983static bool ext4_fc_value_len_isvalid(struct ext4_sb_info *sbi,
1984				      int tag, int len)
1985{
1986	switch (tag) {
1987	case EXT4_FC_TAG_ADD_RANGE:
1988		return len == sizeof(struct ext4_fc_add_range);
1989	case EXT4_FC_TAG_DEL_RANGE:
1990		return len == sizeof(struct ext4_fc_del_range);
1991	case EXT4_FC_TAG_CREAT:
1992	case EXT4_FC_TAG_LINK:
1993	case EXT4_FC_TAG_UNLINK:
1994		len -= sizeof(struct ext4_fc_dentry_info);
1995		return len >= 1 && len <= EXT4_NAME_LEN;
1996	case EXT4_FC_TAG_INODE:
1997		len -= sizeof(struct ext4_fc_inode);
1998		return len >= EXT4_GOOD_OLD_INODE_SIZE &&
1999			len <= sbi->s_inode_size;
2000	case EXT4_FC_TAG_PAD:
2001		return true; /* padding can have any length */
2002	case EXT4_FC_TAG_TAIL:
2003		return len >= sizeof(struct ext4_fc_tail);
2004	case EXT4_FC_TAG_HEAD:
2005		return len == sizeof(struct ext4_fc_head);
2006	}
2007	return false;
2008}
2009
2010/*
2011 * Recovery Scan phase handler
2012 *
2013 * This function is called during the scan phase and is responsible
2014 * for doing following things:
2015 * - Make sure the fast commit area has valid tags for replay
2016 * - Count number of tags that need to be replayed by the replay handler
2017 * - Verify CRC
2018 * - Create a list of excluded blocks for allocation during replay phase
2019 *
2020 * This function returns JBD2_FC_REPLAY_CONTINUE to indicate that SCAN is
2021 * incomplete and JBD2 should send more blocks. It returns JBD2_FC_REPLAY_STOP
2022 * to indicate that scan has finished and JBD2 can now start replay phase.
2023 * It returns a negative error to indicate that there was an error. At the end
2024 * of a successful scan phase, sbi->s_fc_replay_state.fc_replay_num_tags is set
2025 * to indicate the number of tags that need to replayed during the replay phase.
2026 */
2027static int ext4_fc_replay_scan(journal_t *journal,
2028				struct buffer_head *bh, int off,
2029				tid_t expected_tid)
2030{
2031	struct super_block *sb = journal->j_private;
2032	struct ext4_sb_info *sbi = EXT4_SB(sb);
2033	struct ext4_fc_replay_state *state;
2034	int ret = JBD2_FC_REPLAY_CONTINUE;
2035	struct ext4_fc_add_range ext;
2036	struct ext4_fc_tl_mem tl;
2037	struct ext4_fc_tail tail;
2038	__u8 *start, *end, *cur, *val;
2039	struct ext4_fc_head head;
2040	struct ext4_extent *ex;
2041
2042	state = &sbi->s_fc_replay_state;
2043
2044	start = (u8 *)bh->b_data;
2045	end = start + journal->j_blocksize;
2046
2047	if (state->fc_replay_expected_off == 0) {
2048		state->fc_cur_tag = 0;
2049		state->fc_replay_num_tags = 0;
2050		state->fc_crc = 0;
2051		state->fc_regions = NULL;
2052		state->fc_regions_valid = state->fc_regions_used =
2053			state->fc_regions_size = 0;
2054		/* Check if we can stop early */
2055		if (le16_to_cpu(((struct ext4_fc_tl *)start)->fc_tag)
2056			!= EXT4_FC_TAG_HEAD)
2057			return 0;
2058	}
2059
2060	if (off != state->fc_replay_expected_off) {
2061		ret = -EFSCORRUPTED;
2062		goto out_err;
2063	}
2064
2065	state->fc_replay_expected_off++;
2066	for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN;
2067	     cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) {
2068		ext4_fc_get_tl(&tl, cur);
2069		val = cur + EXT4_FC_TAG_BASE_LEN;
2070		if (tl.fc_len > end - val ||
2071		    !ext4_fc_value_len_isvalid(sbi, tl.fc_tag, tl.fc_len)) {
2072			ret = state->fc_replay_num_tags ?
2073				JBD2_FC_REPLAY_STOP : -ECANCELED;
2074			goto out_err;
2075		}
2076		ext4_debug("Scan phase, tag:%s, blk %lld\n",
2077			   tag2str(tl.fc_tag), bh->b_blocknr);
2078		switch (tl.fc_tag) {
2079		case EXT4_FC_TAG_ADD_RANGE:
2080			memcpy(&ext, val, sizeof(ext));
2081			ex = (struct ext4_extent *)&ext.fc_ex;
2082			ret = ext4_fc_record_regions(sb,
2083				le32_to_cpu(ext.fc_ino),
2084				le32_to_cpu(ex->ee_block), ext4_ext_pblock(ex),
2085				ext4_ext_get_actual_len(ex), 0);
2086			if (ret < 0)
2087				break;
2088			ret = JBD2_FC_REPLAY_CONTINUE;
2089			fallthrough;
2090		case EXT4_FC_TAG_DEL_RANGE:
2091		case EXT4_FC_TAG_LINK:
2092		case EXT4_FC_TAG_UNLINK:
2093		case EXT4_FC_TAG_CREAT:
2094		case EXT4_FC_TAG_INODE:
2095		case EXT4_FC_TAG_PAD:
2096			state->fc_cur_tag++;
2097			state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
2098				EXT4_FC_TAG_BASE_LEN + tl.fc_len);
2099			break;
2100		case EXT4_FC_TAG_TAIL:
2101			state->fc_cur_tag++;
2102			memcpy(&tail, val, sizeof(tail));
2103			state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
2104						EXT4_FC_TAG_BASE_LEN +
2105						offsetof(struct ext4_fc_tail,
2106						fc_crc));
2107			if (le32_to_cpu(tail.fc_tid) == expected_tid &&
2108				le32_to_cpu(tail.fc_crc) == state->fc_crc) {
2109				state->fc_replay_num_tags = state->fc_cur_tag;
2110				state->fc_regions_valid =
2111					state->fc_regions_used;
2112			} else {
2113				ret = state->fc_replay_num_tags ?
2114					JBD2_FC_REPLAY_STOP : -EFSBADCRC;
2115			}
2116			state->fc_crc = 0;
2117			break;
2118		case EXT4_FC_TAG_HEAD:
2119			memcpy(&head, val, sizeof(head));
2120			if (le32_to_cpu(head.fc_features) &
2121				~EXT4_FC_SUPPORTED_FEATURES) {
2122				ret = -EOPNOTSUPP;
2123				break;
2124			}
2125			if (le32_to_cpu(head.fc_tid) != expected_tid) {
2126				ret = JBD2_FC_REPLAY_STOP;
2127				break;
2128			}
2129			state->fc_cur_tag++;
2130			state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
2131				EXT4_FC_TAG_BASE_LEN + tl.fc_len);
2132			break;
2133		default:
2134			ret = state->fc_replay_num_tags ?
2135				JBD2_FC_REPLAY_STOP : -ECANCELED;
2136		}
2137		if (ret < 0 || ret == JBD2_FC_REPLAY_STOP)
2138			break;
2139	}
2140
2141out_err:
2142	trace_ext4_fc_replay_scan(sb, ret, off);
2143	return ret;
2144}
2145
2146/*
2147 * Main recovery path entry point.
2148 * The meaning of return codes is similar as above.
2149 */
2150static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh,
2151				enum passtype pass, int off, tid_t expected_tid)
2152{
2153	struct super_block *sb = journal->j_private;
2154	struct ext4_sb_info *sbi = EXT4_SB(sb);
2155	struct ext4_fc_tl_mem tl;
2156	__u8 *start, *end, *cur, *val;
2157	int ret = JBD2_FC_REPLAY_CONTINUE;
2158	struct ext4_fc_replay_state *state = &sbi->s_fc_replay_state;
2159	struct ext4_fc_tail tail;
2160
2161	if (pass == PASS_SCAN) {
2162		state->fc_current_pass = PASS_SCAN;
2163		return ext4_fc_replay_scan(journal, bh, off, expected_tid);
2164	}
2165
2166	if (state->fc_current_pass != pass) {
2167		state->fc_current_pass = pass;
2168		sbi->s_mount_state |= EXT4_FC_REPLAY;
2169	}
2170	if (!sbi->s_fc_replay_state.fc_replay_num_tags) {
2171		ext4_debug("Replay stops\n");
2172		ext4_fc_set_bitmaps_and_counters(sb);
2173		return 0;
2174	}
2175
2176#ifdef CONFIG_EXT4_DEBUG
2177	if (sbi->s_fc_debug_max_replay && off >= sbi->s_fc_debug_max_replay) {
2178		pr_warn("Dropping fc block %d because max_replay set\n", off);
2179		return JBD2_FC_REPLAY_STOP;
2180	}
2181#endif
2182
2183	start = (u8 *)bh->b_data;
2184	end = start + journal->j_blocksize;
2185
2186	for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN;
2187	     cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) {
2188		ext4_fc_get_tl(&tl, cur);
2189		val = cur + EXT4_FC_TAG_BASE_LEN;
2190
2191		if (state->fc_replay_num_tags == 0) {
2192			ret = JBD2_FC_REPLAY_STOP;
2193			ext4_fc_set_bitmaps_and_counters(sb);
2194			break;
2195		}
2196
2197		ext4_debug("Replay phase, tag:%s\n", tag2str(tl.fc_tag));
2198		state->fc_replay_num_tags--;
2199		switch (tl.fc_tag) {
2200		case EXT4_FC_TAG_LINK:
2201			ret = ext4_fc_replay_link(sb, &tl, val);
2202			break;
2203		case EXT4_FC_TAG_UNLINK:
2204			ret = ext4_fc_replay_unlink(sb, &tl, val);
2205			break;
2206		case EXT4_FC_TAG_ADD_RANGE:
2207			ret = ext4_fc_replay_add_range(sb, &tl, val);
2208			break;
2209		case EXT4_FC_TAG_CREAT:
2210			ret = ext4_fc_replay_create(sb, &tl, val);
2211			break;
2212		case EXT4_FC_TAG_DEL_RANGE:
2213			ret = ext4_fc_replay_del_range(sb, &tl, val);
2214			break;
2215		case EXT4_FC_TAG_INODE:
2216			ret = ext4_fc_replay_inode(sb, &tl, val);
2217			break;
2218		case EXT4_FC_TAG_PAD:
2219			trace_ext4_fc_replay(sb, EXT4_FC_TAG_PAD, 0,
2220					     tl.fc_len, 0);
2221			break;
2222		case EXT4_FC_TAG_TAIL:
2223			trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL,
2224					     0, tl.fc_len, 0);
2225			memcpy(&tail, val, sizeof(tail));
2226			WARN_ON(le32_to_cpu(tail.fc_tid) != expected_tid);
2227			break;
2228		case EXT4_FC_TAG_HEAD:
2229			break;
2230		default:
2231			trace_ext4_fc_replay(sb, tl.fc_tag, 0, tl.fc_len, 0);
 
2232			ret = -ECANCELED;
2233			break;
2234		}
2235		if (ret < 0)
2236			break;
2237		ret = JBD2_FC_REPLAY_CONTINUE;
2238	}
2239	return ret;
2240}
2241
2242void ext4_fc_init(struct super_block *sb, journal_t *journal)
2243{
2244	/*
2245	 * We set replay callback even if fast commit disabled because we may
2246	 * could still have fast commit blocks that need to be replayed even if
2247	 * fast commit has now been turned off.
2248	 */
2249	journal->j_fc_replay_callback = ext4_fc_replay;
2250	if (!test_opt2(sb, JOURNAL_FAST_COMMIT))
2251		return;
2252	journal->j_fc_cleanup_callback = ext4_fc_cleanup;
2253}
2254
2255static const char * const fc_ineligible_reasons[] = {
2256	[EXT4_FC_REASON_XATTR] = "Extended attributes changed",
2257	[EXT4_FC_REASON_CROSS_RENAME] = "Cross rename",
2258	[EXT4_FC_REASON_JOURNAL_FLAG_CHANGE] = "Journal flag changed",
2259	[EXT4_FC_REASON_NOMEM] = "Insufficient memory",
2260	[EXT4_FC_REASON_SWAP_BOOT] = "Swap boot",
2261	[EXT4_FC_REASON_RESIZE] = "Resize",
2262	[EXT4_FC_REASON_RENAME_DIR] = "Dir renamed",
2263	[EXT4_FC_REASON_FALLOC_RANGE] = "Falloc range op",
2264	[EXT4_FC_REASON_INODE_JOURNAL_DATA] = "Data journalling",
2265	[EXT4_FC_REASON_ENCRYPTED_FILENAME] = "Encrypted filename",
2266};
2267
2268int ext4_fc_info_show(struct seq_file *seq, void *v)
2269{
2270	struct ext4_sb_info *sbi = EXT4_SB((struct super_block *)seq->private);
2271	struct ext4_fc_stats *stats = &sbi->s_fc_stats;
2272	int i;
2273
2274	if (v != SEQ_START_TOKEN)
2275		return 0;
2276
2277	seq_printf(seq,
2278		"fc stats:\n%ld commits\n%ld ineligible\n%ld numblks\n%lluus avg_commit_time\n",
2279		   stats->fc_num_commits, stats->fc_ineligible_commits,
2280		   stats->fc_numblks,
2281		   div_u64(stats->s_fc_avg_commit_time, 1000));
2282	seq_puts(seq, "Ineligible reasons:\n");
2283	for (i = 0; i < EXT4_FC_REASON_MAX; i++)
2284		seq_printf(seq, "\"%s\":\t%d\n", fc_ineligible_reasons[i],
2285			stats->fc_ineligible_reason_count[i]);
2286
2287	return 0;
2288}
2289
2290int __init ext4_fc_init_dentry_cache(void)
2291{
2292	ext4_fc_dentry_cachep = KMEM_CACHE(ext4_fc_dentry_update,
2293					   SLAB_RECLAIM_ACCOUNT);
2294
2295	if (ext4_fc_dentry_cachep == NULL)
2296		return -ENOMEM;
2297
2298	return 0;
2299}
2300
2301void ext4_fc_destroy_dentry_cache(void)
2302{
2303	kmem_cache_destroy(ext4_fc_dentry_cachep);
2304}