Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * fs/f2fs/f2fs.h
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8#ifndef _LINUX_F2FS_H
9#define _LINUX_F2FS_H
10
11#include <linux/uio.h>
12#include <linux/types.h>
13#include <linux/page-flags.h>
14#include <linux/buffer_head.h>
15#include <linux/slab.h>
16#include <linux/crc32.h>
17#include <linux/magic.h>
18#include <linux/kobject.h>
19#include <linux/sched.h>
20#include <linux/cred.h>
21#include <linux/vmalloc.h>
22#include <linux/bio.h>
23#include <linux/blkdev.h>
24#include <linux/quotaops.h>
25#include <linux/part_stat.h>
26#include <crypto/hash.h>
27
28#include <linux/fscrypt.h>
29#include <linux/fsverity.h>
30
31#ifdef CONFIG_F2FS_CHECK_FS
32#define f2fs_bug_on(sbi, condition) BUG_ON(condition)
33#else
34#define f2fs_bug_on(sbi, condition) \
35 do { \
36 if (WARN_ON(condition)) \
37 set_sbi_flag(sbi, SBI_NEED_FSCK); \
38 } while (0)
39#endif
40
41enum {
42 FAULT_KMALLOC,
43 FAULT_KVMALLOC,
44 FAULT_PAGE_ALLOC,
45 FAULT_PAGE_GET,
46 FAULT_ALLOC_BIO, /* it's obsolete due to bio_alloc() will never fail */
47 FAULT_ALLOC_NID,
48 FAULT_ORPHAN,
49 FAULT_BLOCK,
50 FAULT_DIR_DEPTH,
51 FAULT_EVICT_INODE,
52 FAULT_TRUNCATE,
53 FAULT_READ_IO,
54 FAULT_CHECKPOINT,
55 FAULT_DISCARD,
56 FAULT_WRITE_IO,
57 FAULT_MAX,
58};
59
60#ifdef CONFIG_F2FS_FAULT_INJECTION
61#define F2FS_ALL_FAULT_TYPE ((1 << FAULT_MAX) - 1)
62
63struct f2fs_fault_info {
64 atomic_t inject_ops;
65 unsigned int inject_rate;
66 unsigned int inject_type;
67};
68
69extern const char *f2fs_fault_name[FAULT_MAX];
70#define IS_FAULT_SET(fi, type) ((fi)->inject_type & (1 << (type)))
71#endif
72
73/*
74 * For mount options
75 */
76#define F2FS_MOUNT_DISABLE_ROLL_FORWARD 0x00000002
77#define F2FS_MOUNT_DISCARD 0x00000004
78#define F2FS_MOUNT_NOHEAP 0x00000008
79#define F2FS_MOUNT_XATTR_USER 0x00000010
80#define F2FS_MOUNT_POSIX_ACL 0x00000020
81#define F2FS_MOUNT_DISABLE_EXT_IDENTIFY 0x00000040
82#define F2FS_MOUNT_INLINE_XATTR 0x00000080
83#define F2FS_MOUNT_INLINE_DATA 0x00000100
84#define F2FS_MOUNT_INLINE_DENTRY 0x00000200
85#define F2FS_MOUNT_FLUSH_MERGE 0x00000400
86#define F2FS_MOUNT_NOBARRIER 0x00000800
87#define F2FS_MOUNT_FASTBOOT 0x00001000
88#define F2FS_MOUNT_EXTENT_CACHE 0x00002000
89#define F2FS_MOUNT_DATA_FLUSH 0x00008000
90#define F2FS_MOUNT_FAULT_INJECTION 0x00010000
91#define F2FS_MOUNT_USRQUOTA 0x00080000
92#define F2FS_MOUNT_GRPQUOTA 0x00100000
93#define F2FS_MOUNT_PRJQUOTA 0x00200000
94#define F2FS_MOUNT_QUOTA 0x00400000
95#define F2FS_MOUNT_INLINE_XATTR_SIZE 0x00800000
96#define F2FS_MOUNT_RESERVE_ROOT 0x01000000
97#define F2FS_MOUNT_DISABLE_CHECKPOINT 0x02000000
98#define F2FS_MOUNT_NORECOVERY 0x04000000
99#define F2FS_MOUNT_ATGC 0x08000000
100#define F2FS_MOUNT_MERGE_CHECKPOINT 0x10000000
101#define F2FS_MOUNT_GC_MERGE 0x20000000
102#define F2FS_MOUNT_COMPRESS_CACHE 0x40000000
103
104#define F2FS_OPTION(sbi) ((sbi)->mount_opt)
105#define clear_opt(sbi, option) (F2FS_OPTION(sbi).opt &= ~F2FS_MOUNT_##option)
106#define set_opt(sbi, option) (F2FS_OPTION(sbi).opt |= F2FS_MOUNT_##option)
107#define test_opt(sbi, option) (F2FS_OPTION(sbi).opt & F2FS_MOUNT_##option)
108
109#define ver_after(a, b) (typecheck(unsigned long long, a) && \
110 typecheck(unsigned long long, b) && \
111 ((long long)((a) - (b)) > 0))
112
113typedef u32 block_t; /*
114 * should not change u32, since it is the on-disk block
115 * address format, __le32.
116 */
117typedef u32 nid_t;
118
119#define COMPRESS_EXT_NUM 16
120
121struct f2fs_mount_info {
122 unsigned int opt;
123 int write_io_size_bits; /* Write IO size bits */
124 block_t root_reserved_blocks; /* root reserved blocks */
125 kuid_t s_resuid; /* reserved blocks for uid */
126 kgid_t s_resgid; /* reserved blocks for gid */
127 int active_logs; /* # of active logs */
128 int inline_xattr_size; /* inline xattr size */
129#ifdef CONFIG_F2FS_FAULT_INJECTION
130 struct f2fs_fault_info fault_info; /* For fault injection */
131#endif
132#ifdef CONFIG_QUOTA
133 /* Names of quota files with journalled quota */
134 char *s_qf_names[MAXQUOTAS];
135 int s_jquota_fmt; /* Format of quota to use */
136#endif
137 /* For which write hints are passed down to block layer */
138 int whint_mode;
139 int alloc_mode; /* segment allocation policy */
140 int fsync_mode; /* fsync policy */
141 int fs_mode; /* fs mode: LFS or ADAPTIVE */
142 int bggc_mode; /* bggc mode: off, on or sync */
143 struct fscrypt_dummy_policy dummy_enc_policy; /* test dummy encryption */
144 block_t unusable_cap_perc; /* percentage for cap */
145 block_t unusable_cap; /* Amount of space allowed to be
146 * unusable when disabling checkpoint
147 */
148
149 /* For compression */
150 unsigned char compress_algorithm; /* algorithm type */
151 unsigned char compress_log_size; /* cluster log size */
152 unsigned char compress_level; /* compress level */
153 bool compress_chksum; /* compressed data chksum */
154 unsigned char compress_ext_cnt; /* extension count */
155 unsigned char nocompress_ext_cnt; /* nocompress extension count */
156 int compress_mode; /* compression mode */
157 unsigned char extensions[COMPRESS_EXT_NUM][F2FS_EXTENSION_LEN]; /* extensions */
158 unsigned char noextensions[COMPRESS_EXT_NUM][F2FS_EXTENSION_LEN]; /* extensions */
159};
160
161#define F2FS_FEATURE_ENCRYPT 0x0001
162#define F2FS_FEATURE_BLKZONED 0x0002
163#define F2FS_FEATURE_ATOMIC_WRITE 0x0004
164#define F2FS_FEATURE_EXTRA_ATTR 0x0008
165#define F2FS_FEATURE_PRJQUOTA 0x0010
166#define F2FS_FEATURE_INODE_CHKSUM 0x0020
167#define F2FS_FEATURE_FLEXIBLE_INLINE_XATTR 0x0040
168#define F2FS_FEATURE_QUOTA_INO 0x0080
169#define F2FS_FEATURE_INODE_CRTIME 0x0100
170#define F2FS_FEATURE_LOST_FOUND 0x0200
171#define F2FS_FEATURE_VERITY 0x0400
172#define F2FS_FEATURE_SB_CHKSUM 0x0800
173#define F2FS_FEATURE_CASEFOLD 0x1000
174#define F2FS_FEATURE_COMPRESSION 0x2000
175#define F2FS_FEATURE_RO 0x4000
176
177#define __F2FS_HAS_FEATURE(raw_super, mask) \
178 ((raw_super->feature & cpu_to_le32(mask)) != 0)
179#define F2FS_HAS_FEATURE(sbi, mask) __F2FS_HAS_FEATURE(sbi->raw_super, mask)
180#define F2FS_SET_FEATURE(sbi, mask) \
181 (sbi->raw_super->feature |= cpu_to_le32(mask))
182#define F2FS_CLEAR_FEATURE(sbi, mask) \
183 (sbi->raw_super->feature &= ~cpu_to_le32(mask))
184
185/*
186 * Default values for user and/or group using reserved blocks
187 */
188#define F2FS_DEF_RESUID 0
189#define F2FS_DEF_RESGID 0
190
191/*
192 * For checkpoint manager
193 */
194enum {
195 NAT_BITMAP,
196 SIT_BITMAP
197};
198
199#define CP_UMOUNT 0x00000001
200#define CP_FASTBOOT 0x00000002
201#define CP_SYNC 0x00000004
202#define CP_RECOVERY 0x00000008
203#define CP_DISCARD 0x00000010
204#define CP_TRIMMED 0x00000020
205#define CP_PAUSE 0x00000040
206#define CP_RESIZE 0x00000080
207
208#define MAX_DISCARD_BLOCKS(sbi) BLKS_PER_SEC(sbi)
209#define DEF_MAX_DISCARD_REQUEST 8 /* issue 8 discards per round */
210#define DEF_MIN_DISCARD_ISSUE_TIME 50 /* 50 ms, if exists */
211#define DEF_MID_DISCARD_ISSUE_TIME 500 /* 500 ms, if device busy */
212#define DEF_MAX_DISCARD_ISSUE_TIME 60000 /* 60 s, if no candidates */
213#define DEF_DISCARD_URGENT_UTIL 80 /* do more discard over 80% */
214#define DEF_CP_INTERVAL 60 /* 60 secs */
215#define DEF_IDLE_INTERVAL 5 /* 5 secs */
216#define DEF_DISABLE_INTERVAL 5 /* 5 secs */
217#define DEF_DISABLE_QUICK_INTERVAL 1 /* 1 secs */
218#define DEF_UMOUNT_DISCARD_TIMEOUT 5 /* 5 secs */
219
220struct cp_control {
221 int reason;
222 __u64 trim_start;
223 __u64 trim_end;
224 __u64 trim_minlen;
225};
226
227/*
228 * indicate meta/data type
229 */
230enum {
231 META_CP,
232 META_NAT,
233 META_SIT,
234 META_SSA,
235 META_MAX,
236 META_POR,
237 DATA_GENERIC, /* check range only */
238 DATA_GENERIC_ENHANCE, /* strong check on range and segment bitmap */
239 DATA_GENERIC_ENHANCE_READ, /*
240 * strong check on range and segment
241 * bitmap but no warning due to race
242 * condition of read on truncated area
243 * by extent_cache
244 */
245 META_GENERIC,
246};
247
248/* for the list of ino */
249enum {
250 ORPHAN_INO, /* for orphan ino list */
251 APPEND_INO, /* for append ino list */
252 UPDATE_INO, /* for update ino list */
253 TRANS_DIR_INO, /* for trasactions dir ino list */
254 FLUSH_INO, /* for multiple device flushing */
255 MAX_INO_ENTRY, /* max. list */
256};
257
258struct ino_entry {
259 struct list_head list; /* list head */
260 nid_t ino; /* inode number */
261 unsigned int dirty_device; /* dirty device bitmap */
262};
263
264/* for the list of inodes to be GCed */
265struct inode_entry {
266 struct list_head list; /* list head */
267 struct inode *inode; /* vfs inode pointer */
268};
269
270struct fsync_node_entry {
271 struct list_head list; /* list head */
272 struct page *page; /* warm node page pointer */
273 unsigned int seq_id; /* sequence id */
274};
275
276struct ckpt_req {
277 struct completion wait; /* completion for checkpoint done */
278 struct llist_node llnode; /* llist_node to be linked in wait queue */
279 int ret; /* return code of checkpoint */
280 ktime_t queue_time; /* request queued time */
281};
282
283struct ckpt_req_control {
284 struct task_struct *f2fs_issue_ckpt; /* checkpoint task */
285 int ckpt_thread_ioprio; /* checkpoint merge thread ioprio */
286 wait_queue_head_t ckpt_wait_queue; /* waiting queue for wake-up */
287 atomic_t issued_ckpt; /* # of actually issued ckpts */
288 atomic_t total_ckpt; /* # of total ckpts */
289 atomic_t queued_ckpt; /* # of queued ckpts */
290 struct llist_head issue_list; /* list for command issue */
291 spinlock_t stat_lock; /* lock for below checkpoint time stats */
292 unsigned int cur_time; /* cur wait time in msec for currently issued checkpoint */
293 unsigned int peak_time; /* peak wait time in msec until now */
294};
295
296/* for the bitmap indicate blocks to be discarded */
297struct discard_entry {
298 struct list_head list; /* list head */
299 block_t start_blkaddr; /* start blockaddr of current segment */
300 unsigned char discard_map[SIT_VBLOCK_MAP_SIZE]; /* segment discard bitmap */
301};
302
303/* default discard granularity of inner discard thread, unit: block count */
304#define DEFAULT_DISCARD_GRANULARITY 16
305
306/* max discard pend list number */
307#define MAX_PLIST_NUM 512
308#define plist_idx(blk_num) ((blk_num) >= MAX_PLIST_NUM ? \
309 (MAX_PLIST_NUM - 1) : ((blk_num) - 1))
310
311enum {
312 D_PREP, /* initial */
313 D_PARTIAL, /* partially submitted */
314 D_SUBMIT, /* all submitted */
315 D_DONE, /* finished */
316};
317
318struct discard_info {
319 block_t lstart; /* logical start address */
320 block_t len; /* length */
321 block_t start; /* actual start address in dev */
322};
323
324struct discard_cmd {
325 struct rb_node rb_node; /* rb node located in rb-tree */
326 union {
327 struct {
328 block_t lstart; /* logical start address */
329 block_t len; /* length */
330 block_t start; /* actual start address in dev */
331 };
332 struct discard_info di; /* discard info */
333
334 };
335 struct list_head list; /* command list */
336 struct completion wait; /* compleation */
337 struct block_device *bdev; /* bdev */
338 unsigned short ref; /* reference count */
339 unsigned char state; /* state */
340 unsigned char queued; /* queued discard */
341 int error; /* bio error */
342 spinlock_t lock; /* for state/bio_ref updating */
343 unsigned short bio_ref; /* bio reference count */
344};
345
346enum {
347 DPOLICY_BG,
348 DPOLICY_FORCE,
349 DPOLICY_FSTRIM,
350 DPOLICY_UMOUNT,
351 MAX_DPOLICY,
352};
353
354struct discard_policy {
355 int type; /* type of discard */
356 unsigned int min_interval; /* used for candidates exist */
357 unsigned int mid_interval; /* used for device busy */
358 unsigned int max_interval; /* used for candidates not exist */
359 unsigned int max_requests; /* # of discards issued per round */
360 unsigned int io_aware_gran; /* minimum granularity discard not be aware of I/O */
361 bool io_aware; /* issue discard in idle time */
362 bool sync; /* submit discard with REQ_SYNC flag */
363 bool ordered; /* issue discard by lba order */
364 bool timeout; /* discard timeout for put_super */
365 unsigned int granularity; /* discard granularity */
366};
367
368struct discard_cmd_control {
369 struct task_struct *f2fs_issue_discard; /* discard thread */
370 struct list_head entry_list; /* 4KB discard entry list */
371 struct list_head pend_list[MAX_PLIST_NUM];/* store pending entries */
372 struct list_head wait_list; /* store on-flushing entries */
373 struct list_head fstrim_list; /* in-flight discard from fstrim */
374 wait_queue_head_t discard_wait_queue; /* waiting queue for wake-up */
375 unsigned int discard_wake; /* to wake up discard thread */
376 struct mutex cmd_lock;
377 unsigned int nr_discards; /* # of discards in the list */
378 unsigned int max_discards; /* max. discards to be issued */
379 unsigned int discard_granularity; /* discard granularity */
380 unsigned int undiscard_blks; /* # of undiscard blocks */
381 unsigned int next_pos; /* next discard position */
382 atomic_t issued_discard; /* # of issued discard */
383 atomic_t queued_discard; /* # of queued discard */
384 atomic_t discard_cmd_cnt; /* # of cached cmd count */
385 struct rb_root_cached root; /* root of discard rb-tree */
386 bool rbtree_check; /* config for consistence check */
387};
388
389/* for the list of fsync inodes, used only during recovery */
390struct fsync_inode_entry {
391 struct list_head list; /* list head */
392 struct inode *inode; /* vfs inode pointer */
393 block_t blkaddr; /* block address locating the last fsync */
394 block_t last_dentry; /* block address locating the last dentry */
395};
396
397#define nats_in_cursum(jnl) (le16_to_cpu((jnl)->n_nats))
398#define sits_in_cursum(jnl) (le16_to_cpu((jnl)->n_sits))
399
400#define nat_in_journal(jnl, i) ((jnl)->nat_j.entries[i].ne)
401#define nid_in_journal(jnl, i) ((jnl)->nat_j.entries[i].nid)
402#define sit_in_journal(jnl, i) ((jnl)->sit_j.entries[i].se)
403#define segno_in_journal(jnl, i) ((jnl)->sit_j.entries[i].segno)
404
405#define MAX_NAT_JENTRIES(jnl) (NAT_JOURNAL_ENTRIES - nats_in_cursum(jnl))
406#define MAX_SIT_JENTRIES(jnl) (SIT_JOURNAL_ENTRIES - sits_in_cursum(jnl))
407
408static inline int update_nats_in_cursum(struct f2fs_journal *journal, int i)
409{
410 int before = nats_in_cursum(journal);
411
412 journal->n_nats = cpu_to_le16(before + i);
413 return before;
414}
415
416static inline int update_sits_in_cursum(struct f2fs_journal *journal, int i)
417{
418 int before = sits_in_cursum(journal);
419
420 journal->n_sits = cpu_to_le16(before + i);
421 return before;
422}
423
424static inline bool __has_cursum_space(struct f2fs_journal *journal,
425 int size, int type)
426{
427 if (type == NAT_JOURNAL)
428 return size <= MAX_NAT_JENTRIES(journal);
429 return size <= MAX_SIT_JENTRIES(journal);
430}
431
432/* for inline stuff */
433#define DEF_INLINE_RESERVED_SIZE 1
434static inline int get_extra_isize(struct inode *inode);
435static inline int get_inline_xattr_addrs(struct inode *inode);
436#define MAX_INLINE_DATA(inode) (sizeof(__le32) * \
437 (CUR_ADDRS_PER_INODE(inode) - \
438 get_inline_xattr_addrs(inode) - \
439 DEF_INLINE_RESERVED_SIZE))
440
441/* for inline dir */
442#define NR_INLINE_DENTRY(inode) (MAX_INLINE_DATA(inode) * BITS_PER_BYTE / \
443 ((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * \
444 BITS_PER_BYTE + 1))
445#define INLINE_DENTRY_BITMAP_SIZE(inode) \
446 DIV_ROUND_UP(NR_INLINE_DENTRY(inode), BITS_PER_BYTE)
447#define INLINE_RESERVED_SIZE(inode) (MAX_INLINE_DATA(inode) - \
448 ((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * \
449 NR_INLINE_DENTRY(inode) + \
450 INLINE_DENTRY_BITMAP_SIZE(inode)))
451
452/*
453 * For INODE and NODE manager
454 */
455/* for directory operations */
456
457struct f2fs_filename {
458 /*
459 * The filename the user specified. This is NULL for some
460 * filesystem-internal operations, e.g. converting an inline directory
461 * to a non-inline one, or roll-forward recovering an encrypted dentry.
462 */
463 const struct qstr *usr_fname;
464
465 /*
466 * The on-disk filename. For encrypted directories, this is encrypted.
467 * This may be NULL for lookups in an encrypted dir without the key.
468 */
469 struct fscrypt_str disk_name;
470
471 /* The dirhash of this filename */
472 f2fs_hash_t hash;
473
474#ifdef CONFIG_FS_ENCRYPTION
475 /*
476 * For lookups in encrypted directories: either the buffer backing
477 * disk_name, or a buffer that holds the decoded no-key name.
478 */
479 struct fscrypt_str crypto_buf;
480#endif
481#ifdef CONFIG_UNICODE
482 /*
483 * For casefolded directories: the casefolded name, but it's left NULL
484 * if the original name is not valid Unicode, if the directory is both
485 * casefolded and encrypted and its encryption key is unavailable, or if
486 * the filesystem is doing an internal operation where usr_fname is also
487 * NULL. In all these cases we fall back to treating the name as an
488 * opaque byte sequence.
489 */
490 struct fscrypt_str cf_name;
491#endif
492};
493
494struct f2fs_dentry_ptr {
495 struct inode *inode;
496 void *bitmap;
497 struct f2fs_dir_entry *dentry;
498 __u8 (*filename)[F2FS_SLOT_LEN];
499 int max;
500 int nr_bitmap;
501};
502
503static inline void make_dentry_ptr_block(struct inode *inode,
504 struct f2fs_dentry_ptr *d, struct f2fs_dentry_block *t)
505{
506 d->inode = inode;
507 d->max = NR_DENTRY_IN_BLOCK;
508 d->nr_bitmap = SIZE_OF_DENTRY_BITMAP;
509 d->bitmap = t->dentry_bitmap;
510 d->dentry = t->dentry;
511 d->filename = t->filename;
512}
513
514static inline void make_dentry_ptr_inline(struct inode *inode,
515 struct f2fs_dentry_ptr *d, void *t)
516{
517 int entry_cnt = NR_INLINE_DENTRY(inode);
518 int bitmap_size = INLINE_DENTRY_BITMAP_SIZE(inode);
519 int reserved_size = INLINE_RESERVED_SIZE(inode);
520
521 d->inode = inode;
522 d->max = entry_cnt;
523 d->nr_bitmap = bitmap_size;
524 d->bitmap = t;
525 d->dentry = t + bitmap_size + reserved_size;
526 d->filename = t + bitmap_size + reserved_size +
527 SIZE_OF_DIR_ENTRY * entry_cnt;
528}
529
530/*
531 * XATTR_NODE_OFFSET stores xattrs to one node block per file keeping -1
532 * as its node offset to distinguish from index node blocks.
533 * But some bits are used to mark the node block.
534 */
535#define XATTR_NODE_OFFSET ((((unsigned int)-1) << OFFSET_BIT_SHIFT) \
536 >> OFFSET_BIT_SHIFT)
537enum {
538 ALLOC_NODE, /* allocate a new node page if needed */
539 LOOKUP_NODE, /* look up a node without readahead */
540 LOOKUP_NODE_RA, /*
541 * look up a node with readahead called
542 * by get_data_block.
543 */
544};
545
546#define DEFAULT_RETRY_IO_COUNT 8 /* maximum retry read IO count */
547
548/* congestion wait timeout value, default: 20ms */
549#define DEFAULT_IO_TIMEOUT (msecs_to_jiffies(20))
550
551/* maximum retry quota flush count */
552#define DEFAULT_RETRY_QUOTA_FLUSH_COUNT 8
553
554#define F2FS_LINK_MAX 0xffffffff /* maximum link count per file */
555
556#define MAX_DIR_RA_PAGES 4 /* maximum ra pages of dir */
557
558/* for in-memory extent cache entry */
559#define F2FS_MIN_EXTENT_LEN 64 /* minimum extent length */
560
561/* number of extent info in extent cache we try to shrink */
562#define EXTENT_CACHE_SHRINK_NUMBER 128
563
564struct rb_entry {
565 struct rb_node rb_node; /* rb node located in rb-tree */
566 union {
567 struct {
568 unsigned int ofs; /* start offset of the entry */
569 unsigned int len; /* length of the entry */
570 };
571 unsigned long long key; /* 64-bits key */
572 } __packed;
573};
574
575struct extent_info {
576 unsigned int fofs; /* start offset in a file */
577 unsigned int len; /* length of the extent */
578 u32 blk; /* start block address of the extent */
579};
580
581struct extent_node {
582 struct rb_node rb_node; /* rb node located in rb-tree */
583 struct extent_info ei; /* extent info */
584 struct list_head list; /* node in global extent list of sbi */
585 struct extent_tree *et; /* extent tree pointer */
586};
587
588struct extent_tree {
589 nid_t ino; /* inode number */
590 struct rb_root_cached root; /* root of extent info rb-tree */
591 struct extent_node *cached_en; /* recently accessed extent node */
592 struct extent_info largest; /* largested extent info */
593 struct list_head list; /* to be used by sbi->zombie_list */
594 rwlock_t lock; /* protect extent info rb-tree */
595 atomic_t node_cnt; /* # of extent node in rb-tree*/
596 bool largest_updated; /* largest extent updated */
597};
598
599/*
600 * This structure is taken from ext4_map_blocks.
601 *
602 * Note that, however, f2fs uses NEW and MAPPED flags for f2fs_map_blocks().
603 */
604#define F2FS_MAP_NEW (1 << BH_New)
605#define F2FS_MAP_MAPPED (1 << BH_Mapped)
606#define F2FS_MAP_UNWRITTEN (1 << BH_Unwritten)
607#define F2FS_MAP_FLAGS (F2FS_MAP_NEW | F2FS_MAP_MAPPED |\
608 F2FS_MAP_UNWRITTEN)
609
610struct f2fs_map_blocks {
611 block_t m_pblk;
612 block_t m_lblk;
613 unsigned int m_len;
614 unsigned int m_flags;
615 pgoff_t *m_next_pgofs; /* point next possible non-hole pgofs */
616 pgoff_t *m_next_extent; /* point to next possible extent */
617 int m_seg_type;
618 bool m_may_create; /* indicate it is from write path */
619};
620
621/* for flag in get_data_block */
622enum {
623 F2FS_GET_BLOCK_DEFAULT,
624 F2FS_GET_BLOCK_FIEMAP,
625 F2FS_GET_BLOCK_BMAP,
626 F2FS_GET_BLOCK_DIO,
627 F2FS_GET_BLOCK_PRE_DIO,
628 F2FS_GET_BLOCK_PRE_AIO,
629 F2FS_GET_BLOCK_PRECACHE,
630};
631
632/*
633 * i_advise uses FADVISE_XXX_BIT. We can add additional hints later.
634 */
635#define FADVISE_COLD_BIT 0x01
636#define FADVISE_LOST_PINO_BIT 0x02
637#define FADVISE_ENCRYPT_BIT 0x04
638#define FADVISE_ENC_NAME_BIT 0x08
639#define FADVISE_KEEP_SIZE_BIT 0x10
640#define FADVISE_HOT_BIT 0x20
641#define FADVISE_VERITY_BIT 0x40
642
643#define FADVISE_MODIFIABLE_BITS (FADVISE_COLD_BIT | FADVISE_HOT_BIT)
644
645#define file_is_cold(inode) is_file(inode, FADVISE_COLD_BIT)
646#define file_set_cold(inode) set_file(inode, FADVISE_COLD_BIT)
647#define file_clear_cold(inode) clear_file(inode, FADVISE_COLD_BIT)
648
649#define file_wrong_pino(inode) is_file(inode, FADVISE_LOST_PINO_BIT)
650#define file_lost_pino(inode) set_file(inode, FADVISE_LOST_PINO_BIT)
651#define file_got_pino(inode) clear_file(inode, FADVISE_LOST_PINO_BIT)
652
653#define file_is_encrypt(inode) is_file(inode, FADVISE_ENCRYPT_BIT)
654#define file_set_encrypt(inode) set_file(inode, FADVISE_ENCRYPT_BIT)
655
656#define file_enc_name(inode) is_file(inode, FADVISE_ENC_NAME_BIT)
657#define file_set_enc_name(inode) set_file(inode, FADVISE_ENC_NAME_BIT)
658
659#define file_keep_isize(inode) is_file(inode, FADVISE_KEEP_SIZE_BIT)
660#define file_set_keep_isize(inode) set_file(inode, FADVISE_KEEP_SIZE_BIT)
661
662#define file_is_hot(inode) is_file(inode, FADVISE_HOT_BIT)
663#define file_set_hot(inode) set_file(inode, FADVISE_HOT_BIT)
664#define file_clear_hot(inode) clear_file(inode, FADVISE_HOT_BIT)
665
666#define file_is_verity(inode) is_file(inode, FADVISE_VERITY_BIT)
667#define file_set_verity(inode) set_file(inode, FADVISE_VERITY_BIT)
668
669#define DEF_DIR_LEVEL 0
670
671enum {
672 GC_FAILURE_PIN,
673 GC_FAILURE_ATOMIC,
674 MAX_GC_FAILURE
675};
676
677/* used for f2fs_inode_info->flags */
678enum {
679 FI_NEW_INODE, /* indicate newly allocated inode */
680 FI_DIRTY_INODE, /* indicate inode is dirty or not */
681 FI_AUTO_RECOVER, /* indicate inode is recoverable */
682 FI_DIRTY_DIR, /* indicate directory has dirty pages */
683 FI_INC_LINK, /* need to increment i_nlink */
684 FI_ACL_MODE, /* indicate acl mode */
685 FI_NO_ALLOC, /* should not allocate any blocks */
686 FI_FREE_NID, /* free allocated nide */
687 FI_NO_EXTENT, /* not to use the extent cache */
688 FI_INLINE_XATTR, /* used for inline xattr */
689 FI_INLINE_DATA, /* used for inline data*/
690 FI_INLINE_DENTRY, /* used for inline dentry */
691 FI_APPEND_WRITE, /* inode has appended data */
692 FI_UPDATE_WRITE, /* inode has in-place-update data */
693 FI_NEED_IPU, /* used for ipu per file */
694 FI_ATOMIC_FILE, /* indicate atomic file */
695 FI_ATOMIC_COMMIT, /* indicate the state of atomical committing */
696 FI_VOLATILE_FILE, /* indicate volatile file */
697 FI_FIRST_BLOCK_WRITTEN, /* indicate #0 data block was written */
698 FI_DROP_CACHE, /* drop dirty page cache */
699 FI_DATA_EXIST, /* indicate data exists */
700 FI_INLINE_DOTS, /* indicate inline dot dentries */
701 FI_DO_DEFRAG, /* indicate defragment is running */
702 FI_DIRTY_FILE, /* indicate regular/symlink has dirty pages */
703 FI_NO_PREALLOC, /* indicate skipped preallocated blocks */
704 FI_HOT_DATA, /* indicate file is hot */
705 FI_EXTRA_ATTR, /* indicate file has extra attribute */
706 FI_PROJ_INHERIT, /* indicate file inherits projectid */
707 FI_PIN_FILE, /* indicate file should not be gced */
708 FI_ATOMIC_REVOKE_REQUEST, /* request to drop atomic data */
709 FI_VERITY_IN_PROGRESS, /* building fs-verity Merkle tree */
710 FI_COMPRESSED_FILE, /* indicate file's data can be compressed */
711 FI_COMPRESS_CORRUPT, /* indicate compressed cluster is corrupted */
712 FI_MMAP_FILE, /* indicate file was mmapped */
713 FI_ENABLE_COMPRESS, /* enable compression in "user" compression mode */
714 FI_COMPRESS_RELEASED, /* compressed blocks were released */
715 FI_ALIGNED_WRITE, /* enable aligned write */
716 FI_MAX, /* max flag, never be used */
717};
718
719struct f2fs_inode_info {
720 struct inode vfs_inode; /* serve a vfs inode */
721 unsigned long i_flags; /* keep an inode flags for ioctl */
722 unsigned char i_advise; /* use to give file attribute hints */
723 unsigned char i_dir_level; /* use for dentry level for large dir */
724 unsigned int i_current_depth; /* only for directory depth */
725 /* for gc failure statistic */
726 unsigned int i_gc_failures[MAX_GC_FAILURE];
727 unsigned int i_pino; /* parent inode number */
728 umode_t i_acl_mode; /* keep file acl mode temporarily */
729
730 /* Use below internally in f2fs*/
731 unsigned long flags[BITS_TO_LONGS(FI_MAX)]; /* use to pass per-file flags */
732 struct rw_semaphore i_sem; /* protect fi info */
733 atomic_t dirty_pages; /* # of dirty pages */
734 f2fs_hash_t chash; /* hash value of given file name */
735 unsigned int clevel; /* maximum level of given file name */
736 struct task_struct *task; /* lookup and create consistency */
737 struct task_struct *cp_task; /* separate cp/wb IO stats*/
738 nid_t i_xattr_nid; /* node id that contains xattrs */
739 loff_t last_disk_size; /* lastly written file size */
740 spinlock_t i_size_lock; /* protect last_disk_size */
741
742#ifdef CONFIG_QUOTA
743 struct dquot *i_dquot[MAXQUOTAS];
744
745 /* quota space reservation, managed internally by quota code */
746 qsize_t i_reserved_quota;
747#endif
748 struct list_head dirty_list; /* dirty list for dirs and files */
749 struct list_head gdirty_list; /* linked in global dirty list */
750 struct list_head inmem_ilist; /* list for inmem inodes */
751 struct list_head inmem_pages; /* inmemory pages managed by f2fs */
752 struct task_struct *inmem_task; /* store inmemory task */
753 struct mutex inmem_lock; /* lock for inmemory pages */
754 struct extent_tree *extent_tree; /* cached extent_tree entry */
755
756 /* avoid racing between foreground op and gc */
757 struct rw_semaphore i_gc_rwsem[2];
758 struct rw_semaphore i_mmap_sem;
759 struct rw_semaphore i_xattr_sem; /* avoid racing between reading and changing EAs */
760
761 int i_extra_isize; /* size of extra space located in i_addr */
762 kprojid_t i_projid; /* id for project quota */
763 int i_inline_xattr_size; /* inline xattr size */
764 struct timespec64 i_crtime; /* inode creation time */
765 struct timespec64 i_disk_time[4];/* inode disk times */
766
767 /* for file compress */
768 atomic_t i_compr_blocks; /* # of compressed blocks */
769 unsigned char i_compress_algorithm; /* algorithm type */
770 unsigned char i_log_cluster_size; /* log of cluster size */
771 unsigned char i_compress_level; /* compress level (lz4hc,zstd) */
772 unsigned short i_compress_flag; /* compress flag */
773 unsigned int i_cluster_size; /* cluster size */
774};
775
776static inline void get_extent_info(struct extent_info *ext,
777 struct f2fs_extent *i_ext)
778{
779 ext->fofs = le32_to_cpu(i_ext->fofs);
780 ext->blk = le32_to_cpu(i_ext->blk);
781 ext->len = le32_to_cpu(i_ext->len);
782}
783
784static inline void set_raw_extent(struct extent_info *ext,
785 struct f2fs_extent *i_ext)
786{
787 i_ext->fofs = cpu_to_le32(ext->fofs);
788 i_ext->blk = cpu_to_le32(ext->blk);
789 i_ext->len = cpu_to_le32(ext->len);
790}
791
792static inline void set_extent_info(struct extent_info *ei, unsigned int fofs,
793 u32 blk, unsigned int len)
794{
795 ei->fofs = fofs;
796 ei->blk = blk;
797 ei->len = len;
798}
799
800static inline bool __is_discard_mergeable(struct discard_info *back,
801 struct discard_info *front, unsigned int max_len)
802{
803 return (back->lstart + back->len == front->lstart) &&
804 (back->len + front->len <= max_len);
805}
806
807static inline bool __is_discard_back_mergeable(struct discard_info *cur,
808 struct discard_info *back, unsigned int max_len)
809{
810 return __is_discard_mergeable(back, cur, max_len);
811}
812
813static inline bool __is_discard_front_mergeable(struct discard_info *cur,
814 struct discard_info *front, unsigned int max_len)
815{
816 return __is_discard_mergeable(cur, front, max_len);
817}
818
819static inline bool __is_extent_mergeable(struct extent_info *back,
820 struct extent_info *front)
821{
822 return (back->fofs + back->len == front->fofs &&
823 back->blk + back->len == front->blk);
824}
825
826static inline bool __is_back_mergeable(struct extent_info *cur,
827 struct extent_info *back)
828{
829 return __is_extent_mergeable(back, cur);
830}
831
832static inline bool __is_front_mergeable(struct extent_info *cur,
833 struct extent_info *front)
834{
835 return __is_extent_mergeable(cur, front);
836}
837
838extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync);
839static inline void __try_update_largest_extent(struct extent_tree *et,
840 struct extent_node *en)
841{
842 if (en->ei.len > et->largest.len) {
843 et->largest = en->ei;
844 et->largest_updated = true;
845 }
846}
847
848/*
849 * For free nid management
850 */
851enum nid_state {
852 FREE_NID, /* newly added to free nid list */
853 PREALLOC_NID, /* it is preallocated */
854 MAX_NID_STATE,
855};
856
857enum nat_state {
858 TOTAL_NAT,
859 DIRTY_NAT,
860 RECLAIMABLE_NAT,
861 MAX_NAT_STATE,
862};
863
864struct f2fs_nm_info {
865 block_t nat_blkaddr; /* base disk address of NAT */
866 nid_t max_nid; /* maximum possible node ids */
867 nid_t available_nids; /* # of available node ids */
868 nid_t next_scan_nid; /* the next nid to be scanned */
869 unsigned int ram_thresh; /* control the memory footprint */
870 unsigned int ra_nid_pages; /* # of nid pages to be readaheaded */
871 unsigned int dirty_nats_ratio; /* control dirty nats ratio threshold */
872
873 /* NAT cache management */
874 struct radix_tree_root nat_root;/* root of the nat entry cache */
875 struct radix_tree_root nat_set_root;/* root of the nat set cache */
876 struct rw_semaphore nat_tree_lock; /* protect nat entry tree */
877 struct list_head nat_entries; /* cached nat entry list (clean) */
878 spinlock_t nat_list_lock; /* protect clean nat entry list */
879 unsigned int nat_cnt[MAX_NAT_STATE]; /* the # of cached nat entries */
880 unsigned int nat_blocks; /* # of nat blocks */
881
882 /* free node ids management */
883 struct radix_tree_root free_nid_root;/* root of the free_nid cache */
884 struct list_head free_nid_list; /* list for free nids excluding preallocated nids */
885 unsigned int nid_cnt[MAX_NID_STATE]; /* the number of free node id */
886 spinlock_t nid_list_lock; /* protect nid lists ops */
887 struct mutex build_lock; /* lock for build free nids */
888 unsigned char **free_nid_bitmap;
889 unsigned char *nat_block_bitmap;
890 unsigned short *free_nid_count; /* free nid count of NAT block */
891
892 /* for checkpoint */
893 char *nat_bitmap; /* NAT bitmap pointer */
894
895 unsigned int nat_bits_blocks; /* # of nat bits blocks */
896 unsigned char *nat_bits; /* NAT bits blocks */
897 unsigned char *full_nat_bits; /* full NAT pages */
898 unsigned char *empty_nat_bits; /* empty NAT pages */
899#ifdef CONFIG_F2FS_CHECK_FS
900 char *nat_bitmap_mir; /* NAT bitmap mirror */
901#endif
902 int bitmap_size; /* bitmap size */
903};
904
905/*
906 * this structure is used as one of function parameters.
907 * all the information are dedicated to a given direct node block determined
908 * by the data offset in a file.
909 */
910struct dnode_of_data {
911 struct inode *inode; /* vfs inode pointer */
912 struct page *inode_page; /* its inode page, NULL is possible */
913 struct page *node_page; /* cached direct node page */
914 nid_t nid; /* node id of the direct node block */
915 unsigned int ofs_in_node; /* data offset in the node page */
916 bool inode_page_locked; /* inode page is locked or not */
917 bool node_changed; /* is node block changed */
918 char cur_level; /* level of hole node page */
919 char max_level; /* level of current page located */
920 block_t data_blkaddr; /* block address of the node block */
921};
922
923static inline void set_new_dnode(struct dnode_of_data *dn, struct inode *inode,
924 struct page *ipage, struct page *npage, nid_t nid)
925{
926 memset(dn, 0, sizeof(*dn));
927 dn->inode = inode;
928 dn->inode_page = ipage;
929 dn->node_page = npage;
930 dn->nid = nid;
931}
932
933/*
934 * For SIT manager
935 *
936 * By default, there are 6 active log areas across the whole main area.
937 * When considering hot and cold data separation to reduce cleaning overhead,
938 * we split 3 for data logs and 3 for node logs as hot, warm, and cold types,
939 * respectively.
940 * In the current design, you should not change the numbers intentionally.
941 * Instead, as a mount option such as active_logs=x, you can use 2, 4, and 6
942 * logs individually according to the underlying devices. (default: 6)
943 * Just in case, on-disk layout covers maximum 16 logs that consist of 8 for
944 * data and 8 for node logs.
945 */
946#define NR_CURSEG_DATA_TYPE (3)
947#define NR_CURSEG_NODE_TYPE (3)
948#define NR_CURSEG_INMEM_TYPE (2)
949#define NR_CURSEG_RO_TYPE (2)
950#define NR_CURSEG_PERSIST_TYPE (NR_CURSEG_DATA_TYPE + NR_CURSEG_NODE_TYPE)
951#define NR_CURSEG_TYPE (NR_CURSEG_INMEM_TYPE + NR_CURSEG_PERSIST_TYPE)
952
953enum {
954 CURSEG_HOT_DATA = 0, /* directory entry blocks */
955 CURSEG_WARM_DATA, /* data blocks */
956 CURSEG_COLD_DATA, /* multimedia or GCed data blocks */
957 CURSEG_HOT_NODE, /* direct node blocks of directory files */
958 CURSEG_WARM_NODE, /* direct node blocks of normal files */
959 CURSEG_COLD_NODE, /* indirect node blocks */
960 NR_PERSISTENT_LOG, /* number of persistent log */
961 CURSEG_COLD_DATA_PINNED = NR_PERSISTENT_LOG,
962 /* pinned file that needs consecutive block address */
963 CURSEG_ALL_DATA_ATGC, /* SSR alloctor in hot/warm/cold data area */
964 NO_CHECK_TYPE, /* number of persistent & inmem log */
965};
966
967struct flush_cmd {
968 struct completion wait;
969 struct llist_node llnode;
970 nid_t ino;
971 int ret;
972};
973
974struct flush_cmd_control {
975 struct task_struct *f2fs_issue_flush; /* flush thread */
976 wait_queue_head_t flush_wait_queue; /* waiting queue for wake-up */
977 atomic_t issued_flush; /* # of issued flushes */
978 atomic_t queued_flush; /* # of queued flushes */
979 struct llist_head issue_list; /* list for command issue */
980 struct llist_node *dispatch_list; /* list for command dispatch */
981};
982
983struct f2fs_sm_info {
984 struct sit_info *sit_info; /* whole segment information */
985 struct free_segmap_info *free_info; /* free segment information */
986 struct dirty_seglist_info *dirty_info; /* dirty segment information */
987 struct curseg_info *curseg_array; /* active segment information */
988
989 struct rw_semaphore curseg_lock; /* for preventing curseg change */
990
991 block_t seg0_blkaddr; /* block address of 0'th segment */
992 block_t main_blkaddr; /* start block address of main area */
993 block_t ssa_blkaddr; /* start block address of SSA area */
994
995 unsigned int segment_count; /* total # of segments */
996 unsigned int main_segments; /* # of segments in main area */
997 unsigned int reserved_segments; /* # of reserved segments */
998 unsigned int ovp_segments; /* # of overprovision segments */
999
1000 /* a threshold to reclaim prefree segments */
1001 unsigned int rec_prefree_segments;
1002
1003 /* for batched trimming */
1004 unsigned int trim_sections; /* # of sections to trim */
1005
1006 struct list_head sit_entry_set; /* sit entry set list */
1007
1008 unsigned int ipu_policy; /* in-place-update policy */
1009 unsigned int min_ipu_util; /* in-place-update threshold */
1010 unsigned int min_fsync_blocks; /* threshold for fsync */
1011 unsigned int min_seq_blocks; /* threshold for sequential blocks */
1012 unsigned int min_hot_blocks; /* threshold for hot block allocation */
1013 unsigned int min_ssr_sections; /* threshold to trigger SSR allocation */
1014
1015 /* for flush command control */
1016 struct flush_cmd_control *fcc_info;
1017
1018 /* for discard command control */
1019 struct discard_cmd_control *dcc_info;
1020};
1021
1022/*
1023 * For superblock
1024 */
1025/*
1026 * COUNT_TYPE for monitoring
1027 *
1028 * f2fs monitors the number of several block types such as on-writeback,
1029 * dirty dentry blocks, dirty node blocks, and dirty meta blocks.
1030 */
1031#define WB_DATA_TYPE(p) (__is_cp_guaranteed(p) ? F2FS_WB_CP_DATA : F2FS_WB_DATA)
1032enum count_type {
1033 F2FS_DIRTY_DENTS,
1034 F2FS_DIRTY_DATA,
1035 F2FS_DIRTY_QDATA,
1036 F2FS_DIRTY_NODES,
1037 F2FS_DIRTY_META,
1038 F2FS_INMEM_PAGES,
1039 F2FS_DIRTY_IMETA,
1040 F2FS_WB_CP_DATA,
1041 F2FS_WB_DATA,
1042 F2FS_RD_DATA,
1043 F2FS_RD_NODE,
1044 F2FS_RD_META,
1045 F2FS_DIO_WRITE,
1046 F2FS_DIO_READ,
1047 NR_COUNT_TYPE,
1048};
1049
1050/*
1051 * The below are the page types of bios used in submit_bio().
1052 * The available types are:
1053 * DATA User data pages. It operates as async mode.
1054 * NODE Node pages. It operates as async mode.
1055 * META FS metadata pages such as SIT, NAT, CP.
1056 * NR_PAGE_TYPE The number of page types.
1057 * META_FLUSH Make sure the previous pages are written
1058 * with waiting the bio's completion
1059 * ... Only can be used with META.
1060 */
1061#define PAGE_TYPE_OF_BIO(type) ((type) > META ? META : (type))
1062enum page_type {
1063 DATA,
1064 NODE,
1065 META,
1066 NR_PAGE_TYPE,
1067 META_FLUSH,
1068 INMEM, /* the below types are used by tracepoints only. */
1069 INMEM_DROP,
1070 INMEM_INVALIDATE,
1071 INMEM_REVOKE,
1072 IPU,
1073 OPU,
1074};
1075
1076enum temp_type {
1077 HOT = 0, /* must be zero for meta bio */
1078 WARM,
1079 COLD,
1080 NR_TEMP_TYPE,
1081};
1082
1083enum need_lock_type {
1084 LOCK_REQ = 0,
1085 LOCK_DONE,
1086 LOCK_RETRY,
1087};
1088
1089enum cp_reason_type {
1090 CP_NO_NEEDED,
1091 CP_NON_REGULAR,
1092 CP_COMPRESSED,
1093 CP_HARDLINK,
1094 CP_SB_NEED_CP,
1095 CP_WRONG_PINO,
1096 CP_NO_SPC_ROLL,
1097 CP_NODE_NEED_CP,
1098 CP_FASTBOOT_MODE,
1099 CP_SPEC_LOG_NUM,
1100 CP_RECOVER_DIR,
1101};
1102
1103enum iostat_type {
1104 /* WRITE IO */
1105 APP_DIRECT_IO, /* app direct write IOs */
1106 APP_BUFFERED_IO, /* app buffered write IOs */
1107 APP_WRITE_IO, /* app write IOs */
1108 APP_MAPPED_IO, /* app mapped IOs */
1109 FS_DATA_IO, /* data IOs from kworker/fsync/reclaimer */
1110 FS_NODE_IO, /* node IOs from kworker/fsync/reclaimer */
1111 FS_META_IO, /* meta IOs from kworker/reclaimer */
1112 FS_GC_DATA_IO, /* data IOs from forground gc */
1113 FS_GC_NODE_IO, /* node IOs from forground gc */
1114 FS_CP_DATA_IO, /* data IOs from checkpoint */
1115 FS_CP_NODE_IO, /* node IOs from checkpoint */
1116 FS_CP_META_IO, /* meta IOs from checkpoint */
1117
1118 /* READ IO */
1119 APP_DIRECT_READ_IO, /* app direct read IOs */
1120 APP_BUFFERED_READ_IO, /* app buffered read IOs */
1121 APP_READ_IO, /* app read IOs */
1122 APP_MAPPED_READ_IO, /* app mapped read IOs */
1123 FS_DATA_READ_IO, /* data read IOs */
1124 FS_GDATA_READ_IO, /* data read IOs from background gc */
1125 FS_CDATA_READ_IO, /* compressed data read IOs */
1126 FS_NODE_READ_IO, /* node read IOs */
1127 FS_META_READ_IO, /* meta read IOs */
1128
1129 /* other */
1130 FS_DISCARD, /* discard */
1131 NR_IO_TYPE,
1132};
1133
1134struct f2fs_io_info {
1135 struct f2fs_sb_info *sbi; /* f2fs_sb_info pointer */
1136 nid_t ino; /* inode number */
1137 enum page_type type; /* contains DATA/NODE/META/META_FLUSH */
1138 enum temp_type temp; /* contains HOT/WARM/COLD */
1139 int op; /* contains REQ_OP_ */
1140 int op_flags; /* req_flag_bits */
1141 block_t new_blkaddr; /* new block address to be written */
1142 block_t old_blkaddr; /* old block address before Cow */
1143 struct page *page; /* page to be written */
1144 struct page *encrypted_page; /* encrypted page */
1145 struct page *compressed_page; /* compressed page */
1146 struct list_head list; /* serialize IOs */
1147 bool submitted; /* indicate IO submission */
1148 int need_lock; /* indicate we need to lock cp_rwsem */
1149 bool in_list; /* indicate fio is in io_list */
1150 bool is_por; /* indicate IO is from recovery or not */
1151 bool retry; /* need to reallocate block address */
1152 int compr_blocks; /* # of compressed block addresses */
1153 bool encrypted; /* indicate file is encrypted */
1154 enum iostat_type io_type; /* io type */
1155 struct writeback_control *io_wbc; /* writeback control */
1156 struct bio **bio; /* bio for ipu */
1157 sector_t *last_block; /* last block number in bio */
1158 unsigned char version; /* version of the node */
1159};
1160
1161struct bio_entry {
1162 struct bio *bio;
1163 struct list_head list;
1164};
1165
1166#define is_read_io(rw) ((rw) == READ)
1167struct f2fs_bio_info {
1168 struct f2fs_sb_info *sbi; /* f2fs superblock */
1169 struct bio *bio; /* bios to merge */
1170 sector_t last_block_in_bio; /* last block number */
1171 struct f2fs_io_info fio; /* store buffered io info. */
1172 struct rw_semaphore io_rwsem; /* blocking op for bio */
1173 spinlock_t io_lock; /* serialize DATA/NODE IOs */
1174 struct list_head io_list; /* track fios */
1175 struct list_head bio_list; /* bio entry list head */
1176 struct rw_semaphore bio_list_lock; /* lock to protect bio entry list */
1177};
1178
1179#define FDEV(i) (sbi->devs[i])
1180#define RDEV(i) (raw_super->devs[i])
1181struct f2fs_dev_info {
1182 struct block_device *bdev;
1183 char path[MAX_PATH_LEN];
1184 unsigned int total_segments;
1185 block_t start_blk;
1186 block_t end_blk;
1187#ifdef CONFIG_BLK_DEV_ZONED
1188 unsigned int nr_blkz; /* Total number of zones */
1189 unsigned long *blkz_seq; /* Bitmap indicating sequential zones */
1190 block_t *zone_capacity_blocks; /* Array of zone capacity in blks */
1191#endif
1192};
1193
1194enum inode_type {
1195 DIR_INODE, /* for dirty dir inode */
1196 FILE_INODE, /* for dirty regular/symlink inode */
1197 DIRTY_META, /* for all dirtied inode metadata */
1198 ATOMIC_FILE, /* for all atomic files */
1199 NR_INODE_TYPE,
1200};
1201
1202/* for inner inode cache management */
1203struct inode_management {
1204 struct radix_tree_root ino_root; /* ino entry array */
1205 spinlock_t ino_lock; /* for ino entry lock */
1206 struct list_head ino_list; /* inode list head */
1207 unsigned long ino_num; /* number of entries */
1208};
1209
1210/* for GC_AT */
1211struct atgc_management {
1212 bool atgc_enabled; /* ATGC is enabled or not */
1213 struct rb_root_cached root; /* root of victim rb-tree */
1214 struct list_head victim_list; /* linked with all victim entries */
1215 unsigned int victim_count; /* victim count in rb-tree */
1216 unsigned int candidate_ratio; /* candidate ratio */
1217 unsigned int max_candidate_count; /* max candidate count */
1218 unsigned int age_weight; /* age weight, vblock_weight = 100 - age_weight */
1219 unsigned long long age_threshold; /* age threshold */
1220};
1221
1222/* For s_flag in struct f2fs_sb_info */
1223enum {
1224 SBI_IS_DIRTY, /* dirty flag for checkpoint */
1225 SBI_IS_CLOSE, /* specify unmounting */
1226 SBI_NEED_FSCK, /* need fsck.f2fs to fix */
1227 SBI_POR_DOING, /* recovery is doing or not */
1228 SBI_NEED_SB_WRITE, /* need to recover superblock */
1229 SBI_NEED_CP, /* need to checkpoint */
1230 SBI_IS_SHUTDOWN, /* shutdown by ioctl */
1231 SBI_IS_RECOVERED, /* recovered orphan/data */
1232 SBI_CP_DISABLED, /* CP was disabled last mount */
1233 SBI_CP_DISABLED_QUICK, /* CP was disabled quickly */
1234 SBI_QUOTA_NEED_FLUSH, /* need to flush quota info in CP */
1235 SBI_QUOTA_SKIP_FLUSH, /* skip flushing quota in current CP */
1236 SBI_QUOTA_NEED_REPAIR, /* quota file may be corrupted */
1237 SBI_IS_RESIZEFS, /* resizefs is in process */
1238};
1239
1240enum {
1241 CP_TIME,
1242 REQ_TIME,
1243 DISCARD_TIME,
1244 GC_TIME,
1245 DISABLE_TIME,
1246 UMOUNT_DISCARD_TIMEOUT,
1247 MAX_TIME,
1248};
1249
1250enum {
1251 GC_NORMAL,
1252 GC_IDLE_CB,
1253 GC_IDLE_GREEDY,
1254 GC_IDLE_AT,
1255 GC_URGENT_HIGH,
1256 GC_URGENT_LOW,
1257};
1258
1259enum {
1260 BGGC_MODE_ON, /* background gc is on */
1261 BGGC_MODE_OFF, /* background gc is off */
1262 BGGC_MODE_SYNC, /*
1263 * background gc is on, migrating blocks
1264 * like foreground gc
1265 */
1266};
1267
1268enum {
1269 FS_MODE_ADAPTIVE, /* use both lfs/ssr allocation */
1270 FS_MODE_LFS, /* use lfs allocation only */
1271};
1272
1273enum {
1274 WHINT_MODE_OFF, /* not pass down write hints */
1275 WHINT_MODE_USER, /* try to pass down hints given by users */
1276 WHINT_MODE_FS, /* pass down hints with F2FS policy */
1277};
1278
1279enum {
1280 ALLOC_MODE_DEFAULT, /* stay default */
1281 ALLOC_MODE_REUSE, /* reuse segments as much as possible */
1282};
1283
1284enum fsync_mode {
1285 FSYNC_MODE_POSIX, /* fsync follows posix semantics */
1286 FSYNC_MODE_STRICT, /* fsync behaves in line with ext4 */
1287 FSYNC_MODE_NOBARRIER, /* fsync behaves nobarrier based on posix */
1288};
1289
1290enum {
1291 COMPR_MODE_FS, /*
1292 * automatically compress compression
1293 * enabled files
1294 */
1295 COMPR_MODE_USER, /*
1296 * automatical compression is disabled.
1297 * user can control the file compression
1298 * using ioctls
1299 */
1300};
1301
1302static inline int f2fs_test_bit(unsigned int nr, char *addr);
1303static inline void f2fs_set_bit(unsigned int nr, char *addr);
1304static inline void f2fs_clear_bit(unsigned int nr, char *addr);
1305
1306/*
1307 * Layout of f2fs page.private:
1308 *
1309 * Layout A: lowest bit should be 1
1310 * | bit0 = 1 | bit1 | bit2 | ... | bit MAX | private data .... |
1311 * bit 0 PAGE_PRIVATE_NOT_POINTER
1312 * bit 1 PAGE_PRIVATE_ATOMIC_WRITE
1313 * bit 2 PAGE_PRIVATE_DUMMY_WRITE
1314 * bit 3 PAGE_PRIVATE_ONGOING_MIGRATION
1315 * bit 4 PAGE_PRIVATE_INLINE_INODE
1316 * bit 5 PAGE_PRIVATE_REF_RESOURCE
1317 * bit 6- f2fs private data
1318 *
1319 * Layout B: lowest bit should be 0
1320 * page.private is a wrapped pointer.
1321 */
1322enum {
1323 PAGE_PRIVATE_NOT_POINTER, /* private contains non-pointer data */
1324 PAGE_PRIVATE_ATOMIC_WRITE, /* data page from atomic write path */
1325 PAGE_PRIVATE_DUMMY_WRITE, /* data page for padding aligned IO */
1326 PAGE_PRIVATE_ONGOING_MIGRATION, /* data page which is on-going migrating */
1327 PAGE_PRIVATE_INLINE_INODE, /* inode page contains inline data */
1328 PAGE_PRIVATE_REF_RESOURCE, /* dirty page has referenced resources */
1329 PAGE_PRIVATE_MAX
1330};
1331
1332#define PAGE_PRIVATE_GET_FUNC(name, flagname) \
1333static inline bool page_private_##name(struct page *page) \
1334{ \
1335 return PagePrivate(page) && \
1336 test_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)) && \
1337 test_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
1338}
1339
1340#define PAGE_PRIVATE_SET_FUNC(name, flagname) \
1341static inline void set_page_private_##name(struct page *page) \
1342{ \
1343 if (!PagePrivate(page)) { \
1344 get_page(page); \
1345 SetPagePrivate(page); \
1346 set_page_private(page, 0); \
1347 } \
1348 set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)); \
1349 set_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
1350}
1351
1352#define PAGE_PRIVATE_CLEAR_FUNC(name, flagname) \
1353static inline void clear_page_private_##name(struct page *page) \
1354{ \
1355 clear_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
1356 if (page_private(page) == 1 << PAGE_PRIVATE_NOT_POINTER) { \
1357 set_page_private(page, 0); \
1358 if (PagePrivate(page)) { \
1359 ClearPagePrivate(page); \
1360 put_page(page); \
1361 }\
1362 } \
1363}
1364
1365PAGE_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER);
1366PAGE_PRIVATE_GET_FUNC(reference, REF_RESOURCE);
1367PAGE_PRIVATE_GET_FUNC(inline, INLINE_INODE);
1368PAGE_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION);
1369PAGE_PRIVATE_GET_FUNC(atomic, ATOMIC_WRITE);
1370PAGE_PRIVATE_GET_FUNC(dummy, DUMMY_WRITE);
1371
1372PAGE_PRIVATE_SET_FUNC(reference, REF_RESOURCE);
1373PAGE_PRIVATE_SET_FUNC(inline, INLINE_INODE);
1374PAGE_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION);
1375PAGE_PRIVATE_SET_FUNC(atomic, ATOMIC_WRITE);
1376PAGE_PRIVATE_SET_FUNC(dummy, DUMMY_WRITE);
1377
1378PAGE_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE);
1379PAGE_PRIVATE_CLEAR_FUNC(inline, INLINE_INODE);
1380PAGE_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION);
1381PAGE_PRIVATE_CLEAR_FUNC(atomic, ATOMIC_WRITE);
1382PAGE_PRIVATE_CLEAR_FUNC(dummy, DUMMY_WRITE);
1383
1384static inline unsigned long get_page_private_data(struct page *page)
1385{
1386 unsigned long data = page_private(page);
1387
1388 if (!test_bit(PAGE_PRIVATE_NOT_POINTER, &data))
1389 return 0;
1390 return data >> PAGE_PRIVATE_MAX;
1391}
1392
1393static inline void set_page_private_data(struct page *page, unsigned long data)
1394{
1395 if (!PagePrivate(page)) {
1396 get_page(page);
1397 SetPagePrivate(page);
1398 set_page_private(page, 0);
1399 }
1400 set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page));
1401 page_private(page) |= data << PAGE_PRIVATE_MAX;
1402}
1403
1404static inline void clear_page_private_data(struct page *page)
1405{
1406 page_private(page) &= (1 << PAGE_PRIVATE_MAX) - 1;
1407 if (page_private(page) == 1 << PAGE_PRIVATE_NOT_POINTER) {
1408 set_page_private(page, 0);
1409 if (PagePrivate(page)) {
1410 ClearPagePrivate(page);
1411 put_page(page);
1412 }
1413 }
1414}
1415
1416/* For compression */
1417enum compress_algorithm_type {
1418 COMPRESS_LZO,
1419 COMPRESS_LZ4,
1420 COMPRESS_ZSTD,
1421 COMPRESS_LZORLE,
1422 COMPRESS_MAX,
1423};
1424
1425enum compress_flag {
1426 COMPRESS_CHKSUM,
1427 COMPRESS_MAX_FLAG,
1428};
1429
1430#define COMPRESS_WATERMARK 20
1431#define COMPRESS_PERCENT 20
1432
1433#define COMPRESS_DATA_RESERVED_SIZE 4
1434struct compress_data {
1435 __le32 clen; /* compressed data size */
1436 __le32 chksum; /* compressed data chksum */
1437 __le32 reserved[COMPRESS_DATA_RESERVED_SIZE]; /* reserved */
1438 u8 cdata[]; /* compressed data */
1439};
1440
1441#define COMPRESS_HEADER_SIZE (sizeof(struct compress_data))
1442
1443#define F2FS_COMPRESSED_PAGE_MAGIC 0xF5F2C000
1444
1445#define COMPRESS_LEVEL_OFFSET 8
1446
1447/* compress context */
1448struct compress_ctx {
1449 struct inode *inode; /* inode the context belong to */
1450 pgoff_t cluster_idx; /* cluster index number */
1451 unsigned int cluster_size; /* page count in cluster */
1452 unsigned int log_cluster_size; /* log of cluster size */
1453 struct page **rpages; /* pages store raw data in cluster */
1454 unsigned int nr_rpages; /* total page number in rpages */
1455 struct page **cpages; /* pages store compressed data in cluster */
1456 unsigned int nr_cpages; /* total page number in cpages */
1457 void *rbuf; /* virtual mapped address on rpages */
1458 struct compress_data *cbuf; /* virtual mapped address on cpages */
1459 size_t rlen; /* valid data length in rbuf */
1460 size_t clen; /* valid data length in cbuf */
1461 void *private; /* payload buffer for specified compression algorithm */
1462 void *private2; /* extra payload buffer */
1463};
1464
1465/* compress context for write IO path */
1466struct compress_io_ctx {
1467 u32 magic; /* magic number to indicate page is compressed */
1468 struct inode *inode; /* inode the context belong to */
1469 struct page **rpages; /* pages store raw data in cluster */
1470 unsigned int nr_rpages; /* total page number in rpages */
1471 atomic_t pending_pages; /* in-flight compressed page count */
1472};
1473
1474/* Context for decompressing one cluster on the read IO path */
1475struct decompress_io_ctx {
1476 u32 magic; /* magic number to indicate page is compressed */
1477 struct inode *inode; /* inode the context belong to */
1478 pgoff_t cluster_idx; /* cluster index number */
1479 unsigned int cluster_size; /* page count in cluster */
1480 unsigned int log_cluster_size; /* log of cluster size */
1481 struct page **rpages; /* pages store raw data in cluster */
1482 unsigned int nr_rpages; /* total page number in rpages */
1483 struct page **cpages; /* pages store compressed data in cluster */
1484 unsigned int nr_cpages; /* total page number in cpages */
1485 struct page **tpages; /* temp pages to pad holes in cluster */
1486 void *rbuf; /* virtual mapped address on rpages */
1487 struct compress_data *cbuf; /* virtual mapped address on cpages */
1488 size_t rlen; /* valid data length in rbuf */
1489 size_t clen; /* valid data length in cbuf */
1490
1491 /*
1492 * The number of compressed pages remaining to be read in this cluster.
1493 * This is initially nr_cpages. It is decremented by 1 each time a page
1494 * has been read (or failed to be read). When it reaches 0, the cluster
1495 * is decompressed (or an error is reported).
1496 *
1497 * If an error occurs before all the pages have been submitted for I/O,
1498 * then this will never reach 0. In this case the I/O submitter is
1499 * responsible for calling f2fs_decompress_end_io() instead.
1500 */
1501 atomic_t remaining_pages;
1502
1503 /*
1504 * Number of references to this decompress_io_ctx.
1505 *
1506 * One reference is held for I/O completion. This reference is dropped
1507 * after the pagecache pages are updated and unlocked -- either after
1508 * decompression (and verity if enabled), or after an error.
1509 *
1510 * In addition, each compressed page holds a reference while it is in a
1511 * bio. These references are necessary prevent compressed pages from
1512 * being freed while they are still in a bio.
1513 */
1514 refcount_t refcnt;
1515
1516 bool failed; /* IO error occurred before decompression? */
1517 bool need_verity; /* need fs-verity verification after decompression? */
1518 void *private; /* payload buffer for specified decompression algorithm */
1519 void *private2; /* extra payload buffer */
1520 struct work_struct verity_work; /* work to verify the decompressed pages */
1521};
1522
1523#define NULL_CLUSTER ((unsigned int)(~0))
1524#define MIN_COMPRESS_LOG_SIZE 2
1525#define MAX_COMPRESS_LOG_SIZE 8
1526#define MAX_COMPRESS_WINDOW_SIZE(log_size) ((PAGE_SIZE) << (log_size))
1527
1528struct f2fs_sb_info {
1529 struct super_block *sb; /* pointer to VFS super block */
1530 struct proc_dir_entry *s_proc; /* proc entry */
1531 struct f2fs_super_block *raw_super; /* raw super block pointer */
1532 struct rw_semaphore sb_lock; /* lock for raw super block */
1533 int valid_super_block; /* valid super block no */
1534 unsigned long s_flag; /* flags for sbi */
1535 struct mutex writepages; /* mutex for writepages() */
1536
1537#ifdef CONFIG_BLK_DEV_ZONED
1538 unsigned int blocks_per_blkz; /* F2FS blocks per zone */
1539 unsigned int log_blocks_per_blkz; /* log2 F2FS blocks per zone */
1540#endif
1541
1542 /* for node-related operations */
1543 struct f2fs_nm_info *nm_info; /* node manager */
1544 struct inode *node_inode; /* cache node blocks */
1545
1546 /* for segment-related operations */
1547 struct f2fs_sm_info *sm_info; /* segment manager */
1548
1549 /* for bio operations */
1550 struct f2fs_bio_info *write_io[NR_PAGE_TYPE]; /* for write bios */
1551 /* keep migration IO order for LFS mode */
1552 struct rw_semaphore io_order_lock;
1553 mempool_t *write_io_dummy; /* Dummy pages */
1554
1555 /* for checkpoint */
1556 struct f2fs_checkpoint *ckpt; /* raw checkpoint pointer */
1557 int cur_cp_pack; /* remain current cp pack */
1558 spinlock_t cp_lock; /* for flag in ckpt */
1559 struct inode *meta_inode; /* cache meta blocks */
1560 struct rw_semaphore cp_global_sem; /* checkpoint procedure lock */
1561 struct rw_semaphore cp_rwsem; /* blocking FS operations */
1562 struct rw_semaphore node_write; /* locking node writes */
1563 struct rw_semaphore node_change; /* locking node change */
1564 wait_queue_head_t cp_wait;
1565 unsigned long last_time[MAX_TIME]; /* to store time in jiffies */
1566 long interval_time[MAX_TIME]; /* to store thresholds */
1567 struct ckpt_req_control cprc_info; /* for checkpoint request control */
1568
1569 struct inode_management im[MAX_INO_ENTRY]; /* manage inode cache */
1570
1571 spinlock_t fsync_node_lock; /* for node entry lock */
1572 struct list_head fsync_node_list; /* node list head */
1573 unsigned int fsync_seg_id; /* sequence id */
1574 unsigned int fsync_node_num; /* number of node entries */
1575
1576 /* for orphan inode, use 0'th array */
1577 unsigned int max_orphans; /* max orphan inodes */
1578
1579 /* for inode management */
1580 struct list_head inode_list[NR_INODE_TYPE]; /* dirty inode list */
1581 spinlock_t inode_lock[NR_INODE_TYPE]; /* for dirty inode list lock */
1582 struct mutex flush_lock; /* for flush exclusion */
1583
1584 /* for extent tree cache */
1585 struct radix_tree_root extent_tree_root;/* cache extent cache entries */
1586 struct mutex extent_tree_lock; /* locking extent radix tree */
1587 struct list_head extent_list; /* lru list for shrinker */
1588 spinlock_t extent_lock; /* locking extent lru list */
1589 atomic_t total_ext_tree; /* extent tree count */
1590 struct list_head zombie_list; /* extent zombie tree list */
1591 atomic_t total_zombie_tree; /* extent zombie tree count */
1592 atomic_t total_ext_node; /* extent info count */
1593
1594 /* basic filesystem units */
1595 unsigned int log_sectors_per_block; /* log2 sectors per block */
1596 unsigned int log_blocksize; /* log2 block size */
1597 unsigned int blocksize; /* block size */
1598 unsigned int root_ino_num; /* root inode number*/
1599 unsigned int node_ino_num; /* node inode number*/
1600 unsigned int meta_ino_num; /* meta inode number*/
1601 unsigned int log_blocks_per_seg; /* log2 blocks per segment */
1602 unsigned int blocks_per_seg; /* blocks per segment */
1603 unsigned int segs_per_sec; /* segments per section */
1604 unsigned int secs_per_zone; /* sections per zone */
1605 unsigned int total_sections; /* total section count */
1606 unsigned int total_node_count; /* total node block count */
1607 unsigned int total_valid_node_count; /* valid node block count */
1608 int dir_level; /* directory level */
1609 int readdir_ra; /* readahead inode in readdir */
1610 u64 max_io_bytes; /* max io bytes to merge IOs */
1611
1612 block_t user_block_count; /* # of user blocks */
1613 block_t total_valid_block_count; /* # of valid blocks */
1614 block_t discard_blks; /* discard command candidats */
1615 block_t last_valid_block_count; /* for recovery */
1616 block_t reserved_blocks; /* configurable reserved blocks */
1617 block_t current_reserved_blocks; /* current reserved blocks */
1618
1619 /* Additional tracking for no checkpoint mode */
1620 block_t unusable_block_count; /* # of blocks saved by last cp */
1621
1622 unsigned int nquota_files; /* # of quota sysfile */
1623 struct rw_semaphore quota_sem; /* blocking cp for flags */
1624
1625 /* # of pages, see count_type */
1626 atomic_t nr_pages[NR_COUNT_TYPE];
1627 /* # of allocated blocks */
1628 struct percpu_counter alloc_valid_block_count;
1629
1630 /* writeback control */
1631 atomic_t wb_sync_req[META]; /* count # of WB_SYNC threads */
1632
1633 /* valid inode count */
1634 struct percpu_counter total_valid_inode_count;
1635
1636 struct f2fs_mount_info mount_opt; /* mount options */
1637
1638 /* for cleaning operations */
1639 struct rw_semaphore gc_lock; /*
1640 * semaphore for GC, avoid
1641 * race between GC and GC or CP
1642 */
1643 struct f2fs_gc_kthread *gc_thread; /* GC thread */
1644 struct atgc_management am; /* atgc management */
1645 unsigned int cur_victim_sec; /* current victim section num */
1646 unsigned int gc_mode; /* current GC state */
1647 unsigned int next_victim_seg[2]; /* next segment in victim section */
1648
1649 /* for skip statistic */
1650 unsigned int atomic_files; /* # of opened atomic file */
1651 unsigned long long skipped_atomic_files[2]; /* FG_GC and BG_GC */
1652 unsigned long long skipped_gc_rwsem; /* FG_GC only */
1653
1654 /* threshold for gc trials on pinned files */
1655 u64 gc_pin_file_threshold;
1656 struct rw_semaphore pin_sem;
1657
1658 /* maximum # of trials to find a victim segment for SSR and GC */
1659 unsigned int max_victim_search;
1660 /* migration granularity of garbage collection, unit: segment */
1661 unsigned int migration_granularity;
1662
1663 /*
1664 * for stat information.
1665 * one is for the LFS mode, and the other is for the SSR mode.
1666 */
1667#ifdef CONFIG_F2FS_STAT_FS
1668 struct f2fs_stat_info *stat_info; /* FS status information */
1669 atomic_t meta_count[META_MAX]; /* # of meta blocks */
1670 unsigned int segment_count[2]; /* # of allocated segments */
1671 unsigned int block_count[2]; /* # of allocated blocks */
1672 atomic_t inplace_count; /* # of inplace update */
1673 atomic64_t total_hit_ext; /* # of lookup extent cache */
1674 atomic64_t read_hit_rbtree; /* # of hit rbtree extent node */
1675 atomic64_t read_hit_largest; /* # of hit largest extent node */
1676 atomic64_t read_hit_cached; /* # of hit cached extent node */
1677 atomic_t inline_xattr; /* # of inline_xattr inodes */
1678 atomic_t inline_inode; /* # of inline_data inodes */
1679 atomic_t inline_dir; /* # of inline_dentry inodes */
1680 atomic_t compr_inode; /* # of compressed inodes */
1681 atomic64_t compr_blocks; /* # of compressed blocks */
1682 atomic_t vw_cnt; /* # of volatile writes */
1683 atomic_t max_aw_cnt; /* max # of atomic writes */
1684 atomic_t max_vw_cnt; /* max # of volatile writes */
1685 unsigned int io_skip_bggc; /* skip background gc for in-flight IO */
1686 unsigned int other_skip_bggc; /* skip background gc for other reasons */
1687 unsigned int ndirty_inode[NR_INODE_TYPE]; /* # of dirty inodes */
1688#endif
1689 spinlock_t stat_lock; /* lock for stat operations */
1690
1691 /* For app/fs IO statistics */
1692 spinlock_t iostat_lock;
1693 unsigned long long rw_iostat[NR_IO_TYPE];
1694 unsigned long long prev_rw_iostat[NR_IO_TYPE];
1695 bool iostat_enable;
1696 unsigned long iostat_next_period;
1697 unsigned int iostat_period_ms;
1698
1699 /* to attach REQ_META|REQ_FUA flags */
1700 unsigned int data_io_flag;
1701 unsigned int node_io_flag;
1702
1703 /* For sysfs suppport */
1704 struct kobject s_kobj; /* /sys/fs/f2fs/<devname> */
1705 struct completion s_kobj_unregister;
1706
1707 struct kobject s_stat_kobj; /* /sys/fs/f2fs/<devname>/stat */
1708 struct completion s_stat_kobj_unregister;
1709
1710 struct kobject s_feature_list_kobj; /* /sys/fs/f2fs/<devname>/feature_list */
1711 struct completion s_feature_list_kobj_unregister;
1712
1713 /* For shrinker support */
1714 struct list_head s_list;
1715 int s_ndevs; /* number of devices */
1716 struct f2fs_dev_info *devs; /* for device list */
1717 unsigned int dirty_device; /* for checkpoint data flush */
1718 spinlock_t dev_lock; /* protect dirty_device */
1719 struct mutex umount_mutex;
1720 unsigned int shrinker_run_no;
1721
1722 /* For write statistics */
1723 u64 sectors_written_start;
1724 u64 kbytes_written;
1725
1726 /* Reference to checksum algorithm driver via cryptoapi */
1727 struct crypto_shash *s_chksum_driver;
1728
1729 /* Precomputed FS UUID checksum for seeding other checksums */
1730 __u32 s_chksum_seed;
1731
1732 struct workqueue_struct *post_read_wq; /* post read workqueue */
1733
1734 struct kmem_cache *inline_xattr_slab; /* inline xattr entry */
1735 unsigned int inline_xattr_slab_size; /* default inline xattr slab size */
1736
1737#ifdef CONFIG_F2FS_FS_COMPRESSION
1738 struct kmem_cache *page_array_slab; /* page array entry */
1739 unsigned int page_array_slab_size; /* default page array slab size */
1740
1741 /* For runtime compression statistics */
1742 u64 compr_written_block;
1743 u64 compr_saved_block;
1744 u32 compr_new_inode;
1745
1746 /* For compressed block cache */
1747 struct inode *compress_inode; /* cache compressed blocks */
1748 unsigned int compress_percent; /* cache page percentage */
1749 unsigned int compress_watermark; /* cache page watermark */
1750 atomic_t compress_page_hit; /* cache hit count */
1751#endif
1752};
1753
1754struct f2fs_private_dio {
1755 struct inode *inode;
1756 void *orig_private;
1757 bio_end_io_t *orig_end_io;
1758 bool write;
1759};
1760
1761#ifdef CONFIG_F2FS_FAULT_INJECTION
1762#define f2fs_show_injection_info(sbi, type) \
1763 printk_ratelimited("%sF2FS-fs (%s) : inject %s in %s of %pS\n", \
1764 KERN_INFO, sbi->sb->s_id, \
1765 f2fs_fault_name[type], \
1766 __func__, __builtin_return_address(0))
1767static inline bool time_to_inject(struct f2fs_sb_info *sbi, int type)
1768{
1769 struct f2fs_fault_info *ffi = &F2FS_OPTION(sbi).fault_info;
1770
1771 if (!ffi->inject_rate)
1772 return false;
1773
1774 if (!IS_FAULT_SET(ffi, type))
1775 return false;
1776
1777 atomic_inc(&ffi->inject_ops);
1778 if (atomic_read(&ffi->inject_ops) >= ffi->inject_rate) {
1779 atomic_set(&ffi->inject_ops, 0);
1780 return true;
1781 }
1782 return false;
1783}
1784#else
1785#define f2fs_show_injection_info(sbi, type) do { } while (0)
1786static inline bool time_to_inject(struct f2fs_sb_info *sbi, int type)
1787{
1788 return false;
1789}
1790#endif
1791
1792/*
1793 * Test if the mounted volume is a multi-device volume.
1794 * - For a single regular disk volume, sbi->s_ndevs is 0.
1795 * - For a single zoned disk volume, sbi->s_ndevs is 1.
1796 * - For a multi-device volume, sbi->s_ndevs is always 2 or more.
1797 */
1798static inline bool f2fs_is_multi_device(struct f2fs_sb_info *sbi)
1799{
1800 return sbi->s_ndevs > 1;
1801}
1802
1803static inline void f2fs_update_time(struct f2fs_sb_info *sbi, int type)
1804{
1805 unsigned long now = jiffies;
1806
1807 sbi->last_time[type] = now;
1808
1809 /* DISCARD_TIME and GC_TIME are based on REQ_TIME */
1810 if (type == REQ_TIME) {
1811 sbi->last_time[DISCARD_TIME] = now;
1812 sbi->last_time[GC_TIME] = now;
1813 }
1814}
1815
1816static inline bool f2fs_time_over(struct f2fs_sb_info *sbi, int type)
1817{
1818 unsigned long interval = sbi->interval_time[type] * HZ;
1819
1820 return time_after(jiffies, sbi->last_time[type] + interval);
1821}
1822
1823static inline unsigned int f2fs_time_to_wait(struct f2fs_sb_info *sbi,
1824 int type)
1825{
1826 unsigned long interval = sbi->interval_time[type] * HZ;
1827 unsigned int wait_ms = 0;
1828 long delta;
1829
1830 delta = (sbi->last_time[type] + interval) - jiffies;
1831 if (delta > 0)
1832 wait_ms = jiffies_to_msecs(delta);
1833
1834 return wait_ms;
1835}
1836
1837/*
1838 * Inline functions
1839 */
1840static inline u32 __f2fs_crc32(struct f2fs_sb_info *sbi, u32 crc,
1841 const void *address, unsigned int length)
1842{
1843 struct {
1844 struct shash_desc shash;
1845 char ctx[4];
1846 } desc;
1847 int err;
1848
1849 BUG_ON(crypto_shash_descsize(sbi->s_chksum_driver) != sizeof(desc.ctx));
1850
1851 desc.shash.tfm = sbi->s_chksum_driver;
1852 *(u32 *)desc.ctx = crc;
1853
1854 err = crypto_shash_update(&desc.shash, address, length);
1855 BUG_ON(err);
1856
1857 return *(u32 *)desc.ctx;
1858}
1859
1860static inline u32 f2fs_crc32(struct f2fs_sb_info *sbi, const void *address,
1861 unsigned int length)
1862{
1863 return __f2fs_crc32(sbi, F2FS_SUPER_MAGIC, address, length);
1864}
1865
1866static inline bool f2fs_crc_valid(struct f2fs_sb_info *sbi, __u32 blk_crc,
1867 void *buf, size_t buf_size)
1868{
1869 return f2fs_crc32(sbi, buf, buf_size) == blk_crc;
1870}
1871
1872static inline u32 f2fs_chksum(struct f2fs_sb_info *sbi, u32 crc,
1873 const void *address, unsigned int length)
1874{
1875 return __f2fs_crc32(sbi, crc, address, length);
1876}
1877
1878static inline struct f2fs_inode_info *F2FS_I(struct inode *inode)
1879{
1880 return container_of(inode, struct f2fs_inode_info, vfs_inode);
1881}
1882
1883static inline struct f2fs_sb_info *F2FS_SB(struct super_block *sb)
1884{
1885 return sb->s_fs_info;
1886}
1887
1888static inline struct f2fs_sb_info *F2FS_I_SB(struct inode *inode)
1889{
1890 return F2FS_SB(inode->i_sb);
1891}
1892
1893static inline struct f2fs_sb_info *F2FS_M_SB(struct address_space *mapping)
1894{
1895 return F2FS_I_SB(mapping->host);
1896}
1897
1898static inline struct f2fs_sb_info *F2FS_P_SB(struct page *page)
1899{
1900 return F2FS_M_SB(page_file_mapping(page));
1901}
1902
1903static inline struct f2fs_super_block *F2FS_RAW_SUPER(struct f2fs_sb_info *sbi)
1904{
1905 return (struct f2fs_super_block *)(sbi->raw_super);
1906}
1907
1908static inline struct f2fs_checkpoint *F2FS_CKPT(struct f2fs_sb_info *sbi)
1909{
1910 return (struct f2fs_checkpoint *)(sbi->ckpt);
1911}
1912
1913static inline struct f2fs_node *F2FS_NODE(struct page *page)
1914{
1915 return (struct f2fs_node *)page_address(page);
1916}
1917
1918static inline struct f2fs_inode *F2FS_INODE(struct page *page)
1919{
1920 return &((struct f2fs_node *)page_address(page))->i;
1921}
1922
1923static inline struct f2fs_nm_info *NM_I(struct f2fs_sb_info *sbi)
1924{
1925 return (struct f2fs_nm_info *)(sbi->nm_info);
1926}
1927
1928static inline struct f2fs_sm_info *SM_I(struct f2fs_sb_info *sbi)
1929{
1930 return (struct f2fs_sm_info *)(sbi->sm_info);
1931}
1932
1933static inline struct sit_info *SIT_I(struct f2fs_sb_info *sbi)
1934{
1935 return (struct sit_info *)(SM_I(sbi)->sit_info);
1936}
1937
1938static inline struct free_segmap_info *FREE_I(struct f2fs_sb_info *sbi)
1939{
1940 return (struct free_segmap_info *)(SM_I(sbi)->free_info);
1941}
1942
1943static inline struct dirty_seglist_info *DIRTY_I(struct f2fs_sb_info *sbi)
1944{
1945 return (struct dirty_seglist_info *)(SM_I(sbi)->dirty_info);
1946}
1947
1948static inline struct address_space *META_MAPPING(struct f2fs_sb_info *sbi)
1949{
1950 return sbi->meta_inode->i_mapping;
1951}
1952
1953static inline struct address_space *NODE_MAPPING(struct f2fs_sb_info *sbi)
1954{
1955 return sbi->node_inode->i_mapping;
1956}
1957
1958static inline bool is_sbi_flag_set(struct f2fs_sb_info *sbi, unsigned int type)
1959{
1960 return test_bit(type, &sbi->s_flag);
1961}
1962
1963static inline void set_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type)
1964{
1965 set_bit(type, &sbi->s_flag);
1966}
1967
1968static inline void clear_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type)
1969{
1970 clear_bit(type, &sbi->s_flag);
1971}
1972
1973static inline unsigned long long cur_cp_version(struct f2fs_checkpoint *cp)
1974{
1975 return le64_to_cpu(cp->checkpoint_ver);
1976}
1977
1978static inline unsigned long f2fs_qf_ino(struct super_block *sb, int type)
1979{
1980 if (type < F2FS_MAX_QUOTAS)
1981 return le32_to_cpu(F2FS_SB(sb)->raw_super->qf_ino[type]);
1982 return 0;
1983}
1984
1985static inline __u64 cur_cp_crc(struct f2fs_checkpoint *cp)
1986{
1987 size_t crc_offset = le32_to_cpu(cp->checksum_offset);
1988 return le32_to_cpu(*((__le32 *)((unsigned char *)cp + crc_offset)));
1989}
1990
1991static inline bool __is_set_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
1992{
1993 unsigned int ckpt_flags = le32_to_cpu(cp->ckpt_flags);
1994
1995 return ckpt_flags & f;
1996}
1997
1998static inline bool is_set_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
1999{
2000 return __is_set_ckpt_flags(F2FS_CKPT(sbi), f);
2001}
2002
2003static inline void __set_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
2004{
2005 unsigned int ckpt_flags;
2006
2007 ckpt_flags = le32_to_cpu(cp->ckpt_flags);
2008 ckpt_flags |= f;
2009 cp->ckpt_flags = cpu_to_le32(ckpt_flags);
2010}
2011
2012static inline void set_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
2013{
2014 unsigned long flags;
2015
2016 spin_lock_irqsave(&sbi->cp_lock, flags);
2017 __set_ckpt_flags(F2FS_CKPT(sbi), f);
2018 spin_unlock_irqrestore(&sbi->cp_lock, flags);
2019}
2020
2021static inline void __clear_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
2022{
2023 unsigned int ckpt_flags;
2024
2025 ckpt_flags = le32_to_cpu(cp->ckpt_flags);
2026 ckpt_flags &= (~f);
2027 cp->ckpt_flags = cpu_to_le32(ckpt_flags);
2028}
2029
2030static inline void clear_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
2031{
2032 unsigned long flags;
2033
2034 spin_lock_irqsave(&sbi->cp_lock, flags);
2035 __clear_ckpt_flags(F2FS_CKPT(sbi), f);
2036 spin_unlock_irqrestore(&sbi->cp_lock, flags);
2037}
2038
2039static inline void disable_nat_bits(struct f2fs_sb_info *sbi, bool lock)
2040{
2041 unsigned long flags;
2042 unsigned char *nat_bits;
2043
2044 /*
2045 * In order to re-enable nat_bits we need to call fsck.f2fs by
2046 * set_sbi_flag(sbi, SBI_NEED_FSCK). But it may give huge cost,
2047 * so let's rely on regular fsck or unclean shutdown.
2048 */
2049
2050 if (lock)
2051 spin_lock_irqsave(&sbi->cp_lock, flags);
2052 __clear_ckpt_flags(F2FS_CKPT(sbi), CP_NAT_BITS_FLAG);
2053 nat_bits = NM_I(sbi)->nat_bits;
2054 NM_I(sbi)->nat_bits = NULL;
2055 if (lock)
2056 spin_unlock_irqrestore(&sbi->cp_lock, flags);
2057
2058 kvfree(nat_bits);
2059}
2060
2061static inline bool enabled_nat_bits(struct f2fs_sb_info *sbi,
2062 struct cp_control *cpc)
2063{
2064 bool set = is_set_ckpt_flags(sbi, CP_NAT_BITS_FLAG);
2065
2066 return (cpc) ? (cpc->reason & CP_UMOUNT) && set : set;
2067}
2068
2069static inline void f2fs_lock_op(struct f2fs_sb_info *sbi)
2070{
2071 down_read(&sbi->cp_rwsem);
2072}
2073
2074static inline int f2fs_trylock_op(struct f2fs_sb_info *sbi)
2075{
2076 return down_read_trylock(&sbi->cp_rwsem);
2077}
2078
2079static inline void f2fs_unlock_op(struct f2fs_sb_info *sbi)
2080{
2081 up_read(&sbi->cp_rwsem);
2082}
2083
2084static inline void f2fs_lock_all(struct f2fs_sb_info *sbi)
2085{
2086 down_write(&sbi->cp_rwsem);
2087}
2088
2089static inline void f2fs_unlock_all(struct f2fs_sb_info *sbi)
2090{
2091 up_write(&sbi->cp_rwsem);
2092}
2093
2094static inline int __get_cp_reason(struct f2fs_sb_info *sbi)
2095{
2096 int reason = CP_SYNC;
2097
2098 if (test_opt(sbi, FASTBOOT))
2099 reason = CP_FASTBOOT;
2100 if (is_sbi_flag_set(sbi, SBI_IS_CLOSE))
2101 reason = CP_UMOUNT;
2102 return reason;
2103}
2104
2105static inline bool __remain_node_summaries(int reason)
2106{
2107 return (reason & (CP_UMOUNT | CP_FASTBOOT));
2108}
2109
2110static inline bool __exist_node_summaries(struct f2fs_sb_info *sbi)
2111{
2112 return (is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG) ||
2113 is_set_ckpt_flags(sbi, CP_FASTBOOT_FLAG));
2114}
2115
2116/*
2117 * Check whether the inode has blocks or not
2118 */
2119static inline int F2FS_HAS_BLOCKS(struct inode *inode)
2120{
2121 block_t xattr_block = F2FS_I(inode)->i_xattr_nid ? 1 : 0;
2122
2123 return (inode->i_blocks >> F2FS_LOG_SECTORS_PER_BLOCK) > xattr_block;
2124}
2125
2126static inline bool f2fs_has_xattr_block(unsigned int ofs)
2127{
2128 return ofs == XATTR_NODE_OFFSET;
2129}
2130
2131static inline bool __allow_reserved_blocks(struct f2fs_sb_info *sbi,
2132 struct inode *inode, bool cap)
2133{
2134 if (!inode)
2135 return true;
2136 if (!test_opt(sbi, RESERVE_ROOT))
2137 return false;
2138 if (IS_NOQUOTA(inode))
2139 return true;
2140 if (uid_eq(F2FS_OPTION(sbi).s_resuid, current_fsuid()))
2141 return true;
2142 if (!gid_eq(F2FS_OPTION(sbi).s_resgid, GLOBAL_ROOT_GID) &&
2143 in_group_p(F2FS_OPTION(sbi).s_resgid))
2144 return true;
2145 if (cap && capable(CAP_SYS_RESOURCE))
2146 return true;
2147 return false;
2148}
2149
2150static inline void f2fs_i_blocks_write(struct inode *, block_t, bool, bool);
2151static inline int inc_valid_block_count(struct f2fs_sb_info *sbi,
2152 struct inode *inode, blkcnt_t *count)
2153{
2154 blkcnt_t diff = 0, release = 0;
2155 block_t avail_user_block_count;
2156 int ret;
2157
2158 ret = dquot_reserve_block(inode, *count);
2159 if (ret)
2160 return ret;
2161
2162 if (time_to_inject(sbi, FAULT_BLOCK)) {
2163 f2fs_show_injection_info(sbi, FAULT_BLOCK);
2164 release = *count;
2165 goto release_quota;
2166 }
2167
2168 /*
2169 * let's increase this in prior to actual block count change in order
2170 * for f2fs_sync_file to avoid data races when deciding checkpoint.
2171 */
2172 percpu_counter_add(&sbi->alloc_valid_block_count, (*count));
2173
2174 spin_lock(&sbi->stat_lock);
2175 sbi->total_valid_block_count += (block_t)(*count);
2176 avail_user_block_count = sbi->user_block_count -
2177 sbi->current_reserved_blocks;
2178
2179 if (!__allow_reserved_blocks(sbi, inode, true))
2180 avail_user_block_count -= F2FS_OPTION(sbi).root_reserved_blocks;
2181 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2182 if (avail_user_block_count > sbi->unusable_block_count)
2183 avail_user_block_count -= sbi->unusable_block_count;
2184 else
2185 avail_user_block_count = 0;
2186 }
2187 if (unlikely(sbi->total_valid_block_count > avail_user_block_count)) {
2188 diff = sbi->total_valid_block_count - avail_user_block_count;
2189 if (diff > *count)
2190 diff = *count;
2191 *count -= diff;
2192 release = diff;
2193 sbi->total_valid_block_count -= diff;
2194 if (!*count) {
2195 spin_unlock(&sbi->stat_lock);
2196 goto enospc;
2197 }
2198 }
2199 spin_unlock(&sbi->stat_lock);
2200
2201 if (unlikely(release)) {
2202 percpu_counter_sub(&sbi->alloc_valid_block_count, release);
2203 dquot_release_reservation_block(inode, release);
2204 }
2205 f2fs_i_blocks_write(inode, *count, true, true);
2206 return 0;
2207
2208enospc:
2209 percpu_counter_sub(&sbi->alloc_valid_block_count, release);
2210release_quota:
2211 dquot_release_reservation_block(inode, release);
2212 return -ENOSPC;
2213}
2214
2215__printf(2, 3)
2216void f2fs_printk(struct f2fs_sb_info *sbi, const char *fmt, ...);
2217
2218#define f2fs_err(sbi, fmt, ...) \
2219 f2fs_printk(sbi, KERN_ERR fmt, ##__VA_ARGS__)
2220#define f2fs_warn(sbi, fmt, ...) \
2221 f2fs_printk(sbi, KERN_WARNING fmt, ##__VA_ARGS__)
2222#define f2fs_notice(sbi, fmt, ...) \
2223 f2fs_printk(sbi, KERN_NOTICE fmt, ##__VA_ARGS__)
2224#define f2fs_info(sbi, fmt, ...) \
2225 f2fs_printk(sbi, KERN_INFO fmt, ##__VA_ARGS__)
2226#define f2fs_debug(sbi, fmt, ...) \
2227 f2fs_printk(sbi, KERN_DEBUG fmt, ##__VA_ARGS__)
2228
2229static inline void dec_valid_block_count(struct f2fs_sb_info *sbi,
2230 struct inode *inode,
2231 block_t count)
2232{
2233 blkcnt_t sectors = count << F2FS_LOG_SECTORS_PER_BLOCK;
2234
2235 spin_lock(&sbi->stat_lock);
2236 f2fs_bug_on(sbi, sbi->total_valid_block_count < (block_t) count);
2237 sbi->total_valid_block_count -= (block_t)count;
2238 if (sbi->reserved_blocks &&
2239 sbi->current_reserved_blocks < sbi->reserved_blocks)
2240 sbi->current_reserved_blocks = min(sbi->reserved_blocks,
2241 sbi->current_reserved_blocks + count);
2242 spin_unlock(&sbi->stat_lock);
2243 if (unlikely(inode->i_blocks < sectors)) {
2244 f2fs_warn(sbi, "Inconsistent i_blocks, ino:%lu, iblocks:%llu, sectors:%llu",
2245 inode->i_ino,
2246 (unsigned long long)inode->i_blocks,
2247 (unsigned long long)sectors);
2248 set_sbi_flag(sbi, SBI_NEED_FSCK);
2249 return;
2250 }
2251 f2fs_i_blocks_write(inode, count, false, true);
2252}
2253
2254static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type)
2255{
2256 atomic_inc(&sbi->nr_pages[count_type]);
2257
2258 if (count_type == F2FS_DIRTY_DENTS ||
2259 count_type == F2FS_DIRTY_NODES ||
2260 count_type == F2FS_DIRTY_META ||
2261 count_type == F2FS_DIRTY_QDATA ||
2262 count_type == F2FS_DIRTY_IMETA)
2263 set_sbi_flag(sbi, SBI_IS_DIRTY);
2264}
2265
2266static inline void inode_inc_dirty_pages(struct inode *inode)
2267{
2268 atomic_inc(&F2FS_I(inode)->dirty_pages);
2269 inc_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
2270 F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA);
2271 if (IS_NOQUOTA(inode))
2272 inc_page_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA);
2273}
2274
2275static inline void dec_page_count(struct f2fs_sb_info *sbi, int count_type)
2276{
2277 atomic_dec(&sbi->nr_pages[count_type]);
2278}
2279
2280static inline void inode_dec_dirty_pages(struct inode *inode)
2281{
2282 if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) &&
2283 !S_ISLNK(inode->i_mode))
2284 return;
2285
2286 atomic_dec(&F2FS_I(inode)->dirty_pages);
2287 dec_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
2288 F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA);
2289 if (IS_NOQUOTA(inode))
2290 dec_page_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA);
2291}
2292
2293static inline s64 get_pages(struct f2fs_sb_info *sbi, int count_type)
2294{
2295 return atomic_read(&sbi->nr_pages[count_type]);
2296}
2297
2298static inline int get_dirty_pages(struct inode *inode)
2299{
2300 return atomic_read(&F2FS_I(inode)->dirty_pages);
2301}
2302
2303static inline int get_blocktype_secs(struct f2fs_sb_info *sbi, int block_type)
2304{
2305 unsigned int pages_per_sec = sbi->segs_per_sec * sbi->blocks_per_seg;
2306 unsigned int segs = (get_pages(sbi, block_type) + pages_per_sec - 1) >>
2307 sbi->log_blocks_per_seg;
2308
2309 return segs / sbi->segs_per_sec;
2310}
2311
2312static inline block_t valid_user_blocks(struct f2fs_sb_info *sbi)
2313{
2314 return sbi->total_valid_block_count;
2315}
2316
2317static inline block_t discard_blocks(struct f2fs_sb_info *sbi)
2318{
2319 return sbi->discard_blks;
2320}
2321
2322static inline unsigned long __bitmap_size(struct f2fs_sb_info *sbi, int flag)
2323{
2324 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2325
2326 /* return NAT or SIT bitmap */
2327 if (flag == NAT_BITMAP)
2328 return le32_to_cpu(ckpt->nat_ver_bitmap_bytesize);
2329 else if (flag == SIT_BITMAP)
2330 return le32_to_cpu(ckpt->sit_ver_bitmap_bytesize);
2331
2332 return 0;
2333}
2334
2335static inline block_t __cp_payload(struct f2fs_sb_info *sbi)
2336{
2337 return le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_payload);
2338}
2339
2340static inline void *__bitmap_ptr(struct f2fs_sb_info *sbi, int flag)
2341{
2342 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2343 void *tmp_ptr = &ckpt->sit_nat_version_bitmap;
2344 int offset;
2345
2346 if (is_set_ckpt_flags(sbi, CP_LARGE_NAT_BITMAP_FLAG)) {
2347 offset = (flag == SIT_BITMAP) ?
2348 le32_to_cpu(ckpt->nat_ver_bitmap_bytesize) : 0;
2349 /*
2350 * if large_nat_bitmap feature is enabled, leave checksum
2351 * protection for all nat/sit bitmaps.
2352 */
2353 return tmp_ptr + offset + sizeof(__le32);
2354 }
2355
2356 if (__cp_payload(sbi) > 0) {
2357 if (flag == NAT_BITMAP)
2358 return &ckpt->sit_nat_version_bitmap;
2359 else
2360 return (unsigned char *)ckpt + F2FS_BLKSIZE;
2361 } else {
2362 offset = (flag == NAT_BITMAP) ?
2363 le32_to_cpu(ckpt->sit_ver_bitmap_bytesize) : 0;
2364 return tmp_ptr + offset;
2365 }
2366}
2367
2368static inline block_t __start_cp_addr(struct f2fs_sb_info *sbi)
2369{
2370 block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr);
2371
2372 if (sbi->cur_cp_pack == 2)
2373 start_addr += sbi->blocks_per_seg;
2374 return start_addr;
2375}
2376
2377static inline block_t __start_cp_next_addr(struct f2fs_sb_info *sbi)
2378{
2379 block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr);
2380
2381 if (sbi->cur_cp_pack == 1)
2382 start_addr += sbi->blocks_per_seg;
2383 return start_addr;
2384}
2385
2386static inline void __set_cp_next_pack(struct f2fs_sb_info *sbi)
2387{
2388 sbi->cur_cp_pack = (sbi->cur_cp_pack == 1) ? 2 : 1;
2389}
2390
2391static inline block_t __start_sum_addr(struct f2fs_sb_info *sbi)
2392{
2393 return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum);
2394}
2395
2396static inline int inc_valid_node_count(struct f2fs_sb_info *sbi,
2397 struct inode *inode, bool is_inode)
2398{
2399 block_t valid_block_count;
2400 unsigned int valid_node_count, user_block_count;
2401 int err;
2402
2403 if (is_inode) {
2404 if (inode) {
2405 err = dquot_alloc_inode(inode);
2406 if (err)
2407 return err;
2408 }
2409 } else {
2410 err = dquot_reserve_block(inode, 1);
2411 if (err)
2412 return err;
2413 }
2414
2415 if (time_to_inject(sbi, FAULT_BLOCK)) {
2416 f2fs_show_injection_info(sbi, FAULT_BLOCK);
2417 goto enospc;
2418 }
2419
2420 spin_lock(&sbi->stat_lock);
2421
2422 valid_block_count = sbi->total_valid_block_count +
2423 sbi->current_reserved_blocks + 1;
2424
2425 if (!__allow_reserved_blocks(sbi, inode, false))
2426 valid_block_count += F2FS_OPTION(sbi).root_reserved_blocks;
2427 user_block_count = sbi->user_block_count;
2428 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2429 user_block_count -= sbi->unusable_block_count;
2430
2431 if (unlikely(valid_block_count > user_block_count)) {
2432 spin_unlock(&sbi->stat_lock);
2433 goto enospc;
2434 }
2435
2436 valid_node_count = sbi->total_valid_node_count + 1;
2437 if (unlikely(valid_node_count > sbi->total_node_count)) {
2438 spin_unlock(&sbi->stat_lock);
2439 goto enospc;
2440 }
2441
2442 sbi->total_valid_node_count++;
2443 sbi->total_valid_block_count++;
2444 spin_unlock(&sbi->stat_lock);
2445
2446 if (inode) {
2447 if (is_inode)
2448 f2fs_mark_inode_dirty_sync(inode, true);
2449 else
2450 f2fs_i_blocks_write(inode, 1, true, true);
2451 }
2452
2453 percpu_counter_inc(&sbi->alloc_valid_block_count);
2454 return 0;
2455
2456enospc:
2457 if (is_inode) {
2458 if (inode)
2459 dquot_free_inode(inode);
2460 } else {
2461 dquot_release_reservation_block(inode, 1);
2462 }
2463 return -ENOSPC;
2464}
2465
2466static inline void dec_valid_node_count(struct f2fs_sb_info *sbi,
2467 struct inode *inode, bool is_inode)
2468{
2469 spin_lock(&sbi->stat_lock);
2470
2471 f2fs_bug_on(sbi, !sbi->total_valid_block_count);
2472 f2fs_bug_on(sbi, !sbi->total_valid_node_count);
2473
2474 sbi->total_valid_node_count--;
2475 sbi->total_valid_block_count--;
2476 if (sbi->reserved_blocks &&
2477 sbi->current_reserved_blocks < sbi->reserved_blocks)
2478 sbi->current_reserved_blocks++;
2479
2480 spin_unlock(&sbi->stat_lock);
2481
2482 if (is_inode) {
2483 dquot_free_inode(inode);
2484 } else {
2485 if (unlikely(inode->i_blocks == 0)) {
2486 f2fs_warn(sbi, "dec_valid_node_count: inconsistent i_blocks, ino:%lu, iblocks:%llu",
2487 inode->i_ino,
2488 (unsigned long long)inode->i_blocks);
2489 set_sbi_flag(sbi, SBI_NEED_FSCK);
2490 return;
2491 }
2492 f2fs_i_blocks_write(inode, 1, false, true);
2493 }
2494}
2495
2496static inline unsigned int valid_node_count(struct f2fs_sb_info *sbi)
2497{
2498 return sbi->total_valid_node_count;
2499}
2500
2501static inline void inc_valid_inode_count(struct f2fs_sb_info *sbi)
2502{
2503 percpu_counter_inc(&sbi->total_valid_inode_count);
2504}
2505
2506static inline void dec_valid_inode_count(struct f2fs_sb_info *sbi)
2507{
2508 percpu_counter_dec(&sbi->total_valid_inode_count);
2509}
2510
2511static inline s64 valid_inode_count(struct f2fs_sb_info *sbi)
2512{
2513 return percpu_counter_sum_positive(&sbi->total_valid_inode_count);
2514}
2515
2516static inline struct page *f2fs_grab_cache_page(struct address_space *mapping,
2517 pgoff_t index, bool for_write)
2518{
2519 struct page *page;
2520
2521 if (IS_ENABLED(CONFIG_F2FS_FAULT_INJECTION)) {
2522 if (!for_write)
2523 page = find_get_page_flags(mapping, index,
2524 FGP_LOCK | FGP_ACCESSED);
2525 else
2526 page = find_lock_page(mapping, index);
2527 if (page)
2528 return page;
2529
2530 if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_ALLOC)) {
2531 f2fs_show_injection_info(F2FS_M_SB(mapping),
2532 FAULT_PAGE_ALLOC);
2533 return NULL;
2534 }
2535 }
2536
2537 if (!for_write)
2538 return grab_cache_page(mapping, index);
2539 return grab_cache_page_write_begin(mapping, index, AOP_FLAG_NOFS);
2540}
2541
2542static inline struct page *f2fs_pagecache_get_page(
2543 struct address_space *mapping, pgoff_t index,
2544 int fgp_flags, gfp_t gfp_mask)
2545{
2546 if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_GET)) {
2547 f2fs_show_injection_info(F2FS_M_SB(mapping), FAULT_PAGE_GET);
2548 return NULL;
2549 }
2550
2551 return pagecache_get_page(mapping, index, fgp_flags, gfp_mask);
2552}
2553
2554static inline void f2fs_copy_page(struct page *src, struct page *dst)
2555{
2556 char *src_kaddr = kmap(src);
2557 char *dst_kaddr = kmap(dst);
2558
2559 memcpy(dst_kaddr, src_kaddr, PAGE_SIZE);
2560 kunmap(dst);
2561 kunmap(src);
2562}
2563
2564static inline void f2fs_put_page(struct page *page, int unlock)
2565{
2566 if (!page)
2567 return;
2568
2569 if (unlock) {
2570 f2fs_bug_on(F2FS_P_SB(page), !PageLocked(page));
2571 unlock_page(page);
2572 }
2573 put_page(page);
2574}
2575
2576static inline void f2fs_put_dnode(struct dnode_of_data *dn)
2577{
2578 if (dn->node_page)
2579 f2fs_put_page(dn->node_page, 1);
2580 if (dn->inode_page && dn->node_page != dn->inode_page)
2581 f2fs_put_page(dn->inode_page, 0);
2582 dn->node_page = NULL;
2583 dn->inode_page = NULL;
2584}
2585
2586static inline struct kmem_cache *f2fs_kmem_cache_create(const char *name,
2587 size_t size)
2588{
2589 return kmem_cache_create(name, size, 0, SLAB_RECLAIM_ACCOUNT, NULL);
2590}
2591
2592static inline void *f2fs_kmem_cache_alloc(struct kmem_cache *cachep,
2593 gfp_t flags)
2594{
2595 void *entry;
2596
2597 entry = kmem_cache_alloc(cachep, flags);
2598 if (!entry)
2599 entry = kmem_cache_alloc(cachep, flags | __GFP_NOFAIL);
2600 return entry;
2601}
2602
2603static inline bool is_inflight_io(struct f2fs_sb_info *sbi, int type)
2604{
2605 if (get_pages(sbi, F2FS_RD_DATA) || get_pages(sbi, F2FS_RD_NODE) ||
2606 get_pages(sbi, F2FS_RD_META) || get_pages(sbi, F2FS_WB_DATA) ||
2607 get_pages(sbi, F2FS_WB_CP_DATA) ||
2608 get_pages(sbi, F2FS_DIO_READ) ||
2609 get_pages(sbi, F2FS_DIO_WRITE))
2610 return true;
2611
2612 if (type != DISCARD_TIME && SM_I(sbi) && SM_I(sbi)->dcc_info &&
2613 atomic_read(&SM_I(sbi)->dcc_info->queued_discard))
2614 return true;
2615
2616 if (SM_I(sbi) && SM_I(sbi)->fcc_info &&
2617 atomic_read(&SM_I(sbi)->fcc_info->queued_flush))
2618 return true;
2619 return false;
2620}
2621
2622static inline bool is_idle(struct f2fs_sb_info *sbi, int type)
2623{
2624 if (sbi->gc_mode == GC_URGENT_HIGH)
2625 return true;
2626
2627 if (is_inflight_io(sbi, type))
2628 return false;
2629
2630 if (sbi->gc_mode == GC_URGENT_LOW &&
2631 (type == DISCARD_TIME || type == GC_TIME))
2632 return true;
2633
2634 return f2fs_time_over(sbi, type);
2635}
2636
2637static inline void f2fs_radix_tree_insert(struct radix_tree_root *root,
2638 unsigned long index, void *item)
2639{
2640 while (radix_tree_insert(root, index, item))
2641 cond_resched();
2642}
2643
2644#define RAW_IS_INODE(p) ((p)->footer.nid == (p)->footer.ino)
2645
2646static inline bool IS_INODE(struct page *page)
2647{
2648 struct f2fs_node *p = F2FS_NODE(page);
2649
2650 return RAW_IS_INODE(p);
2651}
2652
2653static inline int offset_in_addr(struct f2fs_inode *i)
2654{
2655 return (i->i_inline & F2FS_EXTRA_ATTR) ?
2656 (le16_to_cpu(i->i_extra_isize) / sizeof(__le32)) : 0;
2657}
2658
2659static inline __le32 *blkaddr_in_node(struct f2fs_node *node)
2660{
2661 return RAW_IS_INODE(node) ? node->i.i_addr : node->dn.addr;
2662}
2663
2664static inline int f2fs_has_extra_attr(struct inode *inode);
2665static inline block_t data_blkaddr(struct inode *inode,
2666 struct page *node_page, unsigned int offset)
2667{
2668 struct f2fs_node *raw_node;
2669 __le32 *addr_array;
2670 int base = 0;
2671 bool is_inode = IS_INODE(node_page);
2672
2673 raw_node = F2FS_NODE(node_page);
2674
2675 if (is_inode) {
2676 if (!inode)
2677 /* from GC path only */
2678 base = offset_in_addr(&raw_node->i);
2679 else if (f2fs_has_extra_attr(inode))
2680 base = get_extra_isize(inode);
2681 }
2682
2683 addr_array = blkaddr_in_node(raw_node);
2684 return le32_to_cpu(addr_array[base + offset]);
2685}
2686
2687static inline block_t f2fs_data_blkaddr(struct dnode_of_data *dn)
2688{
2689 return data_blkaddr(dn->inode, dn->node_page, dn->ofs_in_node);
2690}
2691
2692static inline int f2fs_test_bit(unsigned int nr, char *addr)
2693{
2694 int mask;
2695
2696 addr += (nr >> 3);
2697 mask = 1 << (7 - (nr & 0x07));
2698 return mask & *addr;
2699}
2700
2701static inline void f2fs_set_bit(unsigned int nr, char *addr)
2702{
2703 int mask;
2704
2705 addr += (nr >> 3);
2706 mask = 1 << (7 - (nr & 0x07));
2707 *addr |= mask;
2708}
2709
2710static inline void f2fs_clear_bit(unsigned int nr, char *addr)
2711{
2712 int mask;
2713
2714 addr += (nr >> 3);
2715 mask = 1 << (7 - (nr & 0x07));
2716 *addr &= ~mask;
2717}
2718
2719static inline int f2fs_test_and_set_bit(unsigned int nr, char *addr)
2720{
2721 int mask;
2722 int ret;
2723
2724 addr += (nr >> 3);
2725 mask = 1 << (7 - (nr & 0x07));
2726 ret = mask & *addr;
2727 *addr |= mask;
2728 return ret;
2729}
2730
2731static inline int f2fs_test_and_clear_bit(unsigned int nr, char *addr)
2732{
2733 int mask;
2734 int ret;
2735
2736 addr += (nr >> 3);
2737 mask = 1 << (7 - (nr & 0x07));
2738 ret = mask & *addr;
2739 *addr &= ~mask;
2740 return ret;
2741}
2742
2743static inline void f2fs_change_bit(unsigned int nr, char *addr)
2744{
2745 int mask;
2746
2747 addr += (nr >> 3);
2748 mask = 1 << (7 - (nr & 0x07));
2749 *addr ^= mask;
2750}
2751
2752/*
2753 * On-disk inode flags (f2fs_inode::i_flags)
2754 */
2755#define F2FS_COMPR_FL 0x00000004 /* Compress file */
2756#define F2FS_SYNC_FL 0x00000008 /* Synchronous updates */
2757#define F2FS_IMMUTABLE_FL 0x00000010 /* Immutable file */
2758#define F2FS_APPEND_FL 0x00000020 /* writes to file may only append */
2759#define F2FS_NODUMP_FL 0x00000040 /* do not dump file */
2760#define F2FS_NOATIME_FL 0x00000080 /* do not update atime */
2761#define F2FS_NOCOMP_FL 0x00000400 /* Don't compress */
2762#define F2FS_INDEX_FL 0x00001000 /* hash-indexed directory */
2763#define F2FS_DIRSYNC_FL 0x00010000 /* dirsync behaviour (directories only) */
2764#define F2FS_PROJINHERIT_FL 0x20000000 /* Create with parents projid */
2765#define F2FS_CASEFOLD_FL 0x40000000 /* Casefolded file */
2766
2767/* Flags that should be inherited by new inodes from their parent. */
2768#define F2FS_FL_INHERITED (F2FS_SYNC_FL | F2FS_NODUMP_FL | F2FS_NOATIME_FL | \
2769 F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL | \
2770 F2FS_CASEFOLD_FL | F2FS_COMPR_FL | F2FS_NOCOMP_FL)
2771
2772/* Flags that are appropriate for regular files (all but dir-specific ones). */
2773#define F2FS_REG_FLMASK (~(F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL | \
2774 F2FS_CASEFOLD_FL))
2775
2776/* Flags that are appropriate for non-directories/regular files. */
2777#define F2FS_OTHER_FLMASK (F2FS_NODUMP_FL | F2FS_NOATIME_FL)
2778
2779static inline __u32 f2fs_mask_flags(umode_t mode, __u32 flags)
2780{
2781 if (S_ISDIR(mode))
2782 return flags;
2783 else if (S_ISREG(mode))
2784 return flags & F2FS_REG_FLMASK;
2785 else
2786 return flags & F2FS_OTHER_FLMASK;
2787}
2788
2789static inline void __mark_inode_dirty_flag(struct inode *inode,
2790 int flag, bool set)
2791{
2792 switch (flag) {
2793 case FI_INLINE_XATTR:
2794 case FI_INLINE_DATA:
2795 case FI_INLINE_DENTRY:
2796 case FI_NEW_INODE:
2797 if (set)
2798 return;
2799 fallthrough;
2800 case FI_DATA_EXIST:
2801 case FI_INLINE_DOTS:
2802 case FI_PIN_FILE:
2803 case FI_COMPRESS_RELEASED:
2804 f2fs_mark_inode_dirty_sync(inode, true);
2805 }
2806}
2807
2808static inline void set_inode_flag(struct inode *inode, int flag)
2809{
2810 set_bit(flag, F2FS_I(inode)->flags);
2811 __mark_inode_dirty_flag(inode, flag, true);
2812}
2813
2814static inline int is_inode_flag_set(struct inode *inode, int flag)
2815{
2816 return test_bit(flag, F2FS_I(inode)->flags);
2817}
2818
2819static inline void clear_inode_flag(struct inode *inode, int flag)
2820{
2821 clear_bit(flag, F2FS_I(inode)->flags);
2822 __mark_inode_dirty_flag(inode, flag, false);
2823}
2824
2825static inline bool f2fs_verity_in_progress(struct inode *inode)
2826{
2827 return IS_ENABLED(CONFIG_FS_VERITY) &&
2828 is_inode_flag_set(inode, FI_VERITY_IN_PROGRESS);
2829}
2830
2831static inline void set_acl_inode(struct inode *inode, umode_t mode)
2832{
2833 F2FS_I(inode)->i_acl_mode = mode;
2834 set_inode_flag(inode, FI_ACL_MODE);
2835 f2fs_mark_inode_dirty_sync(inode, false);
2836}
2837
2838static inline void f2fs_i_links_write(struct inode *inode, bool inc)
2839{
2840 if (inc)
2841 inc_nlink(inode);
2842 else
2843 drop_nlink(inode);
2844 f2fs_mark_inode_dirty_sync(inode, true);
2845}
2846
2847static inline void f2fs_i_blocks_write(struct inode *inode,
2848 block_t diff, bool add, bool claim)
2849{
2850 bool clean = !is_inode_flag_set(inode, FI_DIRTY_INODE);
2851 bool recover = is_inode_flag_set(inode, FI_AUTO_RECOVER);
2852
2853 /* add = 1, claim = 1 should be dquot_reserve_block in pair */
2854 if (add) {
2855 if (claim)
2856 dquot_claim_block(inode, diff);
2857 else
2858 dquot_alloc_block_nofail(inode, diff);
2859 } else {
2860 dquot_free_block(inode, diff);
2861 }
2862
2863 f2fs_mark_inode_dirty_sync(inode, true);
2864 if (clean || recover)
2865 set_inode_flag(inode, FI_AUTO_RECOVER);
2866}
2867
2868static inline void f2fs_i_size_write(struct inode *inode, loff_t i_size)
2869{
2870 bool clean = !is_inode_flag_set(inode, FI_DIRTY_INODE);
2871 bool recover = is_inode_flag_set(inode, FI_AUTO_RECOVER);
2872
2873 if (i_size_read(inode) == i_size)
2874 return;
2875
2876 i_size_write(inode, i_size);
2877 f2fs_mark_inode_dirty_sync(inode, true);
2878 if (clean || recover)
2879 set_inode_flag(inode, FI_AUTO_RECOVER);
2880}
2881
2882static inline void f2fs_i_depth_write(struct inode *inode, unsigned int depth)
2883{
2884 F2FS_I(inode)->i_current_depth = depth;
2885 f2fs_mark_inode_dirty_sync(inode, true);
2886}
2887
2888static inline void f2fs_i_gc_failures_write(struct inode *inode,
2889 unsigned int count)
2890{
2891 F2FS_I(inode)->i_gc_failures[GC_FAILURE_PIN] = count;
2892 f2fs_mark_inode_dirty_sync(inode, true);
2893}
2894
2895static inline void f2fs_i_xnid_write(struct inode *inode, nid_t xnid)
2896{
2897 F2FS_I(inode)->i_xattr_nid = xnid;
2898 f2fs_mark_inode_dirty_sync(inode, true);
2899}
2900
2901static inline void f2fs_i_pino_write(struct inode *inode, nid_t pino)
2902{
2903 F2FS_I(inode)->i_pino = pino;
2904 f2fs_mark_inode_dirty_sync(inode, true);
2905}
2906
2907static inline void get_inline_info(struct inode *inode, struct f2fs_inode *ri)
2908{
2909 struct f2fs_inode_info *fi = F2FS_I(inode);
2910
2911 if (ri->i_inline & F2FS_INLINE_XATTR)
2912 set_bit(FI_INLINE_XATTR, fi->flags);
2913 if (ri->i_inline & F2FS_INLINE_DATA)
2914 set_bit(FI_INLINE_DATA, fi->flags);
2915 if (ri->i_inline & F2FS_INLINE_DENTRY)
2916 set_bit(FI_INLINE_DENTRY, fi->flags);
2917 if (ri->i_inline & F2FS_DATA_EXIST)
2918 set_bit(FI_DATA_EXIST, fi->flags);
2919 if (ri->i_inline & F2FS_INLINE_DOTS)
2920 set_bit(FI_INLINE_DOTS, fi->flags);
2921 if (ri->i_inline & F2FS_EXTRA_ATTR)
2922 set_bit(FI_EXTRA_ATTR, fi->flags);
2923 if (ri->i_inline & F2FS_PIN_FILE)
2924 set_bit(FI_PIN_FILE, fi->flags);
2925 if (ri->i_inline & F2FS_COMPRESS_RELEASED)
2926 set_bit(FI_COMPRESS_RELEASED, fi->flags);
2927}
2928
2929static inline void set_raw_inline(struct inode *inode, struct f2fs_inode *ri)
2930{
2931 ri->i_inline = 0;
2932
2933 if (is_inode_flag_set(inode, FI_INLINE_XATTR))
2934 ri->i_inline |= F2FS_INLINE_XATTR;
2935 if (is_inode_flag_set(inode, FI_INLINE_DATA))
2936 ri->i_inline |= F2FS_INLINE_DATA;
2937 if (is_inode_flag_set(inode, FI_INLINE_DENTRY))
2938 ri->i_inline |= F2FS_INLINE_DENTRY;
2939 if (is_inode_flag_set(inode, FI_DATA_EXIST))
2940 ri->i_inline |= F2FS_DATA_EXIST;
2941 if (is_inode_flag_set(inode, FI_INLINE_DOTS))
2942 ri->i_inline |= F2FS_INLINE_DOTS;
2943 if (is_inode_flag_set(inode, FI_EXTRA_ATTR))
2944 ri->i_inline |= F2FS_EXTRA_ATTR;
2945 if (is_inode_flag_set(inode, FI_PIN_FILE))
2946 ri->i_inline |= F2FS_PIN_FILE;
2947 if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED))
2948 ri->i_inline |= F2FS_COMPRESS_RELEASED;
2949}
2950
2951static inline int f2fs_has_extra_attr(struct inode *inode)
2952{
2953 return is_inode_flag_set(inode, FI_EXTRA_ATTR);
2954}
2955
2956static inline int f2fs_has_inline_xattr(struct inode *inode)
2957{
2958 return is_inode_flag_set(inode, FI_INLINE_XATTR);
2959}
2960
2961static inline int f2fs_compressed_file(struct inode *inode)
2962{
2963 return S_ISREG(inode->i_mode) &&
2964 is_inode_flag_set(inode, FI_COMPRESSED_FILE);
2965}
2966
2967static inline bool f2fs_need_compress_data(struct inode *inode)
2968{
2969 int compress_mode = F2FS_OPTION(F2FS_I_SB(inode)).compress_mode;
2970
2971 if (!f2fs_compressed_file(inode))
2972 return false;
2973
2974 if (compress_mode == COMPR_MODE_FS)
2975 return true;
2976 else if (compress_mode == COMPR_MODE_USER &&
2977 is_inode_flag_set(inode, FI_ENABLE_COMPRESS))
2978 return true;
2979
2980 return false;
2981}
2982
2983static inline unsigned int addrs_per_inode(struct inode *inode)
2984{
2985 unsigned int addrs = CUR_ADDRS_PER_INODE(inode) -
2986 get_inline_xattr_addrs(inode);
2987
2988 if (!f2fs_compressed_file(inode))
2989 return addrs;
2990 return ALIGN_DOWN(addrs, F2FS_I(inode)->i_cluster_size);
2991}
2992
2993static inline unsigned int addrs_per_block(struct inode *inode)
2994{
2995 if (!f2fs_compressed_file(inode))
2996 return DEF_ADDRS_PER_BLOCK;
2997 return ALIGN_DOWN(DEF_ADDRS_PER_BLOCK, F2FS_I(inode)->i_cluster_size);
2998}
2999
3000static inline void *inline_xattr_addr(struct inode *inode, struct page *page)
3001{
3002 struct f2fs_inode *ri = F2FS_INODE(page);
3003
3004 return (void *)&(ri->i_addr[DEF_ADDRS_PER_INODE -
3005 get_inline_xattr_addrs(inode)]);
3006}
3007
3008static inline int inline_xattr_size(struct inode *inode)
3009{
3010 if (f2fs_has_inline_xattr(inode))
3011 return get_inline_xattr_addrs(inode) * sizeof(__le32);
3012 return 0;
3013}
3014
3015static inline int f2fs_has_inline_data(struct inode *inode)
3016{
3017 return is_inode_flag_set(inode, FI_INLINE_DATA);
3018}
3019
3020static inline int f2fs_exist_data(struct inode *inode)
3021{
3022 return is_inode_flag_set(inode, FI_DATA_EXIST);
3023}
3024
3025static inline int f2fs_has_inline_dots(struct inode *inode)
3026{
3027 return is_inode_flag_set(inode, FI_INLINE_DOTS);
3028}
3029
3030static inline int f2fs_is_mmap_file(struct inode *inode)
3031{
3032 return is_inode_flag_set(inode, FI_MMAP_FILE);
3033}
3034
3035static inline bool f2fs_is_pinned_file(struct inode *inode)
3036{
3037 return is_inode_flag_set(inode, FI_PIN_FILE);
3038}
3039
3040static inline bool f2fs_is_atomic_file(struct inode *inode)
3041{
3042 return is_inode_flag_set(inode, FI_ATOMIC_FILE);
3043}
3044
3045static inline bool f2fs_is_commit_atomic_write(struct inode *inode)
3046{
3047 return is_inode_flag_set(inode, FI_ATOMIC_COMMIT);
3048}
3049
3050static inline bool f2fs_is_volatile_file(struct inode *inode)
3051{
3052 return is_inode_flag_set(inode, FI_VOLATILE_FILE);
3053}
3054
3055static inline bool f2fs_is_first_block_written(struct inode *inode)
3056{
3057 return is_inode_flag_set(inode, FI_FIRST_BLOCK_WRITTEN);
3058}
3059
3060static inline bool f2fs_is_drop_cache(struct inode *inode)
3061{
3062 return is_inode_flag_set(inode, FI_DROP_CACHE);
3063}
3064
3065static inline void *inline_data_addr(struct inode *inode, struct page *page)
3066{
3067 struct f2fs_inode *ri = F2FS_INODE(page);
3068 int extra_size = get_extra_isize(inode);
3069
3070 return (void *)&(ri->i_addr[extra_size + DEF_INLINE_RESERVED_SIZE]);
3071}
3072
3073static inline int f2fs_has_inline_dentry(struct inode *inode)
3074{
3075 return is_inode_flag_set(inode, FI_INLINE_DENTRY);
3076}
3077
3078static inline int is_file(struct inode *inode, int type)
3079{
3080 return F2FS_I(inode)->i_advise & type;
3081}
3082
3083static inline void set_file(struct inode *inode, int type)
3084{
3085 F2FS_I(inode)->i_advise |= type;
3086 f2fs_mark_inode_dirty_sync(inode, true);
3087}
3088
3089static inline void clear_file(struct inode *inode, int type)
3090{
3091 F2FS_I(inode)->i_advise &= ~type;
3092 f2fs_mark_inode_dirty_sync(inode, true);
3093}
3094
3095static inline bool f2fs_is_time_consistent(struct inode *inode)
3096{
3097 if (!timespec64_equal(F2FS_I(inode)->i_disk_time, &inode->i_atime))
3098 return false;
3099 if (!timespec64_equal(F2FS_I(inode)->i_disk_time + 1, &inode->i_ctime))
3100 return false;
3101 if (!timespec64_equal(F2FS_I(inode)->i_disk_time + 2, &inode->i_mtime))
3102 return false;
3103 if (!timespec64_equal(F2FS_I(inode)->i_disk_time + 3,
3104 &F2FS_I(inode)->i_crtime))
3105 return false;
3106 return true;
3107}
3108
3109static inline bool f2fs_skip_inode_update(struct inode *inode, int dsync)
3110{
3111 bool ret;
3112
3113 if (dsync) {
3114 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3115
3116 spin_lock(&sbi->inode_lock[DIRTY_META]);
3117 ret = list_empty(&F2FS_I(inode)->gdirty_list);
3118 spin_unlock(&sbi->inode_lock[DIRTY_META]);
3119 return ret;
3120 }
3121 if (!is_inode_flag_set(inode, FI_AUTO_RECOVER) ||
3122 file_keep_isize(inode) ||
3123 i_size_read(inode) & ~PAGE_MASK)
3124 return false;
3125
3126 if (!f2fs_is_time_consistent(inode))
3127 return false;
3128
3129 spin_lock(&F2FS_I(inode)->i_size_lock);
3130 ret = F2FS_I(inode)->last_disk_size == i_size_read(inode);
3131 spin_unlock(&F2FS_I(inode)->i_size_lock);
3132
3133 return ret;
3134}
3135
3136static inline bool f2fs_readonly(struct super_block *sb)
3137{
3138 return sb_rdonly(sb);
3139}
3140
3141static inline bool f2fs_cp_error(struct f2fs_sb_info *sbi)
3142{
3143 return is_set_ckpt_flags(sbi, CP_ERROR_FLAG);
3144}
3145
3146static inline bool is_dot_dotdot(const u8 *name, size_t len)
3147{
3148 if (len == 1 && name[0] == '.')
3149 return true;
3150
3151 if (len == 2 && name[0] == '.' && name[1] == '.')
3152 return true;
3153
3154 return false;
3155}
3156
3157static inline void *f2fs_kmalloc(struct f2fs_sb_info *sbi,
3158 size_t size, gfp_t flags)
3159{
3160 if (time_to_inject(sbi, FAULT_KMALLOC)) {
3161 f2fs_show_injection_info(sbi, FAULT_KMALLOC);
3162 return NULL;
3163 }
3164
3165 return kmalloc(size, flags);
3166}
3167
3168static inline void *f2fs_kzalloc(struct f2fs_sb_info *sbi,
3169 size_t size, gfp_t flags)
3170{
3171 return f2fs_kmalloc(sbi, size, flags | __GFP_ZERO);
3172}
3173
3174static inline void *f2fs_kvmalloc(struct f2fs_sb_info *sbi,
3175 size_t size, gfp_t flags)
3176{
3177 if (time_to_inject(sbi, FAULT_KVMALLOC)) {
3178 f2fs_show_injection_info(sbi, FAULT_KVMALLOC);
3179 return NULL;
3180 }
3181
3182 return kvmalloc(size, flags);
3183}
3184
3185static inline void *f2fs_kvzalloc(struct f2fs_sb_info *sbi,
3186 size_t size, gfp_t flags)
3187{
3188 return f2fs_kvmalloc(sbi, size, flags | __GFP_ZERO);
3189}
3190
3191static inline int get_extra_isize(struct inode *inode)
3192{
3193 return F2FS_I(inode)->i_extra_isize / sizeof(__le32);
3194}
3195
3196static inline int get_inline_xattr_addrs(struct inode *inode)
3197{
3198 return F2FS_I(inode)->i_inline_xattr_size;
3199}
3200
3201#define f2fs_get_inode_mode(i) \
3202 ((is_inode_flag_set(i, FI_ACL_MODE)) ? \
3203 (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))
3204
3205#define F2FS_TOTAL_EXTRA_ATTR_SIZE \
3206 (offsetof(struct f2fs_inode, i_extra_end) - \
3207 offsetof(struct f2fs_inode, i_extra_isize)) \
3208
3209#define F2FS_OLD_ATTRIBUTE_SIZE (offsetof(struct f2fs_inode, i_addr))
3210#define F2FS_FITS_IN_INODE(f2fs_inode, extra_isize, field) \
3211 ((offsetof(typeof(*(f2fs_inode)), field) + \
3212 sizeof((f2fs_inode)->field)) \
3213 <= (F2FS_OLD_ATTRIBUTE_SIZE + (extra_isize))) \
3214
3215#define DEFAULT_IOSTAT_PERIOD_MS 3000
3216#define MIN_IOSTAT_PERIOD_MS 100
3217/* maximum period of iostat tracing is 1 day */
3218#define MAX_IOSTAT_PERIOD_MS 8640000
3219
3220static inline void f2fs_reset_iostat(struct f2fs_sb_info *sbi)
3221{
3222 int i;
3223
3224 spin_lock(&sbi->iostat_lock);
3225 for (i = 0; i < NR_IO_TYPE; i++) {
3226 sbi->rw_iostat[i] = 0;
3227 sbi->prev_rw_iostat[i] = 0;
3228 }
3229 spin_unlock(&sbi->iostat_lock);
3230}
3231
3232extern void f2fs_record_iostat(struct f2fs_sb_info *sbi);
3233
3234static inline void f2fs_update_iostat(struct f2fs_sb_info *sbi,
3235 enum iostat_type type, unsigned long long io_bytes)
3236{
3237 if (!sbi->iostat_enable)
3238 return;
3239 spin_lock(&sbi->iostat_lock);
3240 sbi->rw_iostat[type] += io_bytes;
3241
3242 if (type == APP_WRITE_IO || type == APP_DIRECT_IO)
3243 sbi->rw_iostat[APP_BUFFERED_IO] =
3244 sbi->rw_iostat[APP_WRITE_IO] -
3245 sbi->rw_iostat[APP_DIRECT_IO];
3246
3247 if (type == APP_READ_IO || type == APP_DIRECT_READ_IO)
3248 sbi->rw_iostat[APP_BUFFERED_READ_IO] =
3249 sbi->rw_iostat[APP_READ_IO] -
3250 sbi->rw_iostat[APP_DIRECT_READ_IO];
3251 spin_unlock(&sbi->iostat_lock);
3252
3253 f2fs_record_iostat(sbi);
3254}
3255
3256#define __is_large_section(sbi) ((sbi)->segs_per_sec > 1)
3257
3258#define __is_meta_io(fio) (PAGE_TYPE_OF_BIO((fio)->type) == META)
3259
3260bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
3261 block_t blkaddr, int type);
3262static inline void verify_blkaddr(struct f2fs_sb_info *sbi,
3263 block_t blkaddr, int type)
3264{
3265 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, type)) {
3266 f2fs_err(sbi, "invalid blkaddr: %u, type: %d, run fsck to fix.",
3267 blkaddr, type);
3268 f2fs_bug_on(sbi, 1);
3269 }
3270}
3271
3272static inline bool __is_valid_data_blkaddr(block_t blkaddr)
3273{
3274 if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR ||
3275 blkaddr == COMPRESS_ADDR)
3276 return false;
3277 return true;
3278}
3279
3280/*
3281 * file.c
3282 */
3283int f2fs_sync_file(struct file *file, loff_t start, loff_t end, int datasync);
3284void f2fs_truncate_data_blocks(struct dnode_of_data *dn);
3285int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock);
3286int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock);
3287int f2fs_truncate(struct inode *inode);
3288int f2fs_getattr(struct user_namespace *mnt_userns, const struct path *path,
3289 struct kstat *stat, u32 request_mask, unsigned int flags);
3290int f2fs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry,
3291 struct iattr *attr);
3292int f2fs_truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end);
3293void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count);
3294int f2fs_precache_extents(struct inode *inode);
3295int f2fs_fileattr_get(struct dentry *dentry, struct fileattr *fa);
3296int f2fs_fileattr_set(struct user_namespace *mnt_userns,
3297 struct dentry *dentry, struct fileattr *fa);
3298long f2fs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
3299long f2fs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
3300int f2fs_transfer_project_quota(struct inode *inode, kprojid_t kprojid);
3301int f2fs_pin_file_control(struct inode *inode, bool inc);
3302
3303/*
3304 * inode.c
3305 */
3306void f2fs_set_inode_flags(struct inode *inode);
3307bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct page *page);
3308void f2fs_inode_chksum_set(struct f2fs_sb_info *sbi, struct page *page);
3309struct inode *f2fs_iget(struct super_block *sb, unsigned long ino);
3310struct inode *f2fs_iget_retry(struct super_block *sb, unsigned long ino);
3311int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink);
3312void f2fs_update_inode(struct inode *inode, struct page *node_page);
3313void f2fs_update_inode_page(struct inode *inode);
3314int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc);
3315void f2fs_evict_inode(struct inode *inode);
3316void f2fs_handle_failed_inode(struct inode *inode);
3317
3318/*
3319 * namei.c
3320 */
3321int f2fs_update_extension_list(struct f2fs_sb_info *sbi, const char *name,
3322 bool hot, bool set);
3323struct dentry *f2fs_get_parent(struct dentry *child);
3324
3325/*
3326 * dir.c
3327 */
3328unsigned char f2fs_get_de_type(struct f2fs_dir_entry *de);
3329int f2fs_init_casefolded_name(const struct inode *dir,
3330 struct f2fs_filename *fname);
3331int f2fs_setup_filename(struct inode *dir, const struct qstr *iname,
3332 int lookup, struct f2fs_filename *fname);
3333int f2fs_prepare_lookup(struct inode *dir, struct dentry *dentry,
3334 struct f2fs_filename *fname);
3335void f2fs_free_filename(struct f2fs_filename *fname);
3336struct f2fs_dir_entry *f2fs_find_target_dentry(const struct f2fs_dentry_ptr *d,
3337 const struct f2fs_filename *fname, int *max_slots);
3338int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
3339 unsigned int start_pos, struct fscrypt_str *fstr);
3340void f2fs_do_make_empty_dir(struct inode *inode, struct inode *parent,
3341 struct f2fs_dentry_ptr *d);
3342struct page *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir,
3343 const struct f2fs_filename *fname, struct page *dpage);
3344void f2fs_update_parent_metadata(struct inode *dir, struct inode *inode,
3345 unsigned int current_depth);
3346int f2fs_room_for_filename(const void *bitmap, int slots, int max_slots);
3347void f2fs_drop_nlink(struct inode *dir, struct inode *inode);
3348struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir,
3349 const struct f2fs_filename *fname,
3350 struct page **res_page);
3351struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir,
3352 const struct qstr *child, struct page **res_page);
3353struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, struct page **p);
3354ino_t f2fs_inode_by_name(struct inode *dir, const struct qstr *qstr,
3355 struct page **page);
3356void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de,
3357 struct page *page, struct inode *inode);
3358bool f2fs_has_enough_room(struct inode *dir, struct page *ipage,
3359 const struct f2fs_filename *fname);
3360void f2fs_update_dentry(nid_t ino, umode_t mode, struct f2fs_dentry_ptr *d,
3361 const struct fscrypt_str *name, f2fs_hash_t name_hash,
3362 unsigned int bit_pos);
3363int f2fs_add_regular_entry(struct inode *dir, const struct f2fs_filename *fname,
3364 struct inode *inode, nid_t ino, umode_t mode);
3365int f2fs_add_dentry(struct inode *dir, const struct f2fs_filename *fname,
3366 struct inode *inode, nid_t ino, umode_t mode);
3367int f2fs_do_add_link(struct inode *dir, const struct qstr *name,
3368 struct inode *inode, nid_t ino, umode_t mode);
3369void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page,
3370 struct inode *dir, struct inode *inode);
3371int f2fs_do_tmpfile(struct inode *inode, struct inode *dir);
3372bool f2fs_empty_dir(struct inode *dir);
3373
3374static inline int f2fs_add_link(struct dentry *dentry, struct inode *inode)
3375{
3376 if (fscrypt_is_nokey_name(dentry))
3377 return -ENOKEY;
3378 return f2fs_do_add_link(d_inode(dentry->d_parent), &dentry->d_name,
3379 inode, inode->i_ino, inode->i_mode);
3380}
3381
3382/*
3383 * super.c
3384 */
3385int f2fs_inode_dirtied(struct inode *inode, bool sync);
3386void f2fs_inode_synced(struct inode *inode);
3387int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly);
3388int f2fs_quota_sync(struct super_block *sb, int type);
3389loff_t max_file_blocks(struct inode *inode);
3390void f2fs_quota_off_umount(struct super_block *sb);
3391int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover);
3392int f2fs_sync_fs(struct super_block *sb, int sync);
3393int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi);
3394
3395/*
3396 * hash.c
3397 */
3398void f2fs_hash_filename(const struct inode *dir, struct f2fs_filename *fname);
3399
3400/*
3401 * node.c
3402 */
3403struct node_info;
3404
3405int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid);
3406bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type);
3407bool f2fs_in_warm_node_list(struct f2fs_sb_info *sbi, struct page *page);
3408void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi);
3409void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct page *page);
3410void f2fs_reset_fsync_node_info(struct f2fs_sb_info *sbi);
3411int f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid);
3412bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid);
3413bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino);
3414int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid,
3415 struct node_info *ni);
3416pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs);
3417int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode);
3418int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from);
3419int f2fs_truncate_xattr_node(struct inode *inode);
3420int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi,
3421 unsigned int seq_id);
3422int f2fs_remove_inode_page(struct inode *inode);
3423struct page *f2fs_new_inode_page(struct inode *inode);
3424struct page *f2fs_new_node_page(struct dnode_of_data *dn, unsigned int ofs);
3425void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid);
3426struct page *f2fs_get_node_page(struct f2fs_sb_info *sbi, pgoff_t nid);
3427struct page *f2fs_get_node_page_ra(struct page *parent, int start);
3428int f2fs_move_node_page(struct page *node_page, int gc_type);
3429void f2fs_flush_inline_data(struct f2fs_sb_info *sbi);
3430int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode,
3431 struct writeback_control *wbc, bool atomic,
3432 unsigned int *seq_id);
3433int f2fs_sync_node_pages(struct f2fs_sb_info *sbi,
3434 struct writeback_control *wbc,
3435 bool do_balance, enum iostat_type io_type);
3436int f2fs_build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount);
3437bool f2fs_alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid);
3438void f2fs_alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid);
3439void f2fs_alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid);
3440int f2fs_try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink);
3441int f2fs_recover_inline_xattr(struct inode *inode, struct page *page);
3442int f2fs_recover_xattr_data(struct inode *inode, struct page *page);
3443int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct page *page);
3444int f2fs_restore_node_summary(struct f2fs_sb_info *sbi,
3445 unsigned int segno, struct f2fs_summary_block *sum);
3446int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc);
3447int f2fs_build_node_manager(struct f2fs_sb_info *sbi);
3448void f2fs_destroy_node_manager(struct f2fs_sb_info *sbi);
3449int __init f2fs_create_node_manager_caches(void);
3450void f2fs_destroy_node_manager_caches(void);
3451
3452/*
3453 * segment.c
3454 */
3455bool f2fs_need_SSR(struct f2fs_sb_info *sbi);
3456void f2fs_register_inmem_page(struct inode *inode, struct page *page);
3457void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure);
3458void f2fs_drop_inmem_pages(struct inode *inode);
3459void f2fs_drop_inmem_page(struct inode *inode, struct page *page);
3460int f2fs_commit_inmem_pages(struct inode *inode);
3461void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need);
3462void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg);
3463int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino);
3464int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi);
3465int f2fs_flush_device_cache(struct f2fs_sb_info *sbi);
3466void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free);
3467void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr);
3468bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr);
3469void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi);
3470void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi);
3471bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi);
3472void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
3473 struct cp_control *cpc);
3474void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi);
3475block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi);
3476int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable);
3477void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi);
3478int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra);
3479bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno);
3480void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi);
3481void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi);
3482void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi);
3483void f2fs_get_new_segment(struct f2fs_sb_info *sbi,
3484 unsigned int *newseg, bool new_sec, int dir);
3485void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
3486 unsigned int start, unsigned int end);
3487void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force);
3488void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi);
3489int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range);
3490bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
3491 struct cp_control *cpc);
3492struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno);
3493void f2fs_update_meta_page(struct f2fs_sb_info *sbi, void *src,
3494 block_t blk_addr);
3495void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
3496 enum iostat_type io_type);
3497void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio);
3498void f2fs_outplace_write_data(struct dnode_of_data *dn,
3499 struct f2fs_io_info *fio);
3500int f2fs_inplace_write_data(struct f2fs_io_info *fio);
3501void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
3502 block_t old_blkaddr, block_t new_blkaddr,
3503 bool recover_curseg, bool recover_newaddr,
3504 bool from_gc);
3505void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
3506 block_t old_addr, block_t new_addr,
3507 unsigned char version, bool recover_curseg,
3508 bool recover_newaddr);
3509void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
3510 block_t old_blkaddr, block_t *new_blkaddr,
3511 struct f2fs_summary *sum, int type,
3512 struct f2fs_io_info *fio);
3513void f2fs_wait_on_page_writeback(struct page *page,
3514 enum page_type type, bool ordered, bool locked);
3515void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr);
3516void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
3517 block_t len);
3518void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk);
3519void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk);
3520int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
3521 unsigned int val, int alloc);
3522void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc);
3523int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi);
3524int f2fs_check_write_pointer(struct f2fs_sb_info *sbi);
3525int f2fs_build_segment_manager(struct f2fs_sb_info *sbi);
3526void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi);
3527int __init f2fs_create_segment_manager_caches(void);
3528void f2fs_destroy_segment_manager_caches(void);
3529int f2fs_rw_hint_to_seg_type(enum rw_hint hint);
3530enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
3531 enum page_type type, enum temp_type temp);
3532unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
3533 unsigned int segno);
3534unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
3535 unsigned int segno);
3536
3537/*
3538 * checkpoint.c
3539 */
3540void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io);
3541struct page *f2fs_grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index);
3542struct page *f2fs_get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index);
3543struct page *f2fs_get_meta_page_retry(struct f2fs_sb_info *sbi, pgoff_t index);
3544struct page *f2fs_get_tmp_page(struct f2fs_sb_info *sbi, pgoff_t index);
3545bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
3546 block_t blkaddr, int type);
3547int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
3548 int type, bool sync);
3549void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index);
3550long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type,
3551 long nr_to_write, enum iostat_type io_type);
3552void f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type);
3553void f2fs_remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type);
3554void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all);
3555bool f2fs_exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode);
3556void f2fs_set_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
3557 unsigned int devidx, int type);
3558bool f2fs_is_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
3559 unsigned int devidx, int type);
3560int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi);
3561int f2fs_acquire_orphan_inode(struct f2fs_sb_info *sbi);
3562void f2fs_release_orphan_inode(struct f2fs_sb_info *sbi);
3563void f2fs_add_orphan_inode(struct inode *inode);
3564void f2fs_remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino);
3565int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi);
3566int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi);
3567void f2fs_update_dirty_page(struct inode *inode, struct page *page);
3568void f2fs_remove_dirty_inode(struct inode *inode);
3569int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type);
3570void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type);
3571u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi);
3572int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc);
3573void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi);
3574int __init f2fs_create_checkpoint_caches(void);
3575void f2fs_destroy_checkpoint_caches(void);
3576int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi);
3577int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi);
3578void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi);
3579void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi);
3580
3581/*
3582 * data.c
3583 */
3584int __init f2fs_init_bioset(void);
3585void f2fs_destroy_bioset(void);
3586int f2fs_init_bio_entry_cache(void);
3587void f2fs_destroy_bio_entry_cache(void);
3588void f2fs_submit_bio(struct f2fs_sb_info *sbi,
3589 struct bio *bio, enum page_type type);
3590void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type);
3591void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
3592 struct inode *inode, struct page *page,
3593 nid_t ino, enum page_type type);
3594void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
3595 struct bio **bio, struct page *page);
3596void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi);
3597int f2fs_submit_page_bio(struct f2fs_io_info *fio);
3598int f2fs_merge_page_bio(struct f2fs_io_info *fio);
3599void f2fs_submit_page_write(struct f2fs_io_info *fio);
3600struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
3601 block_t blk_addr, struct bio *bio);
3602int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr);
3603void f2fs_set_data_blkaddr(struct dnode_of_data *dn);
3604void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr);
3605int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count);
3606int f2fs_reserve_new_block(struct dnode_of_data *dn);
3607int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index);
3608int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from);
3609int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index);
3610struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
3611 int op_flags, bool for_write);
3612struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index);
3613struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index,
3614 bool for_write);
3615struct page *f2fs_get_new_data_page(struct inode *inode,
3616 struct page *ipage, pgoff_t index, bool new_i_size);
3617int f2fs_do_write_data_page(struct f2fs_io_info *fio);
3618void f2fs_do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock);
3619int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
3620 int create, int flag);
3621int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
3622 u64 start, u64 len);
3623int f2fs_encrypt_one_page(struct f2fs_io_info *fio);
3624bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio);
3625bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio);
3626int f2fs_write_single_data_page(struct page *page, int *submitted,
3627 struct bio **bio, sector_t *last_block,
3628 struct writeback_control *wbc,
3629 enum iostat_type io_type,
3630 int compr_blocks, bool allow_balance);
3631void f2fs_invalidate_page(struct page *page, unsigned int offset,
3632 unsigned int length);
3633int f2fs_release_page(struct page *page, gfp_t wait);
3634#ifdef CONFIG_MIGRATION
3635int f2fs_migrate_page(struct address_space *mapping, struct page *newpage,
3636 struct page *page, enum migrate_mode mode);
3637#endif
3638bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len);
3639void f2fs_clear_page_cache_dirty_tag(struct page *page);
3640int f2fs_init_post_read_processing(void);
3641void f2fs_destroy_post_read_processing(void);
3642int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi);
3643void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi);
3644
3645/*
3646 * gc.c
3647 */
3648int f2fs_start_gc_thread(struct f2fs_sb_info *sbi);
3649void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi);
3650block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode);
3651int f2fs_gc(struct f2fs_sb_info *sbi, bool sync, bool background, bool force,
3652 unsigned int segno);
3653void f2fs_build_gc_manager(struct f2fs_sb_info *sbi);
3654int f2fs_resize_fs(struct f2fs_sb_info *sbi, __u64 block_count);
3655int __init f2fs_create_garbage_collection_cache(void);
3656void f2fs_destroy_garbage_collection_cache(void);
3657
3658/*
3659 * recovery.c
3660 */
3661int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only);
3662bool f2fs_space_for_roll_forward(struct f2fs_sb_info *sbi);
3663int __init f2fs_create_recovery_cache(void);
3664void f2fs_destroy_recovery_cache(void);
3665
3666/*
3667 * debug.c
3668 */
3669#ifdef CONFIG_F2FS_STAT_FS
3670struct f2fs_stat_info {
3671 struct list_head stat_list;
3672 struct f2fs_sb_info *sbi;
3673 int all_area_segs, sit_area_segs, nat_area_segs, ssa_area_segs;
3674 int main_area_segs, main_area_sections, main_area_zones;
3675 unsigned long long hit_largest, hit_cached, hit_rbtree;
3676 unsigned long long hit_total, total_ext;
3677 int ext_tree, zombie_tree, ext_node;
3678 int ndirty_node, ndirty_dent, ndirty_meta, ndirty_imeta;
3679 int ndirty_data, ndirty_qdata;
3680 int inmem_pages;
3681 unsigned int ndirty_dirs, ndirty_files, nquota_files, ndirty_all;
3682 int nats, dirty_nats, sits, dirty_sits;
3683 int free_nids, avail_nids, alloc_nids;
3684 int total_count, utilization;
3685 int bg_gc, nr_wb_cp_data, nr_wb_data;
3686 int nr_rd_data, nr_rd_node, nr_rd_meta;
3687 int nr_dio_read, nr_dio_write;
3688 unsigned int io_skip_bggc, other_skip_bggc;
3689 int nr_flushing, nr_flushed, flush_list_empty;
3690 int nr_discarding, nr_discarded;
3691 int nr_discard_cmd;
3692 unsigned int undiscard_blks;
3693 int nr_issued_ckpt, nr_total_ckpt, nr_queued_ckpt;
3694 unsigned int cur_ckpt_time, peak_ckpt_time;
3695 int inline_xattr, inline_inode, inline_dir, append, update, orphans;
3696 int compr_inode;
3697 unsigned long long compr_blocks;
3698 int aw_cnt, max_aw_cnt, vw_cnt, max_vw_cnt;
3699 unsigned int valid_count, valid_node_count, valid_inode_count, discard_blks;
3700 unsigned int bimodal, avg_vblocks;
3701 int util_free, util_valid, util_invalid;
3702 int rsvd_segs, overp_segs;
3703 int dirty_count, node_pages, meta_pages, compress_pages;
3704 int compress_page_hit;
3705 int prefree_count, call_count, cp_count, bg_cp_count;
3706 int tot_segs, node_segs, data_segs, free_segs, free_secs;
3707 int bg_node_segs, bg_data_segs;
3708 int tot_blks, data_blks, node_blks;
3709 int bg_data_blks, bg_node_blks;
3710 unsigned long long skipped_atomic_files[2];
3711 int curseg[NR_CURSEG_TYPE];
3712 int cursec[NR_CURSEG_TYPE];
3713 int curzone[NR_CURSEG_TYPE];
3714 unsigned int dirty_seg[NR_CURSEG_TYPE];
3715 unsigned int full_seg[NR_CURSEG_TYPE];
3716 unsigned int valid_blks[NR_CURSEG_TYPE];
3717
3718 unsigned int meta_count[META_MAX];
3719 unsigned int segment_count[2];
3720 unsigned int block_count[2];
3721 unsigned int inplace_count;
3722 unsigned long long base_mem, cache_mem, page_mem;
3723};
3724
3725static inline struct f2fs_stat_info *F2FS_STAT(struct f2fs_sb_info *sbi)
3726{
3727 return (struct f2fs_stat_info *)sbi->stat_info;
3728}
3729
3730#define stat_inc_cp_count(si) ((si)->cp_count++)
3731#define stat_inc_bg_cp_count(si) ((si)->bg_cp_count++)
3732#define stat_inc_call_count(si) ((si)->call_count++)
3733#define stat_inc_bggc_count(si) ((si)->bg_gc++)
3734#define stat_io_skip_bggc_count(sbi) ((sbi)->io_skip_bggc++)
3735#define stat_other_skip_bggc_count(sbi) ((sbi)->other_skip_bggc++)
3736#define stat_inc_dirty_inode(sbi, type) ((sbi)->ndirty_inode[type]++)
3737#define stat_dec_dirty_inode(sbi, type) ((sbi)->ndirty_inode[type]--)
3738#define stat_inc_total_hit(sbi) (atomic64_inc(&(sbi)->total_hit_ext))
3739#define stat_inc_rbtree_node_hit(sbi) (atomic64_inc(&(sbi)->read_hit_rbtree))
3740#define stat_inc_largest_node_hit(sbi) (atomic64_inc(&(sbi)->read_hit_largest))
3741#define stat_inc_cached_node_hit(sbi) (atomic64_inc(&(sbi)->read_hit_cached))
3742#define stat_inc_inline_xattr(inode) \
3743 do { \
3744 if (f2fs_has_inline_xattr(inode)) \
3745 (atomic_inc(&F2FS_I_SB(inode)->inline_xattr)); \
3746 } while (0)
3747#define stat_dec_inline_xattr(inode) \
3748 do { \
3749 if (f2fs_has_inline_xattr(inode)) \
3750 (atomic_dec(&F2FS_I_SB(inode)->inline_xattr)); \
3751 } while (0)
3752#define stat_inc_inline_inode(inode) \
3753 do { \
3754 if (f2fs_has_inline_data(inode)) \
3755 (atomic_inc(&F2FS_I_SB(inode)->inline_inode)); \
3756 } while (0)
3757#define stat_dec_inline_inode(inode) \
3758 do { \
3759 if (f2fs_has_inline_data(inode)) \
3760 (atomic_dec(&F2FS_I_SB(inode)->inline_inode)); \
3761 } while (0)
3762#define stat_inc_inline_dir(inode) \
3763 do { \
3764 if (f2fs_has_inline_dentry(inode)) \
3765 (atomic_inc(&F2FS_I_SB(inode)->inline_dir)); \
3766 } while (0)
3767#define stat_dec_inline_dir(inode) \
3768 do { \
3769 if (f2fs_has_inline_dentry(inode)) \
3770 (atomic_dec(&F2FS_I_SB(inode)->inline_dir)); \
3771 } while (0)
3772#define stat_inc_compr_inode(inode) \
3773 do { \
3774 if (f2fs_compressed_file(inode)) \
3775 (atomic_inc(&F2FS_I_SB(inode)->compr_inode)); \
3776 } while (0)
3777#define stat_dec_compr_inode(inode) \
3778 do { \
3779 if (f2fs_compressed_file(inode)) \
3780 (atomic_dec(&F2FS_I_SB(inode)->compr_inode)); \
3781 } while (0)
3782#define stat_add_compr_blocks(inode, blocks) \
3783 (atomic64_add(blocks, &F2FS_I_SB(inode)->compr_blocks))
3784#define stat_sub_compr_blocks(inode, blocks) \
3785 (atomic64_sub(blocks, &F2FS_I_SB(inode)->compr_blocks))
3786#define stat_inc_meta_count(sbi, blkaddr) \
3787 do { \
3788 if (blkaddr < SIT_I(sbi)->sit_base_addr) \
3789 atomic_inc(&(sbi)->meta_count[META_CP]); \
3790 else if (blkaddr < NM_I(sbi)->nat_blkaddr) \
3791 atomic_inc(&(sbi)->meta_count[META_SIT]); \
3792 else if (blkaddr < SM_I(sbi)->ssa_blkaddr) \
3793 atomic_inc(&(sbi)->meta_count[META_NAT]); \
3794 else if (blkaddr < SM_I(sbi)->main_blkaddr) \
3795 atomic_inc(&(sbi)->meta_count[META_SSA]); \
3796 } while (0)
3797#define stat_inc_seg_type(sbi, curseg) \
3798 ((sbi)->segment_count[(curseg)->alloc_type]++)
3799#define stat_inc_block_count(sbi, curseg) \
3800 ((sbi)->block_count[(curseg)->alloc_type]++)
3801#define stat_inc_inplace_blocks(sbi) \
3802 (atomic_inc(&(sbi)->inplace_count))
3803#define stat_update_max_atomic_write(inode) \
3804 do { \
3805 int cur = F2FS_I_SB(inode)->atomic_files; \
3806 int max = atomic_read(&F2FS_I_SB(inode)->max_aw_cnt); \
3807 if (cur > max) \
3808 atomic_set(&F2FS_I_SB(inode)->max_aw_cnt, cur); \
3809 } while (0)
3810#define stat_inc_volatile_write(inode) \
3811 (atomic_inc(&F2FS_I_SB(inode)->vw_cnt))
3812#define stat_dec_volatile_write(inode) \
3813 (atomic_dec(&F2FS_I_SB(inode)->vw_cnt))
3814#define stat_update_max_volatile_write(inode) \
3815 do { \
3816 int cur = atomic_read(&F2FS_I_SB(inode)->vw_cnt); \
3817 int max = atomic_read(&F2FS_I_SB(inode)->max_vw_cnt); \
3818 if (cur > max) \
3819 atomic_set(&F2FS_I_SB(inode)->max_vw_cnt, cur); \
3820 } while (0)
3821#define stat_inc_seg_count(sbi, type, gc_type) \
3822 do { \
3823 struct f2fs_stat_info *si = F2FS_STAT(sbi); \
3824 si->tot_segs++; \
3825 if ((type) == SUM_TYPE_DATA) { \
3826 si->data_segs++; \
3827 si->bg_data_segs += (gc_type == BG_GC) ? 1 : 0; \
3828 } else { \
3829 si->node_segs++; \
3830 si->bg_node_segs += (gc_type == BG_GC) ? 1 : 0; \
3831 } \
3832 } while (0)
3833
3834#define stat_inc_tot_blk_count(si, blks) \
3835 ((si)->tot_blks += (blks))
3836
3837#define stat_inc_data_blk_count(sbi, blks, gc_type) \
3838 do { \
3839 struct f2fs_stat_info *si = F2FS_STAT(sbi); \
3840 stat_inc_tot_blk_count(si, blks); \
3841 si->data_blks += (blks); \
3842 si->bg_data_blks += ((gc_type) == BG_GC) ? (blks) : 0; \
3843 } while (0)
3844
3845#define stat_inc_node_blk_count(sbi, blks, gc_type) \
3846 do { \
3847 struct f2fs_stat_info *si = F2FS_STAT(sbi); \
3848 stat_inc_tot_blk_count(si, blks); \
3849 si->node_blks += (blks); \
3850 si->bg_node_blks += ((gc_type) == BG_GC) ? (blks) : 0; \
3851 } while (0)
3852
3853int f2fs_build_stats(struct f2fs_sb_info *sbi);
3854void f2fs_destroy_stats(struct f2fs_sb_info *sbi);
3855void __init f2fs_create_root_stats(void);
3856void f2fs_destroy_root_stats(void);
3857void f2fs_update_sit_info(struct f2fs_sb_info *sbi);
3858#else
3859#define stat_inc_cp_count(si) do { } while (0)
3860#define stat_inc_bg_cp_count(si) do { } while (0)
3861#define stat_inc_call_count(si) do { } while (0)
3862#define stat_inc_bggc_count(si) do { } while (0)
3863#define stat_io_skip_bggc_count(sbi) do { } while (0)
3864#define stat_other_skip_bggc_count(sbi) do { } while (0)
3865#define stat_inc_dirty_inode(sbi, type) do { } while (0)
3866#define stat_dec_dirty_inode(sbi, type) do { } while (0)
3867#define stat_inc_total_hit(sbi) do { } while (0)
3868#define stat_inc_rbtree_node_hit(sbi) do { } while (0)
3869#define stat_inc_largest_node_hit(sbi) do { } while (0)
3870#define stat_inc_cached_node_hit(sbi) do { } while (0)
3871#define stat_inc_inline_xattr(inode) do { } while (0)
3872#define stat_dec_inline_xattr(inode) do { } while (0)
3873#define stat_inc_inline_inode(inode) do { } while (0)
3874#define stat_dec_inline_inode(inode) do { } while (0)
3875#define stat_inc_inline_dir(inode) do { } while (0)
3876#define stat_dec_inline_dir(inode) do { } while (0)
3877#define stat_inc_compr_inode(inode) do { } while (0)
3878#define stat_dec_compr_inode(inode) do { } while (0)
3879#define stat_add_compr_blocks(inode, blocks) do { } while (0)
3880#define stat_sub_compr_blocks(inode, blocks) do { } while (0)
3881#define stat_update_max_atomic_write(inode) do { } while (0)
3882#define stat_inc_volatile_write(inode) do { } while (0)
3883#define stat_dec_volatile_write(inode) do { } while (0)
3884#define stat_update_max_volatile_write(inode) do { } while (0)
3885#define stat_inc_meta_count(sbi, blkaddr) do { } while (0)
3886#define stat_inc_seg_type(sbi, curseg) do { } while (0)
3887#define stat_inc_block_count(sbi, curseg) do { } while (0)
3888#define stat_inc_inplace_blocks(sbi) do { } while (0)
3889#define stat_inc_seg_count(sbi, type, gc_type) do { } while (0)
3890#define stat_inc_tot_blk_count(si, blks) do { } while (0)
3891#define stat_inc_data_blk_count(sbi, blks, gc_type) do { } while (0)
3892#define stat_inc_node_blk_count(sbi, blks, gc_type) do { } while (0)
3893
3894static inline int f2fs_build_stats(struct f2fs_sb_info *sbi) { return 0; }
3895static inline void f2fs_destroy_stats(struct f2fs_sb_info *sbi) { }
3896static inline void __init f2fs_create_root_stats(void) { }
3897static inline void f2fs_destroy_root_stats(void) { }
3898static inline void f2fs_update_sit_info(struct f2fs_sb_info *sbi) {}
3899#endif
3900
3901extern const struct file_operations f2fs_dir_operations;
3902extern const struct file_operations f2fs_file_operations;
3903extern const struct inode_operations f2fs_file_inode_operations;
3904extern const struct address_space_operations f2fs_dblock_aops;
3905extern const struct address_space_operations f2fs_node_aops;
3906extern const struct address_space_operations f2fs_meta_aops;
3907extern const struct inode_operations f2fs_dir_inode_operations;
3908extern const struct inode_operations f2fs_symlink_inode_operations;
3909extern const struct inode_operations f2fs_encrypted_symlink_inode_operations;
3910extern const struct inode_operations f2fs_special_inode_operations;
3911extern struct kmem_cache *f2fs_inode_entry_slab;
3912
3913/*
3914 * inline.c
3915 */
3916bool f2fs_may_inline_data(struct inode *inode);
3917bool f2fs_may_inline_dentry(struct inode *inode);
3918void f2fs_do_read_inline_data(struct page *page, struct page *ipage);
3919void f2fs_truncate_inline_inode(struct inode *inode,
3920 struct page *ipage, u64 from);
3921int f2fs_read_inline_data(struct inode *inode, struct page *page);
3922int f2fs_convert_inline_page(struct dnode_of_data *dn, struct page *page);
3923int f2fs_convert_inline_inode(struct inode *inode);
3924int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry);
3925int f2fs_write_inline_data(struct inode *inode, struct page *page);
3926int f2fs_recover_inline_data(struct inode *inode, struct page *npage);
3927struct f2fs_dir_entry *f2fs_find_in_inline_dir(struct inode *dir,
3928 const struct f2fs_filename *fname,
3929 struct page **res_page);
3930int f2fs_make_empty_inline_dir(struct inode *inode, struct inode *parent,
3931 struct page *ipage);
3932int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname,
3933 struct inode *inode, nid_t ino, umode_t mode);
3934void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry,
3935 struct page *page, struct inode *dir,
3936 struct inode *inode);
3937bool f2fs_empty_inline_dir(struct inode *dir);
3938int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx,
3939 struct fscrypt_str *fstr);
3940int f2fs_inline_data_fiemap(struct inode *inode,
3941 struct fiemap_extent_info *fieinfo,
3942 __u64 start, __u64 len);
3943
3944/*
3945 * shrinker.c
3946 */
3947unsigned long f2fs_shrink_count(struct shrinker *shrink,
3948 struct shrink_control *sc);
3949unsigned long f2fs_shrink_scan(struct shrinker *shrink,
3950 struct shrink_control *sc);
3951void f2fs_join_shrinker(struct f2fs_sb_info *sbi);
3952void f2fs_leave_shrinker(struct f2fs_sb_info *sbi);
3953
3954/*
3955 * extent_cache.c
3956 */
3957struct rb_entry *f2fs_lookup_rb_tree(struct rb_root_cached *root,
3958 struct rb_entry *cached_re, unsigned int ofs);
3959struct rb_node **f2fs_lookup_rb_tree_ext(struct f2fs_sb_info *sbi,
3960 struct rb_root_cached *root,
3961 struct rb_node **parent,
3962 unsigned long long key, bool *left_most);
3963struct rb_node **f2fs_lookup_rb_tree_for_insert(struct f2fs_sb_info *sbi,
3964 struct rb_root_cached *root,
3965 struct rb_node **parent,
3966 unsigned int ofs, bool *leftmost);
3967struct rb_entry *f2fs_lookup_rb_tree_ret(struct rb_root_cached *root,
3968 struct rb_entry *cached_re, unsigned int ofs,
3969 struct rb_entry **prev_entry, struct rb_entry **next_entry,
3970 struct rb_node ***insert_p, struct rb_node **insert_parent,
3971 bool force, bool *leftmost);
3972bool f2fs_check_rb_tree_consistence(struct f2fs_sb_info *sbi,
3973 struct rb_root_cached *root, bool check_key);
3974unsigned int f2fs_shrink_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink);
3975void f2fs_init_extent_tree(struct inode *inode, struct page *ipage);
3976void f2fs_drop_extent_tree(struct inode *inode);
3977unsigned int f2fs_destroy_extent_node(struct inode *inode);
3978void f2fs_destroy_extent_tree(struct inode *inode);
3979bool f2fs_lookup_extent_cache(struct inode *inode, pgoff_t pgofs,
3980 struct extent_info *ei);
3981void f2fs_update_extent_cache(struct dnode_of_data *dn);
3982void f2fs_update_extent_cache_range(struct dnode_of_data *dn,
3983 pgoff_t fofs, block_t blkaddr, unsigned int len);
3984void f2fs_init_extent_cache_info(struct f2fs_sb_info *sbi);
3985int __init f2fs_create_extent_cache(void);
3986void f2fs_destroy_extent_cache(void);
3987
3988/*
3989 * sysfs.c
3990 */
3991int __init f2fs_init_sysfs(void);
3992void f2fs_exit_sysfs(void);
3993int f2fs_register_sysfs(struct f2fs_sb_info *sbi);
3994void f2fs_unregister_sysfs(struct f2fs_sb_info *sbi);
3995
3996/* verity.c */
3997extern const struct fsverity_operations f2fs_verityops;
3998
3999/*
4000 * crypto support
4001 */
4002static inline bool f2fs_encrypted_file(struct inode *inode)
4003{
4004 return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode);
4005}
4006
4007static inline void f2fs_set_encrypted_inode(struct inode *inode)
4008{
4009#ifdef CONFIG_FS_ENCRYPTION
4010 file_set_encrypt(inode);
4011 f2fs_set_inode_flags(inode);
4012#endif
4013}
4014
4015/*
4016 * Returns true if the reads of the inode's data need to undergo some
4017 * postprocessing step, like decryption or authenticity verification.
4018 */
4019static inline bool f2fs_post_read_required(struct inode *inode)
4020{
4021 return f2fs_encrypted_file(inode) || fsverity_active(inode) ||
4022 f2fs_compressed_file(inode);
4023}
4024
4025/*
4026 * compress.c
4027 */
4028#ifdef CONFIG_F2FS_FS_COMPRESSION
4029bool f2fs_is_compressed_page(struct page *page);
4030struct page *f2fs_compress_control_page(struct page *page);
4031int f2fs_prepare_compress_overwrite(struct inode *inode,
4032 struct page **pagep, pgoff_t index, void **fsdata);
4033bool f2fs_compress_write_end(struct inode *inode, void *fsdata,
4034 pgoff_t index, unsigned copied);
4035int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock);
4036void f2fs_compress_write_end_io(struct bio *bio, struct page *page);
4037bool f2fs_is_compress_backend_ready(struct inode *inode);
4038int f2fs_init_compress_mempool(void);
4039void f2fs_destroy_compress_mempool(void);
4040void f2fs_decompress_cluster(struct decompress_io_ctx *dic);
4041void f2fs_end_read_compressed_page(struct page *page, bool failed,
4042 block_t blkaddr);
4043bool f2fs_cluster_is_empty(struct compress_ctx *cc);
4044bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index);
4045void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page);
4046int f2fs_write_multi_pages(struct compress_ctx *cc,
4047 int *submitted,
4048 struct writeback_control *wbc,
4049 enum iostat_type io_type);
4050int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index);
4051int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
4052 unsigned nr_pages, sector_t *last_block_in_bio,
4053 bool is_readahead, bool for_write);
4054struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc);
4055void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed);
4056void f2fs_put_page_dic(struct page *page);
4057int f2fs_init_compress_ctx(struct compress_ctx *cc);
4058void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse);
4059void f2fs_init_compress_info(struct f2fs_sb_info *sbi);
4060int f2fs_init_compress_inode(struct f2fs_sb_info *sbi);
4061void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi);
4062int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi);
4063void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi);
4064int __init f2fs_init_compress_cache(void);
4065void f2fs_destroy_compress_cache(void);
4066struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi);
4067void f2fs_invalidate_compress_page(struct f2fs_sb_info *sbi, block_t blkaddr);
4068void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, struct page *page,
4069 nid_t ino, block_t blkaddr);
4070bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi, struct page *page,
4071 block_t blkaddr);
4072void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino);
4073#define inc_compr_inode_stat(inode) \
4074 do { \
4075 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); \
4076 sbi->compr_new_inode++; \
4077 } while (0)
4078#define add_compr_block_stat(inode, blocks) \
4079 do { \
4080 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); \
4081 int diff = F2FS_I(inode)->i_cluster_size - blocks; \
4082 sbi->compr_written_block += blocks; \
4083 sbi->compr_saved_block += diff; \
4084 } while (0)
4085#else
4086static inline bool f2fs_is_compressed_page(struct page *page) { return false; }
4087static inline bool f2fs_is_compress_backend_ready(struct inode *inode)
4088{
4089 if (!f2fs_compressed_file(inode))
4090 return true;
4091 /* not support compression */
4092 return false;
4093}
4094static inline struct page *f2fs_compress_control_page(struct page *page)
4095{
4096 WARN_ON_ONCE(1);
4097 return ERR_PTR(-EINVAL);
4098}
4099static inline int f2fs_init_compress_mempool(void) { return 0; }
4100static inline void f2fs_destroy_compress_mempool(void) { }
4101static inline void f2fs_decompress_cluster(struct decompress_io_ctx *dic) { }
4102static inline void f2fs_end_read_compressed_page(struct page *page,
4103 bool failed, block_t blkaddr)
4104{
4105 WARN_ON_ONCE(1);
4106}
4107static inline void f2fs_put_page_dic(struct page *page)
4108{
4109 WARN_ON_ONCE(1);
4110}
4111static inline int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) { return 0; }
4112static inline void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) { }
4113static inline int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) { return 0; }
4114static inline void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) { }
4115static inline int __init f2fs_init_compress_cache(void) { return 0; }
4116static inline void f2fs_destroy_compress_cache(void) { }
4117static inline void f2fs_invalidate_compress_page(struct f2fs_sb_info *sbi,
4118 block_t blkaddr) { }
4119static inline void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi,
4120 struct page *page, nid_t ino, block_t blkaddr) { }
4121static inline bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi,
4122 struct page *page, block_t blkaddr) { return false; }
4123static inline void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi,
4124 nid_t ino) { }
4125#define inc_compr_inode_stat(inode) do { } while (0)
4126#endif
4127
4128static inline void set_compress_context(struct inode *inode)
4129{
4130 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
4131
4132 F2FS_I(inode)->i_compress_algorithm =
4133 F2FS_OPTION(sbi).compress_algorithm;
4134 F2FS_I(inode)->i_log_cluster_size =
4135 F2FS_OPTION(sbi).compress_log_size;
4136 F2FS_I(inode)->i_compress_flag =
4137 F2FS_OPTION(sbi).compress_chksum ?
4138 1 << COMPRESS_CHKSUM : 0;
4139 F2FS_I(inode)->i_cluster_size =
4140 1 << F2FS_I(inode)->i_log_cluster_size;
4141 if ((F2FS_I(inode)->i_compress_algorithm == COMPRESS_LZ4 ||
4142 F2FS_I(inode)->i_compress_algorithm == COMPRESS_ZSTD) &&
4143 F2FS_OPTION(sbi).compress_level)
4144 F2FS_I(inode)->i_compress_flag |=
4145 F2FS_OPTION(sbi).compress_level <<
4146 COMPRESS_LEVEL_OFFSET;
4147 F2FS_I(inode)->i_flags |= F2FS_COMPR_FL;
4148 set_inode_flag(inode, FI_COMPRESSED_FILE);
4149 stat_inc_compr_inode(inode);
4150 inc_compr_inode_stat(inode);
4151 f2fs_mark_inode_dirty_sync(inode, true);
4152}
4153
4154static inline bool f2fs_disable_compressed_file(struct inode *inode)
4155{
4156 struct f2fs_inode_info *fi = F2FS_I(inode);
4157
4158 if (!f2fs_compressed_file(inode))
4159 return true;
4160 if (S_ISREG(inode->i_mode) &&
4161 (get_dirty_pages(inode) || atomic_read(&fi->i_compr_blocks)))
4162 return false;
4163
4164 fi->i_flags &= ~F2FS_COMPR_FL;
4165 stat_dec_compr_inode(inode);
4166 clear_inode_flag(inode, FI_COMPRESSED_FILE);
4167 f2fs_mark_inode_dirty_sync(inode, true);
4168 return true;
4169}
4170
4171#define F2FS_FEATURE_FUNCS(name, flagname) \
4172static inline int f2fs_sb_has_##name(struct f2fs_sb_info *sbi) \
4173{ \
4174 return F2FS_HAS_FEATURE(sbi, F2FS_FEATURE_##flagname); \
4175}
4176
4177F2FS_FEATURE_FUNCS(encrypt, ENCRYPT);
4178F2FS_FEATURE_FUNCS(blkzoned, BLKZONED);
4179F2FS_FEATURE_FUNCS(extra_attr, EXTRA_ATTR);
4180F2FS_FEATURE_FUNCS(project_quota, PRJQUOTA);
4181F2FS_FEATURE_FUNCS(inode_chksum, INODE_CHKSUM);
4182F2FS_FEATURE_FUNCS(flexible_inline_xattr, FLEXIBLE_INLINE_XATTR);
4183F2FS_FEATURE_FUNCS(quota_ino, QUOTA_INO);
4184F2FS_FEATURE_FUNCS(inode_crtime, INODE_CRTIME);
4185F2FS_FEATURE_FUNCS(lost_found, LOST_FOUND);
4186F2FS_FEATURE_FUNCS(verity, VERITY);
4187F2FS_FEATURE_FUNCS(sb_chksum, SB_CHKSUM);
4188F2FS_FEATURE_FUNCS(casefold, CASEFOLD);
4189F2FS_FEATURE_FUNCS(compression, COMPRESSION);
4190F2FS_FEATURE_FUNCS(readonly, RO);
4191
4192static inline bool f2fs_may_extent_tree(struct inode *inode)
4193{
4194 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
4195
4196 if (!test_opt(sbi, EXTENT_CACHE) ||
4197 is_inode_flag_set(inode, FI_NO_EXTENT) ||
4198 (is_inode_flag_set(inode, FI_COMPRESSED_FILE) &&
4199 !f2fs_sb_has_readonly(sbi)))
4200 return false;
4201
4202 /*
4203 * for recovered files during mount do not create extents
4204 * if shrinker is not registered.
4205 */
4206 if (list_empty(&sbi->s_list))
4207 return false;
4208
4209 return S_ISREG(inode->i_mode);
4210}
4211
4212#ifdef CONFIG_BLK_DEV_ZONED
4213static inline bool f2fs_blkz_is_seq(struct f2fs_sb_info *sbi, int devi,
4214 block_t blkaddr)
4215{
4216 unsigned int zno = blkaddr >> sbi->log_blocks_per_blkz;
4217
4218 return test_bit(zno, FDEV(devi).blkz_seq);
4219}
4220#endif
4221
4222static inline bool f2fs_hw_should_discard(struct f2fs_sb_info *sbi)
4223{
4224 return f2fs_sb_has_blkzoned(sbi);
4225}
4226
4227static inline bool f2fs_bdev_support_discard(struct block_device *bdev)
4228{
4229 return blk_queue_discard(bdev_get_queue(bdev)) ||
4230 bdev_is_zoned(bdev);
4231}
4232
4233static inline bool f2fs_hw_support_discard(struct f2fs_sb_info *sbi)
4234{
4235 int i;
4236
4237 if (!f2fs_is_multi_device(sbi))
4238 return f2fs_bdev_support_discard(sbi->sb->s_bdev);
4239
4240 for (i = 0; i < sbi->s_ndevs; i++)
4241 if (f2fs_bdev_support_discard(FDEV(i).bdev))
4242 return true;
4243 return false;
4244}
4245
4246static inline bool f2fs_realtime_discard_enable(struct f2fs_sb_info *sbi)
4247{
4248 return (test_opt(sbi, DISCARD) && f2fs_hw_support_discard(sbi)) ||
4249 f2fs_hw_should_discard(sbi);
4250}
4251
4252static inline bool f2fs_hw_is_readonly(struct f2fs_sb_info *sbi)
4253{
4254 int i;
4255
4256 if (!f2fs_is_multi_device(sbi))
4257 return bdev_read_only(sbi->sb->s_bdev);
4258
4259 for (i = 0; i < sbi->s_ndevs; i++)
4260 if (bdev_read_only(FDEV(i).bdev))
4261 return true;
4262 return false;
4263}
4264
4265static inline bool f2fs_lfs_mode(struct f2fs_sb_info *sbi)
4266{
4267 return F2FS_OPTION(sbi).fs_mode == FS_MODE_LFS;
4268}
4269
4270static inline bool f2fs_may_compress(struct inode *inode)
4271{
4272 if (IS_SWAPFILE(inode) || f2fs_is_pinned_file(inode) ||
4273 f2fs_is_atomic_file(inode) ||
4274 f2fs_is_volatile_file(inode))
4275 return false;
4276 return S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode);
4277}
4278
4279static inline void f2fs_i_compr_blocks_update(struct inode *inode,
4280 u64 blocks, bool add)
4281{
4282 int diff = F2FS_I(inode)->i_cluster_size - blocks;
4283 struct f2fs_inode_info *fi = F2FS_I(inode);
4284
4285 /* don't update i_compr_blocks if saved blocks were released */
4286 if (!add && !atomic_read(&fi->i_compr_blocks))
4287 return;
4288
4289 if (add) {
4290 atomic_add(diff, &fi->i_compr_blocks);
4291 stat_add_compr_blocks(inode, diff);
4292 } else {
4293 atomic_sub(diff, &fi->i_compr_blocks);
4294 stat_sub_compr_blocks(inode, diff);
4295 }
4296 f2fs_mark_inode_dirty_sync(inode, true);
4297}
4298
4299static inline int block_unaligned_IO(struct inode *inode,
4300 struct kiocb *iocb, struct iov_iter *iter)
4301{
4302 unsigned int i_blkbits = READ_ONCE(inode->i_blkbits);
4303 unsigned int blocksize_mask = (1 << i_blkbits) - 1;
4304 loff_t offset = iocb->ki_pos;
4305 unsigned long align = offset | iov_iter_alignment(iter);
4306
4307 return align & blocksize_mask;
4308}
4309
4310static inline int allow_outplace_dio(struct inode *inode,
4311 struct kiocb *iocb, struct iov_iter *iter)
4312{
4313 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
4314 int rw = iov_iter_rw(iter);
4315
4316 return (f2fs_lfs_mode(sbi) && (rw == WRITE) &&
4317 !block_unaligned_IO(inode, iocb, iter));
4318}
4319
4320static inline bool f2fs_force_buffered_io(struct inode *inode,
4321 struct kiocb *iocb, struct iov_iter *iter)
4322{
4323 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
4324 int rw = iov_iter_rw(iter);
4325
4326 if (f2fs_post_read_required(inode))
4327 return true;
4328 if (f2fs_is_multi_device(sbi))
4329 return true;
4330 /*
4331 * for blkzoned device, fallback direct IO to buffered IO, so
4332 * all IOs can be serialized by log-structured write.
4333 */
4334 if (f2fs_sb_has_blkzoned(sbi))
4335 return true;
4336 if (f2fs_lfs_mode(sbi) && (rw == WRITE)) {
4337 if (block_unaligned_IO(inode, iocb, iter))
4338 return true;
4339 if (F2FS_IO_ALIGNED(sbi))
4340 return true;
4341 }
4342 if (is_sbi_flag_set(F2FS_I_SB(inode), SBI_CP_DISABLED))
4343 return true;
4344
4345 return false;
4346}
4347
4348static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx)
4349{
4350 return fsverity_active(inode) &&
4351 idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
4352}
4353
4354#ifdef CONFIG_F2FS_FAULT_INJECTION
4355extern void f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned int rate,
4356 unsigned int type);
4357#else
4358#define f2fs_build_fault_attr(sbi, rate, type) do { } while (0)
4359#endif
4360
4361static inline bool is_journalled_quota(struct f2fs_sb_info *sbi)
4362{
4363#ifdef CONFIG_QUOTA
4364 if (f2fs_sb_has_quota_ino(sbi))
4365 return true;
4366 if (F2FS_OPTION(sbi).s_qf_names[USRQUOTA] ||
4367 F2FS_OPTION(sbi).s_qf_names[GRPQUOTA] ||
4368 F2FS_OPTION(sbi).s_qf_names[PRJQUOTA])
4369 return true;
4370#endif
4371 return false;
4372}
4373
4374#define EFSBADCRC EBADMSG /* Bad CRC detected */
4375#define EFSCORRUPTED EUCLEAN /* Filesystem is corrupted */
4376
4377#endif /* _LINUX_F2FS_H */
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * fs/f2fs/f2fs.h
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8#ifndef _LINUX_F2FS_H
9#define _LINUX_F2FS_H
10
11#include <linux/uio.h>
12#include <linux/types.h>
13#include <linux/page-flags.h>
14#include <linux/buffer_head.h>
15#include <linux/slab.h>
16#include <linux/crc32.h>
17#include <linux/magic.h>
18#include <linux/kobject.h>
19#include <linux/sched.h>
20#include <linux/cred.h>
21#include <linux/sched/mm.h>
22#include <linux/vmalloc.h>
23#include <linux/bio.h>
24#include <linux/blkdev.h>
25#include <linux/quotaops.h>
26#include <linux/part_stat.h>
27#include <crypto/hash.h>
28
29#include <linux/fscrypt.h>
30#include <linux/fsverity.h>
31
32struct pagevec;
33
34#ifdef CONFIG_F2FS_CHECK_FS
35#define f2fs_bug_on(sbi, condition) BUG_ON(condition)
36#else
37#define f2fs_bug_on(sbi, condition) \
38 do { \
39 if (WARN_ON(condition)) \
40 set_sbi_flag(sbi, SBI_NEED_FSCK); \
41 } while (0)
42#endif
43
44enum {
45 FAULT_KMALLOC,
46 FAULT_KVMALLOC,
47 FAULT_PAGE_ALLOC,
48 FAULT_PAGE_GET,
49 FAULT_ALLOC_BIO, /* it's obsolete due to bio_alloc() will never fail */
50 FAULT_ALLOC_NID,
51 FAULT_ORPHAN,
52 FAULT_BLOCK,
53 FAULT_DIR_DEPTH,
54 FAULT_EVICT_INODE,
55 FAULT_TRUNCATE,
56 FAULT_READ_IO,
57 FAULT_CHECKPOINT,
58 FAULT_DISCARD,
59 FAULT_WRITE_IO,
60 FAULT_SLAB_ALLOC,
61 FAULT_DQUOT_INIT,
62 FAULT_LOCK_OP,
63 FAULT_BLKADDR,
64 FAULT_MAX,
65};
66
67#ifdef CONFIG_F2FS_FAULT_INJECTION
68#define F2FS_ALL_FAULT_TYPE ((1 << FAULT_MAX) - 1)
69
70struct f2fs_fault_info {
71 atomic_t inject_ops;
72 unsigned int inject_rate;
73 unsigned int inject_type;
74};
75
76extern const char *f2fs_fault_name[FAULT_MAX];
77#define IS_FAULT_SET(fi, type) ((fi)->inject_type & (1 << (type)))
78#endif
79
80/*
81 * For mount options
82 */
83#define F2FS_MOUNT_DISABLE_ROLL_FORWARD 0x00000002
84#define F2FS_MOUNT_DISCARD 0x00000004
85#define F2FS_MOUNT_NOHEAP 0x00000008
86#define F2FS_MOUNT_XATTR_USER 0x00000010
87#define F2FS_MOUNT_POSIX_ACL 0x00000020
88#define F2FS_MOUNT_DISABLE_EXT_IDENTIFY 0x00000040
89#define F2FS_MOUNT_INLINE_XATTR 0x00000080
90#define F2FS_MOUNT_INLINE_DATA 0x00000100
91#define F2FS_MOUNT_INLINE_DENTRY 0x00000200
92#define F2FS_MOUNT_FLUSH_MERGE 0x00000400
93#define F2FS_MOUNT_NOBARRIER 0x00000800
94#define F2FS_MOUNT_FASTBOOT 0x00001000
95#define F2FS_MOUNT_READ_EXTENT_CACHE 0x00002000
96#define F2FS_MOUNT_DATA_FLUSH 0x00008000
97#define F2FS_MOUNT_FAULT_INJECTION 0x00010000
98#define F2FS_MOUNT_USRQUOTA 0x00080000
99#define F2FS_MOUNT_GRPQUOTA 0x00100000
100#define F2FS_MOUNT_PRJQUOTA 0x00200000
101#define F2FS_MOUNT_QUOTA 0x00400000
102#define F2FS_MOUNT_INLINE_XATTR_SIZE 0x00800000
103#define F2FS_MOUNT_RESERVE_ROOT 0x01000000
104#define F2FS_MOUNT_DISABLE_CHECKPOINT 0x02000000
105#define F2FS_MOUNT_NORECOVERY 0x04000000
106#define F2FS_MOUNT_ATGC 0x08000000
107#define F2FS_MOUNT_MERGE_CHECKPOINT 0x10000000
108#define F2FS_MOUNT_GC_MERGE 0x20000000
109#define F2FS_MOUNT_COMPRESS_CACHE 0x40000000
110#define F2FS_MOUNT_AGE_EXTENT_CACHE 0x80000000
111
112#define F2FS_OPTION(sbi) ((sbi)->mount_opt)
113#define clear_opt(sbi, option) (F2FS_OPTION(sbi).opt &= ~F2FS_MOUNT_##option)
114#define set_opt(sbi, option) (F2FS_OPTION(sbi).opt |= F2FS_MOUNT_##option)
115#define test_opt(sbi, option) (F2FS_OPTION(sbi).opt & F2FS_MOUNT_##option)
116
117#define ver_after(a, b) (typecheck(unsigned long long, a) && \
118 typecheck(unsigned long long, b) && \
119 ((long long)((a) - (b)) > 0))
120
121typedef u32 block_t; /*
122 * should not change u32, since it is the on-disk block
123 * address format, __le32.
124 */
125typedef u32 nid_t;
126
127#define COMPRESS_EXT_NUM 16
128
129/*
130 * An implementation of an rwsem that is explicitly unfair to readers. This
131 * prevents priority inversion when a low-priority reader acquires the read lock
132 * while sleeping on the write lock but the write lock is needed by
133 * higher-priority clients.
134 */
135
136struct f2fs_rwsem {
137 struct rw_semaphore internal_rwsem;
138#ifdef CONFIG_F2FS_UNFAIR_RWSEM
139 wait_queue_head_t read_waiters;
140#endif
141};
142
143struct f2fs_mount_info {
144 unsigned int opt;
145 int write_io_size_bits; /* Write IO size bits */
146 block_t root_reserved_blocks; /* root reserved blocks */
147 kuid_t s_resuid; /* reserved blocks for uid */
148 kgid_t s_resgid; /* reserved blocks for gid */
149 int active_logs; /* # of active logs */
150 int inline_xattr_size; /* inline xattr size */
151#ifdef CONFIG_F2FS_FAULT_INJECTION
152 struct f2fs_fault_info fault_info; /* For fault injection */
153#endif
154#ifdef CONFIG_QUOTA
155 /* Names of quota files with journalled quota */
156 char *s_qf_names[MAXQUOTAS];
157 int s_jquota_fmt; /* Format of quota to use */
158#endif
159 /* For which write hints are passed down to block layer */
160 int alloc_mode; /* segment allocation policy */
161 int fsync_mode; /* fsync policy */
162 int fs_mode; /* fs mode: LFS or ADAPTIVE */
163 int bggc_mode; /* bggc mode: off, on or sync */
164 int memory_mode; /* memory mode */
165 int discard_unit; /*
166 * discard command's offset/size should
167 * be aligned to this unit: block,
168 * segment or section
169 */
170 struct fscrypt_dummy_policy dummy_enc_policy; /* test dummy encryption */
171 block_t unusable_cap_perc; /* percentage for cap */
172 block_t unusable_cap; /* Amount of space allowed to be
173 * unusable when disabling checkpoint
174 */
175
176 /* For compression */
177 unsigned char compress_algorithm; /* algorithm type */
178 unsigned char compress_log_size; /* cluster log size */
179 unsigned char compress_level; /* compress level */
180 bool compress_chksum; /* compressed data chksum */
181 unsigned char compress_ext_cnt; /* extension count */
182 unsigned char nocompress_ext_cnt; /* nocompress extension count */
183 int compress_mode; /* compression mode */
184 unsigned char extensions[COMPRESS_EXT_NUM][F2FS_EXTENSION_LEN]; /* extensions */
185 unsigned char noextensions[COMPRESS_EXT_NUM][F2FS_EXTENSION_LEN]; /* extensions */
186};
187
188#define F2FS_FEATURE_ENCRYPT 0x0001
189#define F2FS_FEATURE_BLKZONED 0x0002
190#define F2FS_FEATURE_ATOMIC_WRITE 0x0004
191#define F2FS_FEATURE_EXTRA_ATTR 0x0008
192#define F2FS_FEATURE_PRJQUOTA 0x0010
193#define F2FS_FEATURE_INODE_CHKSUM 0x0020
194#define F2FS_FEATURE_FLEXIBLE_INLINE_XATTR 0x0040
195#define F2FS_FEATURE_QUOTA_INO 0x0080
196#define F2FS_FEATURE_INODE_CRTIME 0x0100
197#define F2FS_FEATURE_LOST_FOUND 0x0200
198#define F2FS_FEATURE_VERITY 0x0400
199#define F2FS_FEATURE_SB_CHKSUM 0x0800
200#define F2FS_FEATURE_CASEFOLD 0x1000
201#define F2FS_FEATURE_COMPRESSION 0x2000
202#define F2FS_FEATURE_RO 0x4000
203
204#define __F2FS_HAS_FEATURE(raw_super, mask) \
205 ((raw_super->feature & cpu_to_le32(mask)) != 0)
206#define F2FS_HAS_FEATURE(sbi, mask) __F2FS_HAS_FEATURE(sbi->raw_super, mask)
207
208/*
209 * Default values for user and/or group using reserved blocks
210 */
211#define F2FS_DEF_RESUID 0
212#define F2FS_DEF_RESGID 0
213
214/*
215 * For checkpoint manager
216 */
217enum {
218 NAT_BITMAP,
219 SIT_BITMAP
220};
221
222#define CP_UMOUNT 0x00000001
223#define CP_FASTBOOT 0x00000002
224#define CP_SYNC 0x00000004
225#define CP_RECOVERY 0x00000008
226#define CP_DISCARD 0x00000010
227#define CP_TRIMMED 0x00000020
228#define CP_PAUSE 0x00000040
229#define CP_RESIZE 0x00000080
230
231#define DEF_MAX_DISCARD_REQUEST 8 /* issue 8 discards per round */
232#define DEF_MIN_DISCARD_ISSUE_TIME 50 /* 50 ms, if exists */
233#define DEF_MID_DISCARD_ISSUE_TIME 500 /* 500 ms, if device busy */
234#define DEF_MAX_DISCARD_ISSUE_TIME 60000 /* 60 s, if no candidates */
235#define DEF_DISCARD_URGENT_UTIL 80 /* do more discard over 80% */
236#define DEF_CP_INTERVAL 60 /* 60 secs */
237#define DEF_IDLE_INTERVAL 5 /* 5 secs */
238#define DEF_DISABLE_INTERVAL 5 /* 5 secs */
239#define DEF_DISABLE_QUICK_INTERVAL 1 /* 1 secs */
240#define DEF_UMOUNT_DISCARD_TIMEOUT 5 /* 5 secs */
241
242struct cp_control {
243 int reason;
244 __u64 trim_start;
245 __u64 trim_end;
246 __u64 trim_minlen;
247};
248
249/*
250 * indicate meta/data type
251 */
252enum {
253 META_CP,
254 META_NAT,
255 META_SIT,
256 META_SSA,
257 META_MAX,
258 META_POR,
259 DATA_GENERIC, /* check range only */
260 DATA_GENERIC_ENHANCE, /* strong check on range and segment bitmap */
261 DATA_GENERIC_ENHANCE_READ, /*
262 * strong check on range and segment
263 * bitmap but no warning due to race
264 * condition of read on truncated area
265 * by extent_cache
266 */
267 DATA_GENERIC_ENHANCE_UPDATE, /*
268 * strong check on range and segment
269 * bitmap for update case
270 */
271 META_GENERIC,
272};
273
274/* for the list of ino */
275enum {
276 ORPHAN_INO, /* for orphan ino list */
277 APPEND_INO, /* for append ino list */
278 UPDATE_INO, /* for update ino list */
279 TRANS_DIR_INO, /* for transactions dir ino list */
280 FLUSH_INO, /* for multiple device flushing */
281 MAX_INO_ENTRY, /* max. list */
282};
283
284struct ino_entry {
285 struct list_head list; /* list head */
286 nid_t ino; /* inode number */
287 unsigned int dirty_device; /* dirty device bitmap */
288};
289
290/* for the list of inodes to be GCed */
291struct inode_entry {
292 struct list_head list; /* list head */
293 struct inode *inode; /* vfs inode pointer */
294};
295
296struct fsync_node_entry {
297 struct list_head list; /* list head */
298 struct page *page; /* warm node page pointer */
299 unsigned int seq_id; /* sequence id */
300};
301
302struct ckpt_req {
303 struct completion wait; /* completion for checkpoint done */
304 struct llist_node llnode; /* llist_node to be linked in wait queue */
305 int ret; /* return code of checkpoint */
306 ktime_t queue_time; /* request queued time */
307};
308
309struct ckpt_req_control {
310 struct task_struct *f2fs_issue_ckpt; /* checkpoint task */
311 int ckpt_thread_ioprio; /* checkpoint merge thread ioprio */
312 wait_queue_head_t ckpt_wait_queue; /* waiting queue for wake-up */
313 atomic_t issued_ckpt; /* # of actually issued ckpts */
314 atomic_t total_ckpt; /* # of total ckpts */
315 atomic_t queued_ckpt; /* # of queued ckpts */
316 struct llist_head issue_list; /* list for command issue */
317 spinlock_t stat_lock; /* lock for below checkpoint time stats */
318 unsigned int cur_time; /* cur wait time in msec for currently issued checkpoint */
319 unsigned int peak_time; /* peak wait time in msec until now */
320};
321
322/* for the bitmap indicate blocks to be discarded */
323struct discard_entry {
324 struct list_head list; /* list head */
325 block_t start_blkaddr; /* start blockaddr of current segment */
326 unsigned char discard_map[SIT_VBLOCK_MAP_SIZE]; /* segment discard bitmap */
327};
328
329/* minimum discard granularity, unit: block count */
330#define MIN_DISCARD_GRANULARITY 1
331/* default discard granularity of inner discard thread, unit: block count */
332#define DEFAULT_DISCARD_GRANULARITY 16
333/* default maximum discard granularity of ordered discard, unit: block count */
334#define DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY 16
335
336/* max discard pend list number */
337#define MAX_PLIST_NUM 512
338#define plist_idx(blk_num) ((blk_num) >= MAX_PLIST_NUM ? \
339 (MAX_PLIST_NUM - 1) : ((blk_num) - 1))
340
341enum {
342 D_PREP, /* initial */
343 D_PARTIAL, /* partially submitted */
344 D_SUBMIT, /* all submitted */
345 D_DONE, /* finished */
346};
347
348struct discard_info {
349 block_t lstart; /* logical start address */
350 block_t len; /* length */
351 block_t start; /* actual start address in dev */
352};
353
354struct discard_cmd {
355 struct rb_node rb_node; /* rb node located in rb-tree */
356 union {
357 struct {
358 block_t lstart; /* logical start address */
359 block_t len; /* length */
360 block_t start; /* actual start address in dev */
361 };
362 struct discard_info di; /* discard info */
363
364 };
365 struct list_head list; /* command list */
366 struct completion wait; /* compleation */
367 struct block_device *bdev; /* bdev */
368 unsigned short ref; /* reference count */
369 unsigned char state; /* state */
370 unsigned char queued; /* queued discard */
371 int error; /* bio error */
372 spinlock_t lock; /* for state/bio_ref updating */
373 unsigned short bio_ref; /* bio reference count */
374};
375
376enum {
377 DPOLICY_BG,
378 DPOLICY_FORCE,
379 DPOLICY_FSTRIM,
380 DPOLICY_UMOUNT,
381 MAX_DPOLICY,
382};
383
384struct discard_policy {
385 int type; /* type of discard */
386 unsigned int min_interval; /* used for candidates exist */
387 unsigned int mid_interval; /* used for device busy */
388 unsigned int max_interval; /* used for candidates not exist */
389 unsigned int max_requests; /* # of discards issued per round */
390 unsigned int io_aware_gran; /* minimum granularity discard not be aware of I/O */
391 bool io_aware; /* issue discard in idle time */
392 bool sync; /* submit discard with REQ_SYNC flag */
393 bool ordered; /* issue discard by lba order */
394 bool timeout; /* discard timeout for put_super */
395 unsigned int granularity; /* discard granularity */
396};
397
398struct discard_cmd_control {
399 struct task_struct *f2fs_issue_discard; /* discard thread */
400 struct list_head entry_list; /* 4KB discard entry list */
401 struct list_head pend_list[MAX_PLIST_NUM];/* store pending entries */
402 struct list_head wait_list; /* store on-flushing entries */
403 struct list_head fstrim_list; /* in-flight discard from fstrim */
404 wait_queue_head_t discard_wait_queue; /* waiting queue for wake-up */
405 unsigned int discard_wake; /* to wake up discard thread */
406 struct mutex cmd_lock;
407 unsigned int nr_discards; /* # of discards in the list */
408 unsigned int max_discards; /* max. discards to be issued */
409 unsigned int max_discard_request; /* max. discard request per round */
410 unsigned int min_discard_issue_time; /* min. interval between discard issue */
411 unsigned int mid_discard_issue_time; /* mid. interval between discard issue */
412 unsigned int max_discard_issue_time; /* max. interval between discard issue */
413 unsigned int discard_urgent_util; /* utilization which issue discard proactively */
414 unsigned int discard_granularity; /* discard granularity */
415 unsigned int max_ordered_discard; /* maximum discard granularity issued by lba order */
416 unsigned int undiscard_blks; /* # of undiscard blocks */
417 unsigned int next_pos; /* next discard position */
418 atomic_t issued_discard; /* # of issued discard */
419 atomic_t queued_discard; /* # of queued discard */
420 atomic_t discard_cmd_cnt; /* # of cached cmd count */
421 struct rb_root_cached root; /* root of discard rb-tree */
422 bool rbtree_check; /* config for consistence check */
423};
424
425/* for the list of fsync inodes, used only during recovery */
426struct fsync_inode_entry {
427 struct list_head list; /* list head */
428 struct inode *inode; /* vfs inode pointer */
429 block_t blkaddr; /* block address locating the last fsync */
430 block_t last_dentry; /* block address locating the last dentry */
431};
432
433#define nats_in_cursum(jnl) (le16_to_cpu((jnl)->n_nats))
434#define sits_in_cursum(jnl) (le16_to_cpu((jnl)->n_sits))
435
436#define nat_in_journal(jnl, i) ((jnl)->nat_j.entries[i].ne)
437#define nid_in_journal(jnl, i) ((jnl)->nat_j.entries[i].nid)
438#define sit_in_journal(jnl, i) ((jnl)->sit_j.entries[i].se)
439#define segno_in_journal(jnl, i) ((jnl)->sit_j.entries[i].segno)
440
441#define MAX_NAT_JENTRIES(jnl) (NAT_JOURNAL_ENTRIES - nats_in_cursum(jnl))
442#define MAX_SIT_JENTRIES(jnl) (SIT_JOURNAL_ENTRIES - sits_in_cursum(jnl))
443
444static inline int update_nats_in_cursum(struct f2fs_journal *journal, int i)
445{
446 int before = nats_in_cursum(journal);
447
448 journal->n_nats = cpu_to_le16(before + i);
449 return before;
450}
451
452static inline int update_sits_in_cursum(struct f2fs_journal *journal, int i)
453{
454 int before = sits_in_cursum(journal);
455
456 journal->n_sits = cpu_to_le16(before + i);
457 return before;
458}
459
460static inline bool __has_cursum_space(struct f2fs_journal *journal,
461 int size, int type)
462{
463 if (type == NAT_JOURNAL)
464 return size <= MAX_NAT_JENTRIES(journal);
465 return size <= MAX_SIT_JENTRIES(journal);
466}
467
468/* for inline stuff */
469#define DEF_INLINE_RESERVED_SIZE 1
470static inline int get_extra_isize(struct inode *inode);
471static inline int get_inline_xattr_addrs(struct inode *inode);
472#define MAX_INLINE_DATA(inode) (sizeof(__le32) * \
473 (CUR_ADDRS_PER_INODE(inode) - \
474 get_inline_xattr_addrs(inode) - \
475 DEF_INLINE_RESERVED_SIZE))
476
477/* for inline dir */
478#define NR_INLINE_DENTRY(inode) (MAX_INLINE_DATA(inode) * BITS_PER_BYTE / \
479 ((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * \
480 BITS_PER_BYTE + 1))
481#define INLINE_DENTRY_BITMAP_SIZE(inode) \
482 DIV_ROUND_UP(NR_INLINE_DENTRY(inode), BITS_PER_BYTE)
483#define INLINE_RESERVED_SIZE(inode) (MAX_INLINE_DATA(inode) - \
484 ((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * \
485 NR_INLINE_DENTRY(inode) + \
486 INLINE_DENTRY_BITMAP_SIZE(inode)))
487
488/*
489 * For INODE and NODE manager
490 */
491/* for directory operations */
492
493struct f2fs_filename {
494 /*
495 * The filename the user specified. This is NULL for some
496 * filesystem-internal operations, e.g. converting an inline directory
497 * to a non-inline one, or roll-forward recovering an encrypted dentry.
498 */
499 const struct qstr *usr_fname;
500
501 /*
502 * The on-disk filename. For encrypted directories, this is encrypted.
503 * This may be NULL for lookups in an encrypted dir without the key.
504 */
505 struct fscrypt_str disk_name;
506
507 /* The dirhash of this filename */
508 f2fs_hash_t hash;
509
510#ifdef CONFIG_FS_ENCRYPTION
511 /*
512 * For lookups in encrypted directories: either the buffer backing
513 * disk_name, or a buffer that holds the decoded no-key name.
514 */
515 struct fscrypt_str crypto_buf;
516#endif
517#if IS_ENABLED(CONFIG_UNICODE)
518 /*
519 * For casefolded directories: the casefolded name, but it's left NULL
520 * if the original name is not valid Unicode, if the original name is
521 * "." or "..", if the directory is both casefolded and encrypted and
522 * its encryption key is unavailable, or if the filesystem is doing an
523 * internal operation where usr_fname is also NULL. In all these cases
524 * we fall back to treating the name as an opaque byte sequence.
525 */
526 struct fscrypt_str cf_name;
527#endif
528};
529
530struct f2fs_dentry_ptr {
531 struct inode *inode;
532 void *bitmap;
533 struct f2fs_dir_entry *dentry;
534 __u8 (*filename)[F2FS_SLOT_LEN];
535 int max;
536 int nr_bitmap;
537};
538
539static inline void make_dentry_ptr_block(struct inode *inode,
540 struct f2fs_dentry_ptr *d, struct f2fs_dentry_block *t)
541{
542 d->inode = inode;
543 d->max = NR_DENTRY_IN_BLOCK;
544 d->nr_bitmap = SIZE_OF_DENTRY_BITMAP;
545 d->bitmap = t->dentry_bitmap;
546 d->dentry = t->dentry;
547 d->filename = t->filename;
548}
549
550static inline void make_dentry_ptr_inline(struct inode *inode,
551 struct f2fs_dentry_ptr *d, void *t)
552{
553 int entry_cnt = NR_INLINE_DENTRY(inode);
554 int bitmap_size = INLINE_DENTRY_BITMAP_SIZE(inode);
555 int reserved_size = INLINE_RESERVED_SIZE(inode);
556
557 d->inode = inode;
558 d->max = entry_cnt;
559 d->nr_bitmap = bitmap_size;
560 d->bitmap = t;
561 d->dentry = t + bitmap_size + reserved_size;
562 d->filename = t + bitmap_size + reserved_size +
563 SIZE_OF_DIR_ENTRY * entry_cnt;
564}
565
566/*
567 * XATTR_NODE_OFFSET stores xattrs to one node block per file keeping -1
568 * as its node offset to distinguish from index node blocks.
569 * But some bits are used to mark the node block.
570 */
571#define XATTR_NODE_OFFSET ((((unsigned int)-1) << OFFSET_BIT_SHIFT) \
572 >> OFFSET_BIT_SHIFT)
573enum {
574 ALLOC_NODE, /* allocate a new node page if needed */
575 LOOKUP_NODE, /* look up a node without readahead */
576 LOOKUP_NODE_RA, /*
577 * look up a node with readahead called
578 * by get_data_block.
579 */
580};
581
582#define DEFAULT_RETRY_IO_COUNT 8 /* maximum retry read IO or flush count */
583
584/* congestion wait timeout value, default: 20ms */
585#define DEFAULT_IO_TIMEOUT (msecs_to_jiffies(20))
586
587/* maximum retry quota flush count */
588#define DEFAULT_RETRY_QUOTA_FLUSH_COUNT 8
589
590/* maximum retry of EIO'ed page */
591#define MAX_RETRY_PAGE_EIO 100
592
593#define F2FS_LINK_MAX 0xffffffff /* maximum link count per file */
594
595#define MAX_DIR_RA_PAGES 4 /* maximum ra pages of dir */
596
597/* dirty segments threshold for triggering CP */
598#define DEFAULT_DIRTY_THRESHOLD 4
599
600#define RECOVERY_MAX_RA_BLOCKS BIO_MAX_VECS
601#define RECOVERY_MIN_RA_BLOCKS 1
602
603#define F2FS_ONSTACK_PAGES 16 /* nr of onstack pages */
604
605/* for in-memory extent cache entry */
606#define F2FS_MIN_EXTENT_LEN 64 /* minimum extent length */
607
608/* number of extent info in extent cache we try to shrink */
609#define READ_EXTENT_CACHE_SHRINK_NUMBER 128
610
611/* number of age extent info in extent cache we try to shrink */
612#define AGE_EXTENT_CACHE_SHRINK_NUMBER 128
613#define LAST_AGE_WEIGHT 30
614#define SAME_AGE_REGION 1024
615
616/*
617 * Define data block with age less than 1GB as hot data
618 * define data block with age less than 10GB but more than 1GB as warm data
619 */
620#define DEF_HOT_DATA_AGE_THRESHOLD 262144
621#define DEF_WARM_DATA_AGE_THRESHOLD 2621440
622
623/* extent cache type */
624enum extent_type {
625 EX_READ,
626 EX_BLOCK_AGE,
627 NR_EXTENT_CACHES,
628};
629
630struct rb_entry {
631 struct rb_node rb_node; /* rb node located in rb-tree */
632 union {
633 struct {
634 unsigned int ofs; /* start offset of the entry */
635 unsigned int len; /* length of the entry */
636 };
637 unsigned long long key; /* 64-bits key */
638 } __packed;
639};
640
641struct extent_info {
642 unsigned int fofs; /* start offset in a file */
643 unsigned int len; /* length of the extent */
644 union {
645 /* read extent_cache */
646 struct {
647 /* start block address of the extent */
648 block_t blk;
649#ifdef CONFIG_F2FS_FS_COMPRESSION
650 /* physical extent length of compressed blocks */
651 unsigned int c_len;
652#endif
653 };
654 /* block age extent_cache */
655 struct {
656 /* block age of the extent */
657 unsigned long long age;
658 /* last total blocks allocated */
659 unsigned long long last_blocks;
660 };
661 };
662};
663
664struct extent_node {
665 struct rb_node rb_node; /* rb node located in rb-tree */
666 struct extent_info ei; /* extent info */
667 struct list_head list; /* node in global extent list of sbi */
668 struct extent_tree *et; /* extent tree pointer */
669};
670
671struct extent_tree {
672 nid_t ino; /* inode number */
673 enum extent_type type; /* keep the extent tree type */
674 struct rb_root_cached root; /* root of extent info rb-tree */
675 struct extent_node *cached_en; /* recently accessed extent node */
676 struct list_head list; /* to be used by sbi->zombie_list */
677 rwlock_t lock; /* protect extent info rb-tree */
678 atomic_t node_cnt; /* # of extent node in rb-tree*/
679 bool largest_updated; /* largest extent updated */
680 struct extent_info largest; /* largest cached extent for EX_READ */
681};
682
683struct extent_tree_info {
684 struct radix_tree_root extent_tree_root;/* cache extent cache entries */
685 struct mutex extent_tree_lock; /* locking extent radix tree */
686 struct list_head extent_list; /* lru list for shrinker */
687 spinlock_t extent_lock; /* locking extent lru list */
688 atomic_t total_ext_tree; /* extent tree count */
689 struct list_head zombie_list; /* extent zombie tree list */
690 atomic_t total_zombie_tree; /* extent zombie tree count */
691 atomic_t total_ext_node; /* extent info count */
692};
693
694/*
695 * This structure is taken from ext4_map_blocks.
696 *
697 * Note that, however, f2fs uses NEW and MAPPED flags for f2fs_map_blocks().
698 */
699#define F2FS_MAP_NEW (1 << BH_New)
700#define F2FS_MAP_MAPPED (1 << BH_Mapped)
701#define F2FS_MAP_UNWRITTEN (1 << BH_Unwritten)
702#define F2FS_MAP_FLAGS (F2FS_MAP_NEW | F2FS_MAP_MAPPED |\
703 F2FS_MAP_UNWRITTEN)
704
705struct f2fs_map_blocks {
706 struct block_device *m_bdev; /* for multi-device dio */
707 block_t m_pblk;
708 block_t m_lblk;
709 unsigned int m_len;
710 unsigned int m_flags;
711 pgoff_t *m_next_pgofs; /* point next possible non-hole pgofs */
712 pgoff_t *m_next_extent; /* point to next possible extent */
713 int m_seg_type;
714 bool m_may_create; /* indicate it is from write path */
715 bool m_multidev_dio; /* indicate it allows multi-device dio */
716};
717
718/* for flag in get_data_block */
719enum {
720 F2FS_GET_BLOCK_DEFAULT,
721 F2FS_GET_BLOCK_FIEMAP,
722 F2FS_GET_BLOCK_BMAP,
723 F2FS_GET_BLOCK_DIO,
724 F2FS_GET_BLOCK_PRE_DIO,
725 F2FS_GET_BLOCK_PRE_AIO,
726 F2FS_GET_BLOCK_PRECACHE,
727};
728
729/*
730 * i_advise uses FADVISE_XXX_BIT. We can add additional hints later.
731 */
732#define FADVISE_COLD_BIT 0x01
733#define FADVISE_LOST_PINO_BIT 0x02
734#define FADVISE_ENCRYPT_BIT 0x04
735#define FADVISE_ENC_NAME_BIT 0x08
736#define FADVISE_KEEP_SIZE_BIT 0x10
737#define FADVISE_HOT_BIT 0x20
738#define FADVISE_VERITY_BIT 0x40
739#define FADVISE_TRUNC_BIT 0x80
740
741#define FADVISE_MODIFIABLE_BITS (FADVISE_COLD_BIT | FADVISE_HOT_BIT)
742
743#define file_is_cold(inode) is_file(inode, FADVISE_COLD_BIT)
744#define file_set_cold(inode) set_file(inode, FADVISE_COLD_BIT)
745#define file_clear_cold(inode) clear_file(inode, FADVISE_COLD_BIT)
746
747#define file_wrong_pino(inode) is_file(inode, FADVISE_LOST_PINO_BIT)
748#define file_lost_pino(inode) set_file(inode, FADVISE_LOST_PINO_BIT)
749#define file_got_pino(inode) clear_file(inode, FADVISE_LOST_PINO_BIT)
750
751#define file_is_encrypt(inode) is_file(inode, FADVISE_ENCRYPT_BIT)
752#define file_set_encrypt(inode) set_file(inode, FADVISE_ENCRYPT_BIT)
753
754#define file_enc_name(inode) is_file(inode, FADVISE_ENC_NAME_BIT)
755#define file_set_enc_name(inode) set_file(inode, FADVISE_ENC_NAME_BIT)
756
757#define file_keep_isize(inode) is_file(inode, FADVISE_KEEP_SIZE_BIT)
758#define file_set_keep_isize(inode) set_file(inode, FADVISE_KEEP_SIZE_BIT)
759
760#define file_is_hot(inode) is_file(inode, FADVISE_HOT_BIT)
761#define file_set_hot(inode) set_file(inode, FADVISE_HOT_BIT)
762#define file_clear_hot(inode) clear_file(inode, FADVISE_HOT_BIT)
763
764#define file_is_verity(inode) is_file(inode, FADVISE_VERITY_BIT)
765#define file_set_verity(inode) set_file(inode, FADVISE_VERITY_BIT)
766
767#define file_should_truncate(inode) is_file(inode, FADVISE_TRUNC_BIT)
768#define file_need_truncate(inode) set_file(inode, FADVISE_TRUNC_BIT)
769#define file_dont_truncate(inode) clear_file(inode, FADVISE_TRUNC_BIT)
770
771#define DEF_DIR_LEVEL 0
772
773enum {
774 GC_FAILURE_PIN,
775 MAX_GC_FAILURE
776};
777
778/* used for f2fs_inode_info->flags */
779enum {
780 FI_NEW_INODE, /* indicate newly allocated inode */
781 FI_DIRTY_INODE, /* indicate inode is dirty or not */
782 FI_AUTO_RECOVER, /* indicate inode is recoverable */
783 FI_DIRTY_DIR, /* indicate directory has dirty pages */
784 FI_INC_LINK, /* need to increment i_nlink */
785 FI_ACL_MODE, /* indicate acl mode */
786 FI_NO_ALLOC, /* should not allocate any blocks */
787 FI_FREE_NID, /* free allocated nide */
788 FI_NO_EXTENT, /* not to use the extent cache */
789 FI_INLINE_XATTR, /* used for inline xattr */
790 FI_INLINE_DATA, /* used for inline data*/
791 FI_INLINE_DENTRY, /* used for inline dentry */
792 FI_APPEND_WRITE, /* inode has appended data */
793 FI_UPDATE_WRITE, /* inode has in-place-update data */
794 FI_NEED_IPU, /* used for ipu per file */
795 FI_ATOMIC_FILE, /* indicate atomic file */
796 FI_FIRST_BLOCK_WRITTEN, /* indicate #0 data block was written */
797 FI_DROP_CACHE, /* drop dirty page cache */
798 FI_DATA_EXIST, /* indicate data exists */
799 FI_INLINE_DOTS, /* indicate inline dot dentries */
800 FI_SKIP_WRITES, /* should skip data page writeback */
801 FI_OPU_WRITE, /* used for opu per file */
802 FI_DIRTY_FILE, /* indicate regular/symlink has dirty pages */
803 FI_PREALLOCATED_ALL, /* all blocks for write were preallocated */
804 FI_HOT_DATA, /* indicate file is hot */
805 FI_EXTRA_ATTR, /* indicate file has extra attribute */
806 FI_PROJ_INHERIT, /* indicate file inherits projectid */
807 FI_PIN_FILE, /* indicate file should not be gced */
808 FI_VERITY_IN_PROGRESS, /* building fs-verity Merkle tree */
809 FI_COMPRESSED_FILE, /* indicate file's data can be compressed */
810 FI_COMPRESS_CORRUPT, /* indicate compressed cluster is corrupted */
811 FI_MMAP_FILE, /* indicate file was mmapped */
812 FI_ENABLE_COMPRESS, /* enable compression in "user" compression mode */
813 FI_COMPRESS_RELEASED, /* compressed blocks were released */
814 FI_ALIGNED_WRITE, /* enable aligned write */
815 FI_COW_FILE, /* indicate COW file */
816 FI_ATOMIC_COMMITTED, /* indicate atomic commit completed except disk sync */
817 FI_ATOMIC_REPLACE, /* indicate atomic replace */
818 FI_MAX, /* max flag, never be used */
819};
820
821struct f2fs_inode_info {
822 struct inode vfs_inode; /* serve a vfs inode */
823 unsigned long i_flags; /* keep an inode flags for ioctl */
824 unsigned char i_advise; /* use to give file attribute hints */
825 unsigned char i_dir_level; /* use for dentry level for large dir */
826 unsigned int i_current_depth; /* only for directory depth */
827 /* for gc failure statistic */
828 unsigned int i_gc_failures[MAX_GC_FAILURE];
829 unsigned int i_pino; /* parent inode number */
830 umode_t i_acl_mode; /* keep file acl mode temporarily */
831
832 /* Use below internally in f2fs*/
833 unsigned long flags[BITS_TO_LONGS(FI_MAX)]; /* use to pass per-file flags */
834 struct f2fs_rwsem i_sem; /* protect fi info */
835 atomic_t dirty_pages; /* # of dirty pages */
836 f2fs_hash_t chash; /* hash value of given file name */
837 unsigned int clevel; /* maximum level of given file name */
838 struct task_struct *task; /* lookup and create consistency */
839 struct task_struct *cp_task; /* separate cp/wb IO stats*/
840 struct task_struct *wb_task; /* indicate inode is in context of writeback */
841 nid_t i_xattr_nid; /* node id that contains xattrs */
842 loff_t last_disk_size; /* lastly written file size */
843 spinlock_t i_size_lock; /* protect last_disk_size */
844
845#ifdef CONFIG_QUOTA
846 struct dquot *i_dquot[MAXQUOTAS];
847
848 /* quota space reservation, managed internally by quota code */
849 qsize_t i_reserved_quota;
850#endif
851 struct list_head dirty_list; /* dirty list for dirs and files */
852 struct list_head gdirty_list; /* linked in global dirty list */
853 struct task_struct *atomic_write_task; /* store atomic write task */
854 struct extent_tree *extent_tree[NR_EXTENT_CACHES];
855 /* cached extent_tree entry */
856 struct inode *cow_inode; /* copy-on-write inode for atomic write */
857
858 /* avoid racing between foreground op and gc */
859 struct f2fs_rwsem i_gc_rwsem[2];
860 struct f2fs_rwsem i_xattr_sem; /* avoid racing between reading and changing EAs */
861
862 int i_extra_isize; /* size of extra space located in i_addr */
863 kprojid_t i_projid; /* id for project quota */
864 int i_inline_xattr_size; /* inline xattr size */
865 struct timespec64 i_crtime; /* inode creation time */
866 struct timespec64 i_disk_time[4];/* inode disk times */
867
868 /* for file compress */
869 atomic_t i_compr_blocks; /* # of compressed blocks */
870 unsigned char i_compress_algorithm; /* algorithm type */
871 unsigned char i_log_cluster_size; /* log of cluster size */
872 unsigned char i_compress_level; /* compress level (lz4hc,zstd) */
873 unsigned short i_compress_flag; /* compress flag */
874 unsigned int i_cluster_size; /* cluster size */
875
876 unsigned int atomic_write_cnt;
877 loff_t original_i_size; /* original i_size before atomic write */
878};
879
880static inline void get_read_extent_info(struct extent_info *ext,
881 struct f2fs_extent *i_ext)
882{
883 ext->fofs = le32_to_cpu(i_ext->fofs);
884 ext->blk = le32_to_cpu(i_ext->blk);
885 ext->len = le32_to_cpu(i_ext->len);
886}
887
888static inline void set_raw_read_extent(struct extent_info *ext,
889 struct f2fs_extent *i_ext)
890{
891 i_ext->fofs = cpu_to_le32(ext->fofs);
892 i_ext->blk = cpu_to_le32(ext->blk);
893 i_ext->len = cpu_to_le32(ext->len);
894}
895
896static inline bool __is_discard_mergeable(struct discard_info *back,
897 struct discard_info *front, unsigned int max_len)
898{
899 return (back->lstart + back->len == front->lstart) &&
900 (back->len + front->len <= max_len);
901}
902
903static inline bool __is_discard_back_mergeable(struct discard_info *cur,
904 struct discard_info *back, unsigned int max_len)
905{
906 return __is_discard_mergeable(back, cur, max_len);
907}
908
909static inline bool __is_discard_front_mergeable(struct discard_info *cur,
910 struct discard_info *front, unsigned int max_len)
911{
912 return __is_discard_mergeable(cur, front, max_len);
913}
914
915/*
916 * For free nid management
917 */
918enum nid_state {
919 FREE_NID, /* newly added to free nid list */
920 PREALLOC_NID, /* it is preallocated */
921 MAX_NID_STATE,
922};
923
924enum nat_state {
925 TOTAL_NAT,
926 DIRTY_NAT,
927 RECLAIMABLE_NAT,
928 MAX_NAT_STATE,
929};
930
931struct f2fs_nm_info {
932 block_t nat_blkaddr; /* base disk address of NAT */
933 nid_t max_nid; /* maximum possible node ids */
934 nid_t available_nids; /* # of available node ids */
935 nid_t next_scan_nid; /* the next nid to be scanned */
936 nid_t max_rf_node_blocks; /* max # of nodes for recovery */
937 unsigned int ram_thresh; /* control the memory footprint */
938 unsigned int ra_nid_pages; /* # of nid pages to be readaheaded */
939 unsigned int dirty_nats_ratio; /* control dirty nats ratio threshold */
940
941 /* NAT cache management */
942 struct radix_tree_root nat_root;/* root of the nat entry cache */
943 struct radix_tree_root nat_set_root;/* root of the nat set cache */
944 struct f2fs_rwsem nat_tree_lock; /* protect nat entry tree */
945 struct list_head nat_entries; /* cached nat entry list (clean) */
946 spinlock_t nat_list_lock; /* protect clean nat entry list */
947 unsigned int nat_cnt[MAX_NAT_STATE]; /* the # of cached nat entries */
948 unsigned int nat_blocks; /* # of nat blocks */
949
950 /* free node ids management */
951 struct radix_tree_root free_nid_root;/* root of the free_nid cache */
952 struct list_head free_nid_list; /* list for free nids excluding preallocated nids */
953 unsigned int nid_cnt[MAX_NID_STATE]; /* the number of free node id */
954 spinlock_t nid_list_lock; /* protect nid lists ops */
955 struct mutex build_lock; /* lock for build free nids */
956 unsigned char **free_nid_bitmap;
957 unsigned char *nat_block_bitmap;
958 unsigned short *free_nid_count; /* free nid count of NAT block */
959
960 /* for checkpoint */
961 char *nat_bitmap; /* NAT bitmap pointer */
962
963 unsigned int nat_bits_blocks; /* # of nat bits blocks */
964 unsigned char *nat_bits; /* NAT bits blocks */
965 unsigned char *full_nat_bits; /* full NAT pages */
966 unsigned char *empty_nat_bits; /* empty NAT pages */
967#ifdef CONFIG_F2FS_CHECK_FS
968 char *nat_bitmap_mir; /* NAT bitmap mirror */
969#endif
970 int bitmap_size; /* bitmap size */
971};
972
973/*
974 * this structure is used as one of function parameters.
975 * all the information are dedicated to a given direct node block determined
976 * by the data offset in a file.
977 */
978struct dnode_of_data {
979 struct inode *inode; /* vfs inode pointer */
980 struct page *inode_page; /* its inode page, NULL is possible */
981 struct page *node_page; /* cached direct node page */
982 nid_t nid; /* node id of the direct node block */
983 unsigned int ofs_in_node; /* data offset in the node page */
984 bool inode_page_locked; /* inode page is locked or not */
985 bool node_changed; /* is node block changed */
986 char cur_level; /* level of hole node page */
987 char max_level; /* level of current page located */
988 block_t data_blkaddr; /* block address of the node block */
989};
990
991static inline void set_new_dnode(struct dnode_of_data *dn, struct inode *inode,
992 struct page *ipage, struct page *npage, nid_t nid)
993{
994 memset(dn, 0, sizeof(*dn));
995 dn->inode = inode;
996 dn->inode_page = ipage;
997 dn->node_page = npage;
998 dn->nid = nid;
999}
1000
1001/*
1002 * For SIT manager
1003 *
1004 * By default, there are 6 active log areas across the whole main area.
1005 * When considering hot and cold data separation to reduce cleaning overhead,
1006 * we split 3 for data logs and 3 for node logs as hot, warm, and cold types,
1007 * respectively.
1008 * In the current design, you should not change the numbers intentionally.
1009 * Instead, as a mount option such as active_logs=x, you can use 2, 4, and 6
1010 * logs individually according to the underlying devices. (default: 6)
1011 * Just in case, on-disk layout covers maximum 16 logs that consist of 8 for
1012 * data and 8 for node logs.
1013 */
1014#define NR_CURSEG_DATA_TYPE (3)
1015#define NR_CURSEG_NODE_TYPE (3)
1016#define NR_CURSEG_INMEM_TYPE (2)
1017#define NR_CURSEG_RO_TYPE (2)
1018#define NR_CURSEG_PERSIST_TYPE (NR_CURSEG_DATA_TYPE + NR_CURSEG_NODE_TYPE)
1019#define NR_CURSEG_TYPE (NR_CURSEG_INMEM_TYPE + NR_CURSEG_PERSIST_TYPE)
1020
1021enum {
1022 CURSEG_HOT_DATA = 0, /* directory entry blocks */
1023 CURSEG_WARM_DATA, /* data blocks */
1024 CURSEG_COLD_DATA, /* multimedia or GCed data blocks */
1025 CURSEG_HOT_NODE, /* direct node blocks of directory files */
1026 CURSEG_WARM_NODE, /* direct node blocks of normal files */
1027 CURSEG_COLD_NODE, /* indirect node blocks */
1028 NR_PERSISTENT_LOG, /* number of persistent log */
1029 CURSEG_COLD_DATA_PINNED = NR_PERSISTENT_LOG,
1030 /* pinned file that needs consecutive block address */
1031 CURSEG_ALL_DATA_ATGC, /* SSR alloctor in hot/warm/cold data area */
1032 NO_CHECK_TYPE, /* number of persistent & inmem log */
1033};
1034
1035struct flush_cmd {
1036 struct completion wait;
1037 struct llist_node llnode;
1038 nid_t ino;
1039 int ret;
1040};
1041
1042struct flush_cmd_control {
1043 struct task_struct *f2fs_issue_flush; /* flush thread */
1044 wait_queue_head_t flush_wait_queue; /* waiting queue for wake-up */
1045 atomic_t issued_flush; /* # of issued flushes */
1046 atomic_t queued_flush; /* # of queued flushes */
1047 struct llist_head issue_list; /* list for command issue */
1048 struct llist_node *dispatch_list; /* list for command dispatch */
1049};
1050
1051struct f2fs_sm_info {
1052 struct sit_info *sit_info; /* whole segment information */
1053 struct free_segmap_info *free_info; /* free segment information */
1054 struct dirty_seglist_info *dirty_info; /* dirty segment information */
1055 struct curseg_info *curseg_array; /* active segment information */
1056
1057 struct f2fs_rwsem curseg_lock; /* for preventing curseg change */
1058
1059 block_t seg0_blkaddr; /* block address of 0'th segment */
1060 block_t main_blkaddr; /* start block address of main area */
1061 block_t ssa_blkaddr; /* start block address of SSA area */
1062
1063 unsigned int segment_count; /* total # of segments */
1064 unsigned int main_segments; /* # of segments in main area */
1065 unsigned int reserved_segments; /* # of reserved segments */
1066 unsigned int additional_reserved_segments;/* reserved segs for IO align feature */
1067 unsigned int ovp_segments; /* # of overprovision segments */
1068
1069 /* a threshold to reclaim prefree segments */
1070 unsigned int rec_prefree_segments;
1071
1072 struct list_head sit_entry_set; /* sit entry set list */
1073
1074 unsigned int ipu_policy; /* in-place-update policy */
1075 unsigned int min_ipu_util; /* in-place-update threshold */
1076 unsigned int min_fsync_blocks; /* threshold for fsync */
1077 unsigned int min_seq_blocks; /* threshold for sequential blocks */
1078 unsigned int min_hot_blocks; /* threshold for hot block allocation */
1079 unsigned int min_ssr_sections; /* threshold to trigger SSR allocation */
1080
1081 /* for flush command control */
1082 struct flush_cmd_control *fcc_info;
1083
1084 /* for discard command control */
1085 struct discard_cmd_control *dcc_info;
1086};
1087
1088/*
1089 * For superblock
1090 */
1091/*
1092 * COUNT_TYPE for monitoring
1093 *
1094 * f2fs monitors the number of several block types such as on-writeback,
1095 * dirty dentry blocks, dirty node blocks, and dirty meta blocks.
1096 */
1097#define WB_DATA_TYPE(p) (__is_cp_guaranteed(p) ? F2FS_WB_CP_DATA : F2FS_WB_DATA)
1098enum count_type {
1099 F2FS_DIRTY_DENTS,
1100 F2FS_DIRTY_DATA,
1101 F2FS_DIRTY_QDATA,
1102 F2FS_DIRTY_NODES,
1103 F2FS_DIRTY_META,
1104 F2FS_DIRTY_IMETA,
1105 F2FS_WB_CP_DATA,
1106 F2FS_WB_DATA,
1107 F2FS_RD_DATA,
1108 F2FS_RD_NODE,
1109 F2FS_RD_META,
1110 F2FS_DIO_WRITE,
1111 F2FS_DIO_READ,
1112 NR_COUNT_TYPE,
1113};
1114
1115/*
1116 * The below are the page types of bios used in submit_bio().
1117 * The available types are:
1118 * DATA User data pages. It operates as async mode.
1119 * NODE Node pages. It operates as async mode.
1120 * META FS metadata pages such as SIT, NAT, CP.
1121 * NR_PAGE_TYPE The number of page types.
1122 * META_FLUSH Make sure the previous pages are written
1123 * with waiting the bio's completion
1124 * ... Only can be used with META.
1125 */
1126#define PAGE_TYPE_OF_BIO(type) ((type) > META ? META : (type))
1127enum page_type {
1128 DATA = 0,
1129 NODE = 1, /* should not change this */
1130 META,
1131 NR_PAGE_TYPE,
1132 META_FLUSH,
1133 IPU, /* the below types are used by tracepoints only. */
1134 OPU,
1135};
1136
1137enum temp_type {
1138 HOT = 0, /* must be zero for meta bio */
1139 WARM,
1140 COLD,
1141 NR_TEMP_TYPE,
1142};
1143
1144enum need_lock_type {
1145 LOCK_REQ = 0,
1146 LOCK_DONE,
1147 LOCK_RETRY,
1148};
1149
1150enum cp_reason_type {
1151 CP_NO_NEEDED,
1152 CP_NON_REGULAR,
1153 CP_COMPRESSED,
1154 CP_HARDLINK,
1155 CP_SB_NEED_CP,
1156 CP_WRONG_PINO,
1157 CP_NO_SPC_ROLL,
1158 CP_NODE_NEED_CP,
1159 CP_FASTBOOT_MODE,
1160 CP_SPEC_LOG_NUM,
1161 CP_RECOVER_DIR,
1162};
1163
1164enum iostat_type {
1165 /* WRITE IO */
1166 APP_DIRECT_IO, /* app direct write IOs */
1167 APP_BUFFERED_IO, /* app buffered write IOs */
1168 APP_WRITE_IO, /* app write IOs */
1169 APP_MAPPED_IO, /* app mapped IOs */
1170 APP_BUFFERED_CDATA_IO, /* app buffered write IOs on compressed file */
1171 APP_MAPPED_CDATA_IO, /* app mapped write IOs on compressed file */
1172 FS_DATA_IO, /* data IOs from kworker/fsync/reclaimer */
1173 FS_CDATA_IO, /* data IOs from kworker/fsync/reclaimer on compressed file */
1174 FS_NODE_IO, /* node IOs from kworker/fsync/reclaimer */
1175 FS_META_IO, /* meta IOs from kworker/reclaimer */
1176 FS_GC_DATA_IO, /* data IOs from forground gc */
1177 FS_GC_NODE_IO, /* node IOs from forground gc */
1178 FS_CP_DATA_IO, /* data IOs from checkpoint */
1179 FS_CP_NODE_IO, /* node IOs from checkpoint */
1180 FS_CP_META_IO, /* meta IOs from checkpoint */
1181
1182 /* READ IO */
1183 APP_DIRECT_READ_IO, /* app direct read IOs */
1184 APP_BUFFERED_READ_IO, /* app buffered read IOs */
1185 APP_READ_IO, /* app read IOs */
1186 APP_MAPPED_READ_IO, /* app mapped read IOs */
1187 APP_BUFFERED_CDATA_READ_IO, /* app buffered read IOs on compressed file */
1188 APP_MAPPED_CDATA_READ_IO, /* app mapped read IOs on compressed file */
1189 FS_DATA_READ_IO, /* data read IOs */
1190 FS_GDATA_READ_IO, /* data read IOs from background gc */
1191 FS_CDATA_READ_IO, /* compressed data read IOs */
1192 FS_NODE_READ_IO, /* node read IOs */
1193 FS_META_READ_IO, /* meta read IOs */
1194
1195 /* other */
1196 FS_DISCARD, /* discard */
1197 NR_IO_TYPE,
1198};
1199
1200struct f2fs_io_info {
1201 struct f2fs_sb_info *sbi; /* f2fs_sb_info pointer */
1202 nid_t ino; /* inode number */
1203 enum page_type type; /* contains DATA/NODE/META/META_FLUSH */
1204 enum temp_type temp; /* contains HOT/WARM/COLD */
1205 enum req_op op; /* contains REQ_OP_ */
1206 blk_opf_t op_flags; /* req_flag_bits */
1207 block_t new_blkaddr; /* new block address to be written */
1208 block_t old_blkaddr; /* old block address before Cow */
1209 struct page *page; /* page to be written */
1210 struct page *encrypted_page; /* encrypted page */
1211 struct page *compressed_page; /* compressed page */
1212 struct list_head list; /* serialize IOs */
1213 bool submitted; /* indicate IO submission */
1214 int need_lock; /* indicate we need to lock cp_rwsem */
1215 bool in_list; /* indicate fio is in io_list */
1216 bool is_por; /* indicate IO is from recovery or not */
1217 bool retry; /* need to reallocate block address */
1218 int compr_blocks; /* # of compressed block addresses */
1219 bool encrypted; /* indicate file is encrypted */
1220 bool post_read; /* require post read */
1221 enum iostat_type io_type; /* io type */
1222 struct writeback_control *io_wbc; /* writeback control */
1223 struct bio **bio; /* bio for ipu */
1224 sector_t *last_block; /* last block number in bio */
1225 unsigned char version; /* version of the node */
1226};
1227
1228struct bio_entry {
1229 struct bio *bio;
1230 struct list_head list;
1231};
1232
1233#define is_read_io(rw) ((rw) == READ)
1234struct f2fs_bio_info {
1235 struct f2fs_sb_info *sbi; /* f2fs superblock */
1236 struct bio *bio; /* bios to merge */
1237 sector_t last_block_in_bio; /* last block number */
1238 struct f2fs_io_info fio; /* store buffered io info. */
1239 struct f2fs_rwsem io_rwsem; /* blocking op for bio */
1240 spinlock_t io_lock; /* serialize DATA/NODE IOs */
1241 struct list_head io_list; /* track fios */
1242 struct list_head bio_list; /* bio entry list head */
1243 struct f2fs_rwsem bio_list_lock; /* lock to protect bio entry list */
1244};
1245
1246#define FDEV(i) (sbi->devs[i])
1247#define RDEV(i) (raw_super->devs[i])
1248struct f2fs_dev_info {
1249 struct block_device *bdev;
1250 char path[MAX_PATH_LEN];
1251 unsigned int total_segments;
1252 block_t start_blk;
1253 block_t end_blk;
1254#ifdef CONFIG_BLK_DEV_ZONED
1255 unsigned int nr_blkz; /* Total number of zones */
1256 unsigned long *blkz_seq; /* Bitmap indicating sequential zones */
1257#endif
1258};
1259
1260enum inode_type {
1261 DIR_INODE, /* for dirty dir inode */
1262 FILE_INODE, /* for dirty regular/symlink inode */
1263 DIRTY_META, /* for all dirtied inode metadata */
1264 NR_INODE_TYPE,
1265};
1266
1267/* for inner inode cache management */
1268struct inode_management {
1269 struct radix_tree_root ino_root; /* ino entry array */
1270 spinlock_t ino_lock; /* for ino entry lock */
1271 struct list_head ino_list; /* inode list head */
1272 unsigned long ino_num; /* number of entries */
1273};
1274
1275/* for GC_AT */
1276struct atgc_management {
1277 bool atgc_enabled; /* ATGC is enabled or not */
1278 struct rb_root_cached root; /* root of victim rb-tree */
1279 struct list_head victim_list; /* linked with all victim entries */
1280 unsigned int victim_count; /* victim count in rb-tree */
1281 unsigned int candidate_ratio; /* candidate ratio */
1282 unsigned int max_candidate_count; /* max candidate count */
1283 unsigned int age_weight; /* age weight, vblock_weight = 100 - age_weight */
1284 unsigned long long age_threshold; /* age threshold */
1285};
1286
1287struct f2fs_gc_control {
1288 unsigned int victim_segno; /* target victim segment number */
1289 int init_gc_type; /* FG_GC or BG_GC */
1290 bool no_bg_gc; /* check the space and stop bg_gc */
1291 bool should_migrate_blocks; /* should migrate blocks */
1292 bool err_gc_skipped; /* return EAGAIN if GC skipped */
1293 unsigned int nr_free_secs; /* # of free sections to do GC */
1294};
1295
1296/* For s_flag in struct f2fs_sb_info */
1297enum {
1298 SBI_IS_DIRTY, /* dirty flag for checkpoint */
1299 SBI_IS_CLOSE, /* specify unmounting */
1300 SBI_NEED_FSCK, /* need fsck.f2fs to fix */
1301 SBI_POR_DOING, /* recovery is doing or not */
1302 SBI_NEED_SB_WRITE, /* need to recover superblock */
1303 SBI_NEED_CP, /* need to checkpoint */
1304 SBI_IS_SHUTDOWN, /* shutdown by ioctl */
1305 SBI_IS_RECOVERED, /* recovered orphan/data */
1306 SBI_CP_DISABLED, /* CP was disabled last mount */
1307 SBI_CP_DISABLED_QUICK, /* CP was disabled quickly */
1308 SBI_QUOTA_NEED_FLUSH, /* need to flush quota info in CP */
1309 SBI_QUOTA_SKIP_FLUSH, /* skip flushing quota in current CP */
1310 SBI_QUOTA_NEED_REPAIR, /* quota file may be corrupted */
1311 SBI_IS_RESIZEFS, /* resizefs is in process */
1312 SBI_IS_FREEZING, /* freezefs is in process */
1313};
1314
1315enum {
1316 CP_TIME,
1317 REQ_TIME,
1318 DISCARD_TIME,
1319 GC_TIME,
1320 DISABLE_TIME,
1321 UMOUNT_DISCARD_TIMEOUT,
1322 MAX_TIME,
1323};
1324
1325/* Note that you need to keep synchronization with this gc_mode_names array */
1326enum {
1327 GC_NORMAL,
1328 GC_IDLE_CB,
1329 GC_IDLE_GREEDY,
1330 GC_IDLE_AT,
1331 GC_URGENT_HIGH,
1332 GC_URGENT_LOW,
1333 GC_URGENT_MID,
1334 MAX_GC_MODE,
1335};
1336
1337enum {
1338 BGGC_MODE_ON, /* background gc is on */
1339 BGGC_MODE_OFF, /* background gc is off */
1340 BGGC_MODE_SYNC, /*
1341 * background gc is on, migrating blocks
1342 * like foreground gc
1343 */
1344};
1345
1346enum {
1347 FS_MODE_ADAPTIVE, /* use both lfs/ssr allocation */
1348 FS_MODE_LFS, /* use lfs allocation only */
1349 FS_MODE_FRAGMENT_SEG, /* segment fragmentation mode */
1350 FS_MODE_FRAGMENT_BLK, /* block fragmentation mode */
1351};
1352
1353enum {
1354 ALLOC_MODE_DEFAULT, /* stay default */
1355 ALLOC_MODE_REUSE, /* reuse segments as much as possible */
1356};
1357
1358enum fsync_mode {
1359 FSYNC_MODE_POSIX, /* fsync follows posix semantics */
1360 FSYNC_MODE_STRICT, /* fsync behaves in line with ext4 */
1361 FSYNC_MODE_NOBARRIER, /* fsync behaves nobarrier based on posix */
1362};
1363
1364enum {
1365 COMPR_MODE_FS, /*
1366 * automatically compress compression
1367 * enabled files
1368 */
1369 COMPR_MODE_USER, /*
1370 * automatical compression is disabled.
1371 * user can control the file compression
1372 * using ioctls
1373 */
1374};
1375
1376enum {
1377 DISCARD_UNIT_BLOCK, /* basic discard unit is block */
1378 DISCARD_UNIT_SEGMENT, /* basic discard unit is segment */
1379 DISCARD_UNIT_SECTION, /* basic discard unit is section */
1380};
1381
1382enum {
1383 MEMORY_MODE_NORMAL, /* memory mode for normal devices */
1384 MEMORY_MODE_LOW, /* memory mode for low memry devices */
1385};
1386
1387
1388
1389static inline int f2fs_test_bit(unsigned int nr, char *addr);
1390static inline void f2fs_set_bit(unsigned int nr, char *addr);
1391static inline void f2fs_clear_bit(unsigned int nr, char *addr);
1392
1393/*
1394 * Layout of f2fs page.private:
1395 *
1396 * Layout A: lowest bit should be 1
1397 * | bit0 = 1 | bit1 | bit2 | ... | bit MAX | private data .... |
1398 * bit 0 PAGE_PRIVATE_NOT_POINTER
1399 * bit 1 PAGE_PRIVATE_ATOMIC_WRITE
1400 * bit 2 PAGE_PRIVATE_DUMMY_WRITE
1401 * bit 3 PAGE_PRIVATE_ONGOING_MIGRATION
1402 * bit 4 PAGE_PRIVATE_INLINE_INODE
1403 * bit 5 PAGE_PRIVATE_REF_RESOURCE
1404 * bit 6- f2fs private data
1405 *
1406 * Layout B: lowest bit should be 0
1407 * page.private is a wrapped pointer.
1408 */
1409enum {
1410 PAGE_PRIVATE_NOT_POINTER, /* private contains non-pointer data */
1411 PAGE_PRIVATE_ATOMIC_WRITE, /* data page from atomic write path */
1412 PAGE_PRIVATE_DUMMY_WRITE, /* data page for padding aligned IO */
1413 PAGE_PRIVATE_ONGOING_MIGRATION, /* data page which is on-going migrating */
1414 PAGE_PRIVATE_INLINE_INODE, /* inode page contains inline data */
1415 PAGE_PRIVATE_REF_RESOURCE, /* dirty page has referenced resources */
1416 PAGE_PRIVATE_MAX
1417};
1418
1419#define PAGE_PRIVATE_GET_FUNC(name, flagname) \
1420static inline bool page_private_##name(struct page *page) \
1421{ \
1422 return PagePrivate(page) && \
1423 test_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)) && \
1424 test_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
1425}
1426
1427#define PAGE_PRIVATE_SET_FUNC(name, flagname) \
1428static inline void set_page_private_##name(struct page *page) \
1429{ \
1430 if (!PagePrivate(page)) { \
1431 get_page(page); \
1432 SetPagePrivate(page); \
1433 set_page_private(page, 0); \
1434 } \
1435 set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)); \
1436 set_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
1437}
1438
1439#define PAGE_PRIVATE_CLEAR_FUNC(name, flagname) \
1440static inline void clear_page_private_##name(struct page *page) \
1441{ \
1442 clear_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
1443 if (page_private(page) == 1 << PAGE_PRIVATE_NOT_POINTER) { \
1444 set_page_private(page, 0); \
1445 if (PagePrivate(page)) { \
1446 ClearPagePrivate(page); \
1447 put_page(page); \
1448 }\
1449 } \
1450}
1451
1452PAGE_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER);
1453PAGE_PRIVATE_GET_FUNC(reference, REF_RESOURCE);
1454PAGE_PRIVATE_GET_FUNC(inline, INLINE_INODE);
1455PAGE_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION);
1456PAGE_PRIVATE_GET_FUNC(atomic, ATOMIC_WRITE);
1457PAGE_PRIVATE_GET_FUNC(dummy, DUMMY_WRITE);
1458
1459PAGE_PRIVATE_SET_FUNC(reference, REF_RESOURCE);
1460PAGE_PRIVATE_SET_FUNC(inline, INLINE_INODE);
1461PAGE_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION);
1462PAGE_PRIVATE_SET_FUNC(atomic, ATOMIC_WRITE);
1463PAGE_PRIVATE_SET_FUNC(dummy, DUMMY_WRITE);
1464
1465PAGE_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE);
1466PAGE_PRIVATE_CLEAR_FUNC(inline, INLINE_INODE);
1467PAGE_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION);
1468PAGE_PRIVATE_CLEAR_FUNC(atomic, ATOMIC_WRITE);
1469PAGE_PRIVATE_CLEAR_FUNC(dummy, DUMMY_WRITE);
1470
1471static inline unsigned long get_page_private_data(struct page *page)
1472{
1473 unsigned long data = page_private(page);
1474
1475 if (!test_bit(PAGE_PRIVATE_NOT_POINTER, &data))
1476 return 0;
1477 return data >> PAGE_PRIVATE_MAX;
1478}
1479
1480static inline void set_page_private_data(struct page *page, unsigned long data)
1481{
1482 if (!PagePrivate(page)) {
1483 get_page(page);
1484 SetPagePrivate(page);
1485 set_page_private(page, 0);
1486 }
1487 set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page));
1488 page_private(page) |= data << PAGE_PRIVATE_MAX;
1489}
1490
1491static inline void clear_page_private_data(struct page *page)
1492{
1493 page_private(page) &= (1 << PAGE_PRIVATE_MAX) - 1;
1494 if (page_private(page) == 1 << PAGE_PRIVATE_NOT_POINTER) {
1495 set_page_private(page, 0);
1496 if (PagePrivate(page)) {
1497 ClearPagePrivate(page);
1498 put_page(page);
1499 }
1500 }
1501}
1502
1503/* For compression */
1504enum compress_algorithm_type {
1505 COMPRESS_LZO,
1506 COMPRESS_LZ4,
1507 COMPRESS_ZSTD,
1508 COMPRESS_LZORLE,
1509 COMPRESS_MAX,
1510};
1511
1512enum compress_flag {
1513 COMPRESS_CHKSUM,
1514 COMPRESS_MAX_FLAG,
1515};
1516
1517#define COMPRESS_WATERMARK 20
1518#define COMPRESS_PERCENT 20
1519
1520#define COMPRESS_DATA_RESERVED_SIZE 4
1521struct compress_data {
1522 __le32 clen; /* compressed data size */
1523 __le32 chksum; /* compressed data chksum */
1524 __le32 reserved[COMPRESS_DATA_RESERVED_SIZE]; /* reserved */
1525 u8 cdata[]; /* compressed data */
1526};
1527
1528#define COMPRESS_HEADER_SIZE (sizeof(struct compress_data))
1529
1530#define F2FS_COMPRESSED_PAGE_MAGIC 0xF5F2C000
1531
1532#define COMPRESS_LEVEL_OFFSET 8
1533
1534/* compress context */
1535struct compress_ctx {
1536 struct inode *inode; /* inode the context belong to */
1537 pgoff_t cluster_idx; /* cluster index number */
1538 unsigned int cluster_size; /* page count in cluster */
1539 unsigned int log_cluster_size; /* log of cluster size */
1540 struct page **rpages; /* pages store raw data in cluster */
1541 unsigned int nr_rpages; /* total page number in rpages */
1542 struct page **cpages; /* pages store compressed data in cluster */
1543 unsigned int nr_cpages; /* total page number in cpages */
1544 unsigned int valid_nr_cpages; /* valid page number in cpages */
1545 void *rbuf; /* virtual mapped address on rpages */
1546 struct compress_data *cbuf; /* virtual mapped address on cpages */
1547 size_t rlen; /* valid data length in rbuf */
1548 size_t clen; /* valid data length in cbuf */
1549 void *private; /* payload buffer for specified compression algorithm */
1550 void *private2; /* extra payload buffer */
1551};
1552
1553/* compress context for write IO path */
1554struct compress_io_ctx {
1555 u32 magic; /* magic number to indicate page is compressed */
1556 struct inode *inode; /* inode the context belong to */
1557 struct page **rpages; /* pages store raw data in cluster */
1558 unsigned int nr_rpages; /* total page number in rpages */
1559 atomic_t pending_pages; /* in-flight compressed page count */
1560};
1561
1562/* Context for decompressing one cluster on the read IO path */
1563struct decompress_io_ctx {
1564 u32 magic; /* magic number to indicate page is compressed */
1565 struct inode *inode; /* inode the context belong to */
1566 pgoff_t cluster_idx; /* cluster index number */
1567 unsigned int cluster_size; /* page count in cluster */
1568 unsigned int log_cluster_size; /* log of cluster size */
1569 struct page **rpages; /* pages store raw data in cluster */
1570 unsigned int nr_rpages; /* total page number in rpages */
1571 struct page **cpages; /* pages store compressed data in cluster */
1572 unsigned int nr_cpages; /* total page number in cpages */
1573 struct page **tpages; /* temp pages to pad holes in cluster */
1574 void *rbuf; /* virtual mapped address on rpages */
1575 struct compress_data *cbuf; /* virtual mapped address on cpages */
1576 size_t rlen; /* valid data length in rbuf */
1577 size_t clen; /* valid data length in cbuf */
1578
1579 /*
1580 * The number of compressed pages remaining to be read in this cluster.
1581 * This is initially nr_cpages. It is decremented by 1 each time a page
1582 * has been read (or failed to be read). When it reaches 0, the cluster
1583 * is decompressed (or an error is reported).
1584 *
1585 * If an error occurs before all the pages have been submitted for I/O,
1586 * then this will never reach 0. In this case the I/O submitter is
1587 * responsible for calling f2fs_decompress_end_io() instead.
1588 */
1589 atomic_t remaining_pages;
1590
1591 /*
1592 * Number of references to this decompress_io_ctx.
1593 *
1594 * One reference is held for I/O completion. This reference is dropped
1595 * after the pagecache pages are updated and unlocked -- either after
1596 * decompression (and verity if enabled), or after an error.
1597 *
1598 * In addition, each compressed page holds a reference while it is in a
1599 * bio. These references are necessary prevent compressed pages from
1600 * being freed while they are still in a bio.
1601 */
1602 refcount_t refcnt;
1603
1604 bool failed; /* IO error occurred before decompression? */
1605 bool need_verity; /* need fs-verity verification after decompression? */
1606 void *private; /* payload buffer for specified decompression algorithm */
1607 void *private2; /* extra payload buffer */
1608 struct work_struct verity_work; /* work to verify the decompressed pages */
1609 struct work_struct free_work; /* work for late free this structure itself */
1610};
1611
1612#define NULL_CLUSTER ((unsigned int)(~0))
1613#define MIN_COMPRESS_LOG_SIZE 2
1614#define MAX_COMPRESS_LOG_SIZE 8
1615#define MAX_COMPRESS_WINDOW_SIZE(log_size) ((PAGE_SIZE) << (log_size))
1616
1617struct f2fs_sb_info {
1618 struct super_block *sb; /* pointer to VFS super block */
1619 struct proc_dir_entry *s_proc; /* proc entry */
1620 struct f2fs_super_block *raw_super; /* raw super block pointer */
1621 struct f2fs_rwsem sb_lock; /* lock for raw super block */
1622 int valid_super_block; /* valid super block no */
1623 unsigned long s_flag; /* flags for sbi */
1624 struct mutex writepages; /* mutex for writepages() */
1625
1626#ifdef CONFIG_BLK_DEV_ZONED
1627 unsigned int blocks_per_blkz; /* F2FS blocks per zone */
1628 unsigned int log_blocks_per_blkz; /* log2 F2FS blocks per zone */
1629#endif
1630
1631 /* for node-related operations */
1632 struct f2fs_nm_info *nm_info; /* node manager */
1633 struct inode *node_inode; /* cache node blocks */
1634
1635 /* for segment-related operations */
1636 struct f2fs_sm_info *sm_info; /* segment manager */
1637
1638 /* for bio operations */
1639 struct f2fs_bio_info *write_io[NR_PAGE_TYPE]; /* for write bios */
1640 /* keep migration IO order for LFS mode */
1641 struct f2fs_rwsem io_order_lock;
1642 mempool_t *write_io_dummy; /* Dummy pages */
1643 pgoff_t page_eio_ofs[NR_PAGE_TYPE]; /* EIO page offset */
1644 int page_eio_cnt[NR_PAGE_TYPE]; /* EIO count */
1645
1646 /* for checkpoint */
1647 struct f2fs_checkpoint *ckpt; /* raw checkpoint pointer */
1648 int cur_cp_pack; /* remain current cp pack */
1649 spinlock_t cp_lock; /* for flag in ckpt */
1650 struct inode *meta_inode; /* cache meta blocks */
1651 struct f2fs_rwsem cp_global_sem; /* checkpoint procedure lock */
1652 struct f2fs_rwsem cp_rwsem; /* blocking FS operations */
1653 struct f2fs_rwsem node_write; /* locking node writes */
1654 struct f2fs_rwsem node_change; /* locking node change */
1655 wait_queue_head_t cp_wait;
1656 unsigned long last_time[MAX_TIME]; /* to store time in jiffies */
1657 long interval_time[MAX_TIME]; /* to store thresholds */
1658 struct ckpt_req_control cprc_info; /* for checkpoint request control */
1659
1660 struct inode_management im[MAX_INO_ENTRY]; /* manage inode cache */
1661
1662 spinlock_t fsync_node_lock; /* for node entry lock */
1663 struct list_head fsync_node_list; /* node list head */
1664 unsigned int fsync_seg_id; /* sequence id */
1665 unsigned int fsync_node_num; /* number of node entries */
1666
1667 /* for orphan inode, use 0'th array */
1668 unsigned int max_orphans; /* max orphan inodes */
1669
1670 /* for inode management */
1671 struct list_head inode_list[NR_INODE_TYPE]; /* dirty inode list */
1672 spinlock_t inode_lock[NR_INODE_TYPE]; /* for dirty inode list lock */
1673 struct mutex flush_lock; /* for flush exclusion */
1674
1675 /* for extent tree cache */
1676 struct extent_tree_info extent_tree[NR_EXTENT_CACHES];
1677 atomic64_t allocated_data_blocks; /* for block age extent_cache */
1678
1679 /* The threshold used for hot and warm data seperation*/
1680 unsigned int hot_data_age_threshold;
1681 unsigned int warm_data_age_threshold;
1682
1683 /* basic filesystem units */
1684 unsigned int log_sectors_per_block; /* log2 sectors per block */
1685 unsigned int log_blocksize; /* log2 block size */
1686 unsigned int blocksize; /* block size */
1687 unsigned int root_ino_num; /* root inode number*/
1688 unsigned int node_ino_num; /* node inode number*/
1689 unsigned int meta_ino_num; /* meta inode number*/
1690 unsigned int log_blocks_per_seg; /* log2 blocks per segment */
1691 unsigned int blocks_per_seg; /* blocks per segment */
1692 unsigned int unusable_blocks_per_sec; /* unusable blocks per section */
1693 unsigned int segs_per_sec; /* segments per section */
1694 unsigned int secs_per_zone; /* sections per zone */
1695 unsigned int total_sections; /* total section count */
1696 unsigned int total_node_count; /* total node block count */
1697 unsigned int total_valid_node_count; /* valid node block count */
1698 int dir_level; /* directory level */
1699 bool readdir_ra; /* readahead inode in readdir */
1700 u64 max_io_bytes; /* max io bytes to merge IOs */
1701
1702 block_t user_block_count; /* # of user blocks */
1703 block_t total_valid_block_count; /* # of valid blocks */
1704 block_t discard_blks; /* discard command candidats */
1705 block_t last_valid_block_count; /* for recovery */
1706 block_t reserved_blocks; /* configurable reserved blocks */
1707 block_t current_reserved_blocks; /* current reserved blocks */
1708
1709 /* Additional tracking for no checkpoint mode */
1710 block_t unusable_block_count; /* # of blocks saved by last cp */
1711
1712 unsigned int nquota_files; /* # of quota sysfile */
1713 struct f2fs_rwsem quota_sem; /* blocking cp for flags */
1714
1715 /* # of pages, see count_type */
1716 atomic_t nr_pages[NR_COUNT_TYPE];
1717 /* # of allocated blocks */
1718 struct percpu_counter alloc_valid_block_count;
1719 /* # of node block writes as roll forward recovery */
1720 struct percpu_counter rf_node_block_count;
1721
1722 /* writeback control */
1723 atomic_t wb_sync_req[META]; /* count # of WB_SYNC threads */
1724
1725 /* valid inode count */
1726 struct percpu_counter total_valid_inode_count;
1727
1728 struct f2fs_mount_info mount_opt; /* mount options */
1729
1730 /* for cleaning operations */
1731 struct f2fs_rwsem gc_lock; /*
1732 * semaphore for GC, avoid
1733 * race between GC and GC or CP
1734 */
1735 struct f2fs_gc_kthread *gc_thread; /* GC thread */
1736 struct atgc_management am; /* atgc management */
1737 unsigned int cur_victim_sec; /* current victim section num */
1738 unsigned int gc_mode; /* current GC state */
1739 unsigned int next_victim_seg[2]; /* next segment in victim section */
1740 spinlock_t gc_remaining_trials_lock;
1741 /* remaining trial count for GC_URGENT_* and GC_IDLE_* */
1742 unsigned int gc_remaining_trials;
1743
1744 /* for skip statistic */
1745 unsigned long long skipped_gc_rwsem; /* FG_GC only */
1746
1747 /* threshold for gc trials on pinned files */
1748 u64 gc_pin_file_threshold;
1749 struct f2fs_rwsem pin_sem;
1750
1751 /* maximum # of trials to find a victim segment for SSR and GC */
1752 unsigned int max_victim_search;
1753 /* migration granularity of garbage collection, unit: segment */
1754 unsigned int migration_granularity;
1755
1756 /*
1757 * for stat information.
1758 * one is for the LFS mode, and the other is for the SSR mode.
1759 */
1760#ifdef CONFIG_F2FS_STAT_FS
1761 struct f2fs_stat_info *stat_info; /* FS status information */
1762 atomic_t meta_count[META_MAX]; /* # of meta blocks */
1763 unsigned int segment_count[2]; /* # of allocated segments */
1764 unsigned int block_count[2]; /* # of allocated blocks */
1765 atomic_t inplace_count; /* # of inplace update */
1766 /* # of lookup extent cache */
1767 atomic64_t total_hit_ext[NR_EXTENT_CACHES];
1768 /* # of hit rbtree extent node */
1769 atomic64_t read_hit_rbtree[NR_EXTENT_CACHES];
1770 /* # of hit cached extent node */
1771 atomic64_t read_hit_cached[NR_EXTENT_CACHES];
1772 /* # of hit largest extent node in read extent cache */
1773 atomic64_t read_hit_largest;
1774 atomic_t inline_xattr; /* # of inline_xattr inodes */
1775 atomic_t inline_inode; /* # of inline_data inodes */
1776 atomic_t inline_dir; /* # of inline_dentry inodes */
1777 atomic_t compr_inode; /* # of compressed inodes */
1778 atomic64_t compr_blocks; /* # of compressed blocks */
1779 atomic_t swapfile_inode; /* # of swapfile inodes */
1780 atomic_t atomic_files; /* # of opened atomic file */
1781 atomic_t max_aw_cnt; /* max # of atomic writes */
1782 unsigned int io_skip_bggc; /* skip background gc for in-flight IO */
1783 unsigned int other_skip_bggc; /* skip background gc for other reasons */
1784 unsigned int ndirty_inode[NR_INODE_TYPE]; /* # of dirty inodes */
1785#endif
1786 spinlock_t stat_lock; /* lock for stat operations */
1787
1788 /* to attach REQ_META|REQ_FUA flags */
1789 unsigned int data_io_flag;
1790 unsigned int node_io_flag;
1791
1792 /* For sysfs support */
1793 struct kobject s_kobj; /* /sys/fs/f2fs/<devname> */
1794 struct completion s_kobj_unregister;
1795
1796 struct kobject s_stat_kobj; /* /sys/fs/f2fs/<devname>/stat */
1797 struct completion s_stat_kobj_unregister;
1798
1799 struct kobject s_feature_list_kobj; /* /sys/fs/f2fs/<devname>/feature_list */
1800 struct completion s_feature_list_kobj_unregister;
1801
1802 /* For shrinker support */
1803 struct list_head s_list;
1804 struct mutex umount_mutex;
1805 unsigned int shrinker_run_no;
1806
1807 /* For multi devices */
1808 int s_ndevs; /* number of devices */
1809 struct f2fs_dev_info *devs; /* for device list */
1810 unsigned int dirty_device; /* for checkpoint data flush */
1811 spinlock_t dev_lock; /* protect dirty_device */
1812 bool aligned_blksize; /* all devices has the same logical blksize */
1813
1814 /* For write statistics */
1815 u64 sectors_written_start;
1816 u64 kbytes_written;
1817
1818 /* Reference to checksum algorithm driver via cryptoapi */
1819 struct crypto_shash *s_chksum_driver;
1820
1821 /* Precomputed FS UUID checksum for seeding other checksums */
1822 __u32 s_chksum_seed;
1823
1824 struct workqueue_struct *post_read_wq; /* post read workqueue */
1825
1826 unsigned char errors[MAX_F2FS_ERRORS]; /* error flags */
1827 spinlock_t error_lock; /* protect errors array */
1828 bool error_dirty; /* errors of sb is dirty */
1829
1830 struct kmem_cache *inline_xattr_slab; /* inline xattr entry */
1831 unsigned int inline_xattr_slab_size; /* default inline xattr slab size */
1832
1833 /* For reclaimed segs statistics per each GC mode */
1834 unsigned int gc_segment_mode; /* GC state for reclaimed segments */
1835 unsigned int gc_reclaimed_segs[MAX_GC_MODE]; /* Reclaimed segs for each mode */
1836
1837 unsigned long seq_file_ra_mul; /* multiplier for ra_pages of seq. files in fadvise */
1838
1839 int max_fragment_chunk; /* max chunk size for block fragmentation mode */
1840 int max_fragment_hole; /* max hole size for block fragmentation mode */
1841
1842 /* For atomic write statistics */
1843 atomic64_t current_atomic_write;
1844 s64 peak_atomic_write;
1845 u64 committed_atomic_block;
1846 u64 revoked_atomic_block;
1847
1848#ifdef CONFIG_F2FS_FS_COMPRESSION
1849 struct kmem_cache *page_array_slab; /* page array entry */
1850 unsigned int page_array_slab_size; /* default page array slab size */
1851
1852 /* For runtime compression statistics */
1853 u64 compr_written_block;
1854 u64 compr_saved_block;
1855 u32 compr_new_inode;
1856
1857 /* For compressed block cache */
1858 struct inode *compress_inode; /* cache compressed blocks */
1859 unsigned int compress_percent; /* cache page percentage */
1860 unsigned int compress_watermark; /* cache page watermark */
1861 atomic_t compress_page_hit; /* cache hit count */
1862#endif
1863
1864#ifdef CONFIG_F2FS_IOSTAT
1865 /* For app/fs IO statistics */
1866 spinlock_t iostat_lock;
1867 unsigned long long rw_iostat[NR_IO_TYPE];
1868 unsigned long long prev_rw_iostat[NR_IO_TYPE];
1869 bool iostat_enable;
1870 unsigned long iostat_next_period;
1871 unsigned int iostat_period_ms;
1872
1873 /* For io latency related statistics info in one iostat period */
1874 spinlock_t iostat_lat_lock;
1875 struct iostat_lat_info *iostat_io_lat;
1876#endif
1877};
1878
1879#ifdef CONFIG_F2FS_FAULT_INJECTION
1880#define f2fs_show_injection_info(sbi, type) \
1881 printk_ratelimited("%sF2FS-fs (%s) : inject %s in %s of %pS\n", \
1882 KERN_INFO, sbi->sb->s_id, \
1883 f2fs_fault_name[type], \
1884 __func__, __builtin_return_address(0))
1885static inline bool time_to_inject(struct f2fs_sb_info *sbi, int type)
1886{
1887 struct f2fs_fault_info *ffi = &F2FS_OPTION(sbi).fault_info;
1888
1889 if (!ffi->inject_rate)
1890 return false;
1891
1892 if (!IS_FAULT_SET(ffi, type))
1893 return false;
1894
1895 atomic_inc(&ffi->inject_ops);
1896 if (atomic_read(&ffi->inject_ops) >= ffi->inject_rate) {
1897 atomic_set(&ffi->inject_ops, 0);
1898 return true;
1899 }
1900 return false;
1901}
1902#else
1903#define f2fs_show_injection_info(sbi, type) do { } while (0)
1904static inline bool time_to_inject(struct f2fs_sb_info *sbi, int type)
1905{
1906 return false;
1907}
1908#endif
1909
1910/*
1911 * Test if the mounted volume is a multi-device volume.
1912 * - For a single regular disk volume, sbi->s_ndevs is 0.
1913 * - For a single zoned disk volume, sbi->s_ndevs is 1.
1914 * - For a multi-device volume, sbi->s_ndevs is always 2 or more.
1915 */
1916static inline bool f2fs_is_multi_device(struct f2fs_sb_info *sbi)
1917{
1918 return sbi->s_ndevs > 1;
1919}
1920
1921static inline void f2fs_update_time(struct f2fs_sb_info *sbi, int type)
1922{
1923 unsigned long now = jiffies;
1924
1925 sbi->last_time[type] = now;
1926
1927 /* DISCARD_TIME and GC_TIME are based on REQ_TIME */
1928 if (type == REQ_TIME) {
1929 sbi->last_time[DISCARD_TIME] = now;
1930 sbi->last_time[GC_TIME] = now;
1931 }
1932}
1933
1934static inline bool f2fs_time_over(struct f2fs_sb_info *sbi, int type)
1935{
1936 unsigned long interval = sbi->interval_time[type] * HZ;
1937
1938 return time_after(jiffies, sbi->last_time[type] + interval);
1939}
1940
1941static inline unsigned int f2fs_time_to_wait(struct f2fs_sb_info *sbi,
1942 int type)
1943{
1944 unsigned long interval = sbi->interval_time[type] * HZ;
1945 unsigned int wait_ms = 0;
1946 long delta;
1947
1948 delta = (sbi->last_time[type] + interval) - jiffies;
1949 if (delta > 0)
1950 wait_ms = jiffies_to_msecs(delta);
1951
1952 return wait_ms;
1953}
1954
1955/*
1956 * Inline functions
1957 */
1958static inline u32 __f2fs_crc32(struct f2fs_sb_info *sbi, u32 crc,
1959 const void *address, unsigned int length)
1960{
1961 struct {
1962 struct shash_desc shash;
1963 char ctx[4];
1964 } desc;
1965 int err;
1966
1967 BUG_ON(crypto_shash_descsize(sbi->s_chksum_driver) != sizeof(desc.ctx));
1968
1969 desc.shash.tfm = sbi->s_chksum_driver;
1970 *(u32 *)desc.ctx = crc;
1971
1972 err = crypto_shash_update(&desc.shash, address, length);
1973 BUG_ON(err);
1974
1975 return *(u32 *)desc.ctx;
1976}
1977
1978static inline u32 f2fs_crc32(struct f2fs_sb_info *sbi, const void *address,
1979 unsigned int length)
1980{
1981 return __f2fs_crc32(sbi, F2FS_SUPER_MAGIC, address, length);
1982}
1983
1984static inline bool f2fs_crc_valid(struct f2fs_sb_info *sbi, __u32 blk_crc,
1985 void *buf, size_t buf_size)
1986{
1987 return f2fs_crc32(sbi, buf, buf_size) == blk_crc;
1988}
1989
1990static inline u32 f2fs_chksum(struct f2fs_sb_info *sbi, u32 crc,
1991 const void *address, unsigned int length)
1992{
1993 return __f2fs_crc32(sbi, crc, address, length);
1994}
1995
1996static inline struct f2fs_inode_info *F2FS_I(struct inode *inode)
1997{
1998 return container_of(inode, struct f2fs_inode_info, vfs_inode);
1999}
2000
2001static inline struct f2fs_sb_info *F2FS_SB(struct super_block *sb)
2002{
2003 return sb->s_fs_info;
2004}
2005
2006static inline struct f2fs_sb_info *F2FS_I_SB(struct inode *inode)
2007{
2008 return F2FS_SB(inode->i_sb);
2009}
2010
2011static inline struct f2fs_sb_info *F2FS_M_SB(struct address_space *mapping)
2012{
2013 return F2FS_I_SB(mapping->host);
2014}
2015
2016static inline struct f2fs_sb_info *F2FS_P_SB(struct page *page)
2017{
2018 return F2FS_M_SB(page_file_mapping(page));
2019}
2020
2021static inline struct f2fs_super_block *F2FS_RAW_SUPER(struct f2fs_sb_info *sbi)
2022{
2023 return (struct f2fs_super_block *)(sbi->raw_super);
2024}
2025
2026static inline struct f2fs_checkpoint *F2FS_CKPT(struct f2fs_sb_info *sbi)
2027{
2028 return (struct f2fs_checkpoint *)(sbi->ckpt);
2029}
2030
2031static inline struct f2fs_node *F2FS_NODE(struct page *page)
2032{
2033 return (struct f2fs_node *)page_address(page);
2034}
2035
2036static inline struct f2fs_inode *F2FS_INODE(struct page *page)
2037{
2038 return &((struct f2fs_node *)page_address(page))->i;
2039}
2040
2041static inline struct f2fs_nm_info *NM_I(struct f2fs_sb_info *sbi)
2042{
2043 return (struct f2fs_nm_info *)(sbi->nm_info);
2044}
2045
2046static inline struct f2fs_sm_info *SM_I(struct f2fs_sb_info *sbi)
2047{
2048 return (struct f2fs_sm_info *)(sbi->sm_info);
2049}
2050
2051static inline struct sit_info *SIT_I(struct f2fs_sb_info *sbi)
2052{
2053 return (struct sit_info *)(SM_I(sbi)->sit_info);
2054}
2055
2056static inline struct free_segmap_info *FREE_I(struct f2fs_sb_info *sbi)
2057{
2058 return (struct free_segmap_info *)(SM_I(sbi)->free_info);
2059}
2060
2061static inline struct dirty_seglist_info *DIRTY_I(struct f2fs_sb_info *sbi)
2062{
2063 return (struct dirty_seglist_info *)(SM_I(sbi)->dirty_info);
2064}
2065
2066static inline struct address_space *META_MAPPING(struct f2fs_sb_info *sbi)
2067{
2068 return sbi->meta_inode->i_mapping;
2069}
2070
2071static inline struct address_space *NODE_MAPPING(struct f2fs_sb_info *sbi)
2072{
2073 return sbi->node_inode->i_mapping;
2074}
2075
2076static inline bool is_sbi_flag_set(struct f2fs_sb_info *sbi, unsigned int type)
2077{
2078 return test_bit(type, &sbi->s_flag);
2079}
2080
2081static inline void set_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type)
2082{
2083 set_bit(type, &sbi->s_flag);
2084}
2085
2086static inline void clear_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type)
2087{
2088 clear_bit(type, &sbi->s_flag);
2089}
2090
2091static inline unsigned long long cur_cp_version(struct f2fs_checkpoint *cp)
2092{
2093 return le64_to_cpu(cp->checkpoint_ver);
2094}
2095
2096static inline unsigned long f2fs_qf_ino(struct super_block *sb, int type)
2097{
2098 if (type < F2FS_MAX_QUOTAS)
2099 return le32_to_cpu(F2FS_SB(sb)->raw_super->qf_ino[type]);
2100 return 0;
2101}
2102
2103static inline __u64 cur_cp_crc(struct f2fs_checkpoint *cp)
2104{
2105 size_t crc_offset = le32_to_cpu(cp->checksum_offset);
2106 return le32_to_cpu(*((__le32 *)((unsigned char *)cp + crc_offset)));
2107}
2108
2109static inline bool __is_set_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
2110{
2111 unsigned int ckpt_flags = le32_to_cpu(cp->ckpt_flags);
2112
2113 return ckpt_flags & f;
2114}
2115
2116static inline bool is_set_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
2117{
2118 return __is_set_ckpt_flags(F2FS_CKPT(sbi), f);
2119}
2120
2121static inline void __set_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
2122{
2123 unsigned int ckpt_flags;
2124
2125 ckpt_flags = le32_to_cpu(cp->ckpt_flags);
2126 ckpt_flags |= f;
2127 cp->ckpt_flags = cpu_to_le32(ckpt_flags);
2128}
2129
2130static inline void set_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
2131{
2132 unsigned long flags;
2133
2134 spin_lock_irqsave(&sbi->cp_lock, flags);
2135 __set_ckpt_flags(F2FS_CKPT(sbi), f);
2136 spin_unlock_irqrestore(&sbi->cp_lock, flags);
2137}
2138
2139static inline void __clear_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
2140{
2141 unsigned int ckpt_flags;
2142
2143 ckpt_flags = le32_to_cpu(cp->ckpt_flags);
2144 ckpt_flags &= (~f);
2145 cp->ckpt_flags = cpu_to_le32(ckpt_flags);
2146}
2147
2148static inline void clear_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
2149{
2150 unsigned long flags;
2151
2152 spin_lock_irqsave(&sbi->cp_lock, flags);
2153 __clear_ckpt_flags(F2FS_CKPT(sbi), f);
2154 spin_unlock_irqrestore(&sbi->cp_lock, flags);
2155}
2156
2157#define init_f2fs_rwsem(sem) \
2158do { \
2159 static struct lock_class_key __key; \
2160 \
2161 __init_f2fs_rwsem((sem), #sem, &__key); \
2162} while (0)
2163
2164static inline void __init_f2fs_rwsem(struct f2fs_rwsem *sem,
2165 const char *sem_name, struct lock_class_key *key)
2166{
2167 __init_rwsem(&sem->internal_rwsem, sem_name, key);
2168#ifdef CONFIG_F2FS_UNFAIR_RWSEM
2169 init_waitqueue_head(&sem->read_waiters);
2170#endif
2171}
2172
2173static inline int f2fs_rwsem_is_locked(struct f2fs_rwsem *sem)
2174{
2175 return rwsem_is_locked(&sem->internal_rwsem);
2176}
2177
2178static inline int f2fs_rwsem_is_contended(struct f2fs_rwsem *sem)
2179{
2180 return rwsem_is_contended(&sem->internal_rwsem);
2181}
2182
2183static inline void f2fs_down_read(struct f2fs_rwsem *sem)
2184{
2185#ifdef CONFIG_F2FS_UNFAIR_RWSEM
2186 wait_event(sem->read_waiters, down_read_trylock(&sem->internal_rwsem));
2187#else
2188 down_read(&sem->internal_rwsem);
2189#endif
2190}
2191
2192static inline int f2fs_down_read_trylock(struct f2fs_rwsem *sem)
2193{
2194 return down_read_trylock(&sem->internal_rwsem);
2195}
2196
2197#ifdef CONFIG_DEBUG_LOCK_ALLOC
2198static inline void f2fs_down_read_nested(struct f2fs_rwsem *sem, int subclass)
2199{
2200 down_read_nested(&sem->internal_rwsem, subclass);
2201}
2202#else
2203#define f2fs_down_read_nested(sem, subclass) f2fs_down_read(sem)
2204#endif
2205
2206static inline void f2fs_up_read(struct f2fs_rwsem *sem)
2207{
2208 up_read(&sem->internal_rwsem);
2209}
2210
2211static inline void f2fs_down_write(struct f2fs_rwsem *sem)
2212{
2213 down_write(&sem->internal_rwsem);
2214}
2215
2216static inline int f2fs_down_write_trylock(struct f2fs_rwsem *sem)
2217{
2218 return down_write_trylock(&sem->internal_rwsem);
2219}
2220
2221static inline void f2fs_up_write(struct f2fs_rwsem *sem)
2222{
2223 up_write(&sem->internal_rwsem);
2224#ifdef CONFIG_F2FS_UNFAIR_RWSEM
2225 wake_up_all(&sem->read_waiters);
2226#endif
2227}
2228
2229static inline void f2fs_lock_op(struct f2fs_sb_info *sbi)
2230{
2231 f2fs_down_read(&sbi->cp_rwsem);
2232}
2233
2234static inline int f2fs_trylock_op(struct f2fs_sb_info *sbi)
2235{
2236 if (time_to_inject(sbi, FAULT_LOCK_OP)) {
2237 f2fs_show_injection_info(sbi, FAULT_LOCK_OP);
2238 return 0;
2239 }
2240 return f2fs_down_read_trylock(&sbi->cp_rwsem);
2241}
2242
2243static inline void f2fs_unlock_op(struct f2fs_sb_info *sbi)
2244{
2245 f2fs_up_read(&sbi->cp_rwsem);
2246}
2247
2248static inline void f2fs_lock_all(struct f2fs_sb_info *sbi)
2249{
2250 f2fs_down_write(&sbi->cp_rwsem);
2251}
2252
2253static inline void f2fs_unlock_all(struct f2fs_sb_info *sbi)
2254{
2255 f2fs_up_write(&sbi->cp_rwsem);
2256}
2257
2258static inline int __get_cp_reason(struct f2fs_sb_info *sbi)
2259{
2260 int reason = CP_SYNC;
2261
2262 if (test_opt(sbi, FASTBOOT))
2263 reason = CP_FASTBOOT;
2264 if (is_sbi_flag_set(sbi, SBI_IS_CLOSE))
2265 reason = CP_UMOUNT;
2266 return reason;
2267}
2268
2269static inline bool __remain_node_summaries(int reason)
2270{
2271 return (reason & (CP_UMOUNT | CP_FASTBOOT));
2272}
2273
2274static inline bool __exist_node_summaries(struct f2fs_sb_info *sbi)
2275{
2276 return (is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG) ||
2277 is_set_ckpt_flags(sbi, CP_FASTBOOT_FLAG));
2278}
2279
2280/*
2281 * Check whether the inode has blocks or not
2282 */
2283static inline int F2FS_HAS_BLOCKS(struct inode *inode)
2284{
2285 block_t xattr_block = F2FS_I(inode)->i_xattr_nid ? 1 : 0;
2286
2287 return (inode->i_blocks >> F2FS_LOG_SECTORS_PER_BLOCK) > xattr_block;
2288}
2289
2290static inline bool f2fs_has_xattr_block(unsigned int ofs)
2291{
2292 return ofs == XATTR_NODE_OFFSET;
2293}
2294
2295static inline bool __allow_reserved_blocks(struct f2fs_sb_info *sbi,
2296 struct inode *inode, bool cap)
2297{
2298 if (!inode)
2299 return true;
2300 if (!test_opt(sbi, RESERVE_ROOT))
2301 return false;
2302 if (IS_NOQUOTA(inode))
2303 return true;
2304 if (uid_eq(F2FS_OPTION(sbi).s_resuid, current_fsuid()))
2305 return true;
2306 if (!gid_eq(F2FS_OPTION(sbi).s_resgid, GLOBAL_ROOT_GID) &&
2307 in_group_p(F2FS_OPTION(sbi).s_resgid))
2308 return true;
2309 if (cap && capable(CAP_SYS_RESOURCE))
2310 return true;
2311 return false;
2312}
2313
2314static inline void f2fs_i_blocks_write(struct inode *, block_t, bool, bool);
2315static inline int inc_valid_block_count(struct f2fs_sb_info *sbi,
2316 struct inode *inode, blkcnt_t *count)
2317{
2318 blkcnt_t diff = 0, release = 0;
2319 block_t avail_user_block_count;
2320 int ret;
2321
2322 ret = dquot_reserve_block(inode, *count);
2323 if (ret)
2324 return ret;
2325
2326 if (time_to_inject(sbi, FAULT_BLOCK)) {
2327 f2fs_show_injection_info(sbi, FAULT_BLOCK);
2328 release = *count;
2329 goto release_quota;
2330 }
2331
2332 /*
2333 * let's increase this in prior to actual block count change in order
2334 * for f2fs_sync_file to avoid data races when deciding checkpoint.
2335 */
2336 percpu_counter_add(&sbi->alloc_valid_block_count, (*count));
2337
2338 spin_lock(&sbi->stat_lock);
2339 sbi->total_valid_block_count += (block_t)(*count);
2340 avail_user_block_count = sbi->user_block_count -
2341 sbi->current_reserved_blocks;
2342
2343 if (!__allow_reserved_blocks(sbi, inode, true))
2344 avail_user_block_count -= F2FS_OPTION(sbi).root_reserved_blocks;
2345
2346 if (F2FS_IO_ALIGNED(sbi))
2347 avail_user_block_count -= sbi->blocks_per_seg *
2348 SM_I(sbi)->additional_reserved_segments;
2349
2350 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2351 if (avail_user_block_count > sbi->unusable_block_count)
2352 avail_user_block_count -= sbi->unusable_block_count;
2353 else
2354 avail_user_block_count = 0;
2355 }
2356 if (unlikely(sbi->total_valid_block_count > avail_user_block_count)) {
2357 diff = sbi->total_valid_block_count - avail_user_block_count;
2358 if (diff > *count)
2359 diff = *count;
2360 *count -= diff;
2361 release = diff;
2362 sbi->total_valid_block_count -= diff;
2363 if (!*count) {
2364 spin_unlock(&sbi->stat_lock);
2365 goto enospc;
2366 }
2367 }
2368 spin_unlock(&sbi->stat_lock);
2369
2370 if (unlikely(release)) {
2371 percpu_counter_sub(&sbi->alloc_valid_block_count, release);
2372 dquot_release_reservation_block(inode, release);
2373 }
2374 f2fs_i_blocks_write(inode, *count, true, true);
2375 return 0;
2376
2377enospc:
2378 percpu_counter_sub(&sbi->alloc_valid_block_count, release);
2379release_quota:
2380 dquot_release_reservation_block(inode, release);
2381 return -ENOSPC;
2382}
2383
2384__printf(2, 3)
2385void f2fs_printk(struct f2fs_sb_info *sbi, const char *fmt, ...);
2386
2387#define f2fs_err(sbi, fmt, ...) \
2388 f2fs_printk(sbi, KERN_ERR fmt, ##__VA_ARGS__)
2389#define f2fs_warn(sbi, fmt, ...) \
2390 f2fs_printk(sbi, KERN_WARNING fmt, ##__VA_ARGS__)
2391#define f2fs_notice(sbi, fmt, ...) \
2392 f2fs_printk(sbi, KERN_NOTICE fmt, ##__VA_ARGS__)
2393#define f2fs_info(sbi, fmt, ...) \
2394 f2fs_printk(sbi, KERN_INFO fmt, ##__VA_ARGS__)
2395#define f2fs_debug(sbi, fmt, ...) \
2396 f2fs_printk(sbi, KERN_DEBUG fmt, ##__VA_ARGS__)
2397
2398static inline void dec_valid_block_count(struct f2fs_sb_info *sbi,
2399 struct inode *inode,
2400 block_t count)
2401{
2402 blkcnt_t sectors = count << F2FS_LOG_SECTORS_PER_BLOCK;
2403
2404 spin_lock(&sbi->stat_lock);
2405 f2fs_bug_on(sbi, sbi->total_valid_block_count < (block_t) count);
2406 sbi->total_valid_block_count -= (block_t)count;
2407 if (sbi->reserved_blocks &&
2408 sbi->current_reserved_blocks < sbi->reserved_blocks)
2409 sbi->current_reserved_blocks = min(sbi->reserved_blocks,
2410 sbi->current_reserved_blocks + count);
2411 spin_unlock(&sbi->stat_lock);
2412 if (unlikely(inode->i_blocks < sectors)) {
2413 f2fs_warn(sbi, "Inconsistent i_blocks, ino:%lu, iblocks:%llu, sectors:%llu",
2414 inode->i_ino,
2415 (unsigned long long)inode->i_blocks,
2416 (unsigned long long)sectors);
2417 set_sbi_flag(sbi, SBI_NEED_FSCK);
2418 return;
2419 }
2420 f2fs_i_blocks_write(inode, count, false, true);
2421}
2422
2423static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type)
2424{
2425 atomic_inc(&sbi->nr_pages[count_type]);
2426
2427 if (count_type == F2FS_DIRTY_DENTS ||
2428 count_type == F2FS_DIRTY_NODES ||
2429 count_type == F2FS_DIRTY_META ||
2430 count_type == F2FS_DIRTY_QDATA ||
2431 count_type == F2FS_DIRTY_IMETA)
2432 set_sbi_flag(sbi, SBI_IS_DIRTY);
2433}
2434
2435static inline void inode_inc_dirty_pages(struct inode *inode)
2436{
2437 atomic_inc(&F2FS_I(inode)->dirty_pages);
2438 inc_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
2439 F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA);
2440 if (IS_NOQUOTA(inode))
2441 inc_page_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA);
2442}
2443
2444static inline void dec_page_count(struct f2fs_sb_info *sbi, int count_type)
2445{
2446 atomic_dec(&sbi->nr_pages[count_type]);
2447}
2448
2449static inline void inode_dec_dirty_pages(struct inode *inode)
2450{
2451 if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) &&
2452 !S_ISLNK(inode->i_mode))
2453 return;
2454
2455 atomic_dec(&F2FS_I(inode)->dirty_pages);
2456 dec_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
2457 F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA);
2458 if (IS_NOQUOTA(inode))
2459 dec_page_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA);
2460}
2461
2462static inline void inc_atomic_write_cnt(struct inode *inode)
2463{
2464 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2465 struct f2fs_inode_info *fi = F2FS_I(inode);
2466 u64 current_write;
2467
2468 fi->atomic_write_cnt++;
2469 atomic64_inc(&sbi->current_atomic_write);
2470 current_write = atomic64_read(&sbi->current_atomic_write);
2471 if (current_write > sbi->peak_atomic_write)
2472 sbi->peak_atomic_write = current_write;
2473}
2474
2475static inline void release_atomic_write_cnt(struct inode *inode)
2476{
2477 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2478 struct f2fs_inode_info *fi = F2FS_I(inode);
2479
2480 atomic64_sub(fi->atomic_write_cnt, &sbi->current_atomic_write);
2481 fi->atomic_write_cnt = 0;
2482}
2483
2484static inline s64 get_pages(struct f2fs_sb_info *sbi, int count_type)
2485{
2486 return atomic_read(&sbi->nr_pages[count_type]);
2487}
2488
2489static inline int get_dirty_pages(struct inode *inode)
2490{
2491 return atomic_read(&F2FS_I(inode)->dirty_pages);
2492}
2493
2494static inline int get_blocktype_secs(struct f2fs_sb_info *sbi, int block_type)
2495{
2496 unsigned int pages_per_sec = sbi->segs_per_sec * sbi->blocks_per_seg;
2497 unsigned int segs = (get_pages(sbi, block_type) + pages_per_sec - 1) >>
2498 sbi->log_blocks_per_seg;
2499
2500 return segs / sbi->segs_per_sec;
2501}
2502
2503static inline block_t valid_user_blocks(struct f2fs_sb_info *sbi)
2504{
2505 return sbi->total_valid_block_count;
2506}
2507
2508static inline block_t discard_blocks(struct f2fs_sb_info *sbi)
2509{
2510 return sbi->discard_blks;
2511}
2512
2513static inline unsigned long __bitmap_size(struct f2fs_sb_info *sbi, int flag)
2514{
2515 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2516
2517 /* return NAT or SIT bitmap */
2518 if (flag == NAT_BITMAP)
2519 return le32_to_cpu(ckpt->nat_ver_bitmap_bytesize);
2520 else if (flag == SIT_BITMAP)
2521 return le32_to_cpu(ckpt->sit_ver_bitmap_bytesize);
2522
2523 return 0;
2524}
2525
2526static inline block_t __cp_payload(struct f2fs_sb_info *sbi)
2527{
2528 return le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_payload);
2529}
2530
2531static inline void *__bitmap_ptr(struct f2fs_sb_info *sbi, int flag)
2532{
2533 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2534 void *tmp_ptr = &ckpt->sit_nat_version_bitmap;
2535 int offset;
2536
2537 if (is_set_ckpt_flags(sbi, CP_LARGE_NAT_BITMAP_FLAG)) {
2538 offset = (flag == SIT_BITMAP) ?
2539 le32_to_cpu(ckpt->nat_ver_bitmap_bytesize) : 0;
2540 /*
2541 * if large_nat_bitmap feature is enabled, leave checksum
2542 * protection for all nat/sit bitmaps.
2543 */
2544 return tmp_ptr + offset + sizeof(__le32);
2545 }
2546
2547 if (__cp_payload(sbi) > 0) {
2548 if (flag == NAT_BITMAP)
2549 return tmp_ptr;
2550 else
2551 return (unsigned char *)ckpt + F2FS_BLKSIZE;
2552 } else {
2553 offset = (flag == NAT_BITMAP) ?
2554 le32_to_cpu(ckpt->sit_ver_bitmap_bytesize) : 0;
2555 return tmp_ptr + offset;
2556 }
2557}
2558
2559static inline block_t __start_cp_addr(struct f2fs_sb_info *sbi)
2560{
2561 block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr);
2562
2563 if (sbi->cur_cp_pack == 2)
2564 start_addr += sbi->blocks_per_seg;
2565 return start_addr;
2566}
2567
2568static inline block_t __start_cp_next_addr(struct f2fs_sb_info *sbi)
2569{
2570 block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr);
2571
2572 if (sbi->cur_cp_pack == 1)
2573 start_addr += sbi->blocks_per_seg;
2574 return start_addr;
2575}
2576
2577static inline void __set_cp_next_pack(struct f2fs_sb_info *sbi)
2578{
2579 sbi->cur_cp_pack = (sbi->cur_cp_pack == 1) ? 2 : 1;
2580}
2581
2582static inline block_t __start_sum_addr(struct f2fs_sb_info *sbi)
2583{
2584 return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum);
2585}
2586
2587extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync);
2588static inline int inc_valid_node_count(struct f2fs_sb_info *sbi,
2589 struct inode *inode, bool is_inode)
2590{
2591 block_t valid_block_count;
2592 unsigned int valid_node_count, user_block_count;
2593 int err;
2594
2595 if (is_inode) {
2596 if (inode) {
2597 err = dquot_alloc_inode(inode);
2598 if (err)
2599 return err;
2600 }
2601 } else {
2602 err = dquot_reserve_block(inode, 1);
2603 if (err)
2604 return err;
2605 }
2606
2607 if (time_to_inject(sbi, FAULT_BLOCK)) {
2608 f2fs_show_injection_info(sbi, FAULT_BLOCK);
2609 goto enospc;
2610 }
2611
2612 spin_lock(&sbi->stat_lock);
2613
2614 valid_block_count = sbi->total_valid_block_count +
2615 sbi->current_reserved_blocks + 1;
2616
2617 if (!__allow_reserved_blocks(sbi, inode, false))
2618 valid_block_count += F2FS_OPTION(sbi).root_reserved_blocks;
2619
2620 if (F2FS_IO_ALIGNED(sbi))
2621 valid_block_count += sbi->blocks_per_seg *
2622 SM_I(sbi)->additional_reserved_segments;
2623
2624 user_block_count = sbi->user_block_count;
2625 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2626 user_block_count -= sbi->unusable_block_count;
2627
2628 if (unlikely(valid_block_count > user_block_count)) {
2629 spin_unlock(&sbi->stat_lock);
2630 goto enospc;
2631 }
2632
2633 valid_node_count = sbi->total_valid_node_count + 1;
2634 if (unlikely(valid_node_count > sbi->total_node_count)) {
2635 spin_unlock(&sbi->stat_lock);
2636 goto enospc;
2637 }
2638
2639 sbi->total_valid_node_count++;
2640 sbi->total_valid_block_count++;
2641 spin_unlock(&sbi->stat_lock);
2642
2643 if (inode) {
2644 if (is_inode)
2645 f2fs_mark_inode_dirty_sync(inode, true);
2646 else
2647 f2fs_i_blocks_write(inode, 1, true, true);
2648 }
2649
2650 percpu_counter_inc(&sbi->alloc_valid_block_count);
2651 return 0;
2652
2653enospc:
2654 if (is_inode) {
2655 if (inode)
2656 dquot_free_inode(inode);
2657 } else {
2658 dquot_release_reservation_block(inode, 1);
2659 }
2660 return -ENOSPC;
2661}
2662
2663static inline void dec_valid_node_count(struct f2fs_sb_info *sbi,
2664 struct inode *inode, bool is_inode)
2665{
2666 spin_lock(&sbi->stat_lock);
2667
2668 if (unlikely(!sbi->total_valid_block_count ||
2669 !sbi->total_valid_node_count)) {
2670 f2fs_warn(sbi, "dec_valid_node_count: inconsistent block counts, total_valid_block:%u, total_valid_node:%u",
2671 sbi->total_valid_block_count,
2672 sbi->total_valid_node_count);
2673 set_sbi_flag(sbi, SBI_NEED_FSCK);
2674 } else {
2675 sbi->total_valid_block_count--;
2676 sbi->total_valid_node_count--;
2677 }
2678
2679 if (sbi->reserved_blocks &&
2680 sbi->current_reserved_blocks < sbi->reserved_blocks)
2681 sbi->current_reserved_blocks++;
2682
2683 spin_unlock(&sbi->stat_lock);
2684
2685 if (is_inode) {
2686 dquot_free_inode(inode);
2687 } else {
2688 if (unlikely(inode->i_blocks == 0)) {
2689 f2fs_warn(sbi, "dec_valid_node_count: inconsistent i_blocks, ino:%lu, iblocks:%llu",
2690 inode->i_ino,
2691 (unsigned long long)inode->i_blocks);
2692 set_sbi_flag(sbi, SBI_NEED_FSCK);
2693 return;
2694 }
2695 f2fs_i_blocks_write(inode, 1, false, true);
2696 }
2697}
2698
2699static inline unsigned int valid_node_count(struct f2fs_sb_info *sbi)
2700{
2701 return sbi->total_valid_node_count;
2702}
2703
2704static inline void inc_valid_inode_count(struct f2fs_sb_info *sbi)
2705{
2706 percpu_counter_inc(&sbi->total_valid_inode_count);
2707}
2708
2709static inline void dec_valid_inode_count(struct f2fs_sb_info *sbi)
2710{
2711 percpu_counter_dec(&sbi->total_valid_inode_count);
2712}
2713
2714static inline s64 valid_inode_count(struct f2fs_sb_info *sbi)
2715{
2716 return percpu_counter_sum_positive(&sbi->total_valid_inode_count);
2717}
2718
2719static inline struct page *f2fs_grab_cache_page(struct address_space *mapping,
2720 pgoff_t index, bool for_write)
2721{
2722 struct page *page;
2723 unsigned int flags;
2724
2725 if (IS_ENABLED(CONFIG_F2FS_FAULT_INJECTION)) {
2726 if (!for_write)
2727 page = find_get_page_flags(mapping, index,
2728 FGP_LOCK | FGP_ACCESSED);
2729 else
2730 page = find_lock_page(mapping, index);
2731 if (page)
2732 return page;
2733
2734 if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_ALLOC)) {
2735 f2fs_show_injection_info(F2FS_M_SB(mapping),
2736 FAULT_PAGE_ALLOC);
2737 return NULL;
2738 }
2739 }
2740
2741 if (!for_write)
2742 return grab_cache_page(mapping, index);
2743
2744 flags = memalloc_nofs_save();
2745 page = grab_cache_page_write_begin(mapping, index);
2746 memalloc_nofs_restore(flags);
2747
2748 return page;
2749}
2750
2751static inline struct page *f2fs_pagecache_get_page(
2752 struct address_space *mapping, pgoff_t index,
2753 int fgp_flags, gfp_t gfp_mask)
2754{
2755 if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_GET)) {
2756 f2fs_show_injection_info(F2FS_M_SB(mapping), FAULT_PAGE_GET);
2757 return NULL;
2758 }
2759
2760 return pagecache_get_page(mapping, index, fgp_flags, gfp_mask);
2761}
2762
2763static inline void f2fs_put_page(struct page *page, int unlock)
2764{
2765 if (!page)
2766 return;
2767
2768 if (unlock) {
2769 f2fs_bug_on(F2FS_P_SB(page), !PageLocked(page));
2770 unlock_page(page);
2771 }
2772 put_page(page);
2773}
2774
2775static inline void f2fs_put_dnode(struct dnode_of_data *dn)
2776{
2777 if (dn->node_page)
2778 f2fs_put_page(dn->node_page, 1);
2779 if (dn->inode_page && dn->node_page != dn->inode_page)
2780 f2fs_put_page(dn->inode_page, 0);
2781 dn->node_page = NULL;
2782 dn->inode_page = NULL;
2783}
2784
2785static inline struct kmem_cache *f2fs_kmem_cache_create(const char *name,
2786 size_t size)
2787{
2788 return kmem_cache_create(name, size, 0, SLAB_RECLAIM_ACCOUNT, NULL);
2789}
2790
2791static inline void *f2fs_kmem_cache_alloc_nofail(struct kmem_cache *cachep,
2792 gfp_t flags)
2793{
2794 void *entry;
2795
2796 entry = kmem_cache_alloc(cachep, flags);
2797 if (!entry)
2798 entry = kmem_cache_alloc(cachep, flags | __GFP_NOFAIL);
2799 return entry;
2800}
2801
2802static inline void *f2fs_kmem_cache_alloc(struct kmem_cache *cachep,
2803 gfp_t flags, bool nofail, struct f2fs_sb_info *sbi)
2804{
2805 if (nofail)
2806 return f2fs_kmem_cache_alloc_nofail(cachep, flags);
2807
2808 if (time_to_inject(sbi, FAULT_SLAB_ALLOC)) {
2809 f2fs_show_injection_info(sbi, FAULT_SLAB_ALLOC);
2810 return NULL;
2811 }
2812
2813 return kmem_cache_alloc(cachep, flags);
2814}
2815
2816static inline bool is_inflight_io(struct f2fs_sb_info *sbi, int type)
2817{
2818 if (get_pages(sbi, F2FS_RD_DATA) || get_pages(sbi, F2FS_RD_NODE) ||
2819 get_pages(sbi, F2FS_RD_META) || get_pages(sbi, F2FS_WB_DATA) ||
2820 get_pages(sbi, F2FS_WB_CP_DATA) ||
2821 get_pages(sbi, F2FS_DIO_READ) ||
2822 get_pages(sbi, F2FS_DIO_WRITE))
2823 return true;
2824
2825 if (type != DISCARD_TIME && SM_I(sbi) && SM_I(sbi)->dcc_info &&
2826 atomic_read(&SM_I(sbi)->dcc_info->queued_discard))
2827 return true;
2828
2829 if (SM_I(sbi) && SM_I(sbi)->fcc_info &&
2830 atomic_read(&SM_I(sbi)->fcc_info->queued_flush))
2831 return true;
2832 return false;
2833}
2834
2835static inline bool is_idle(struct f2fs_sb_info *sbi, int type)
2836{
2837 if (sbi->gc_mode == GC_URGENT_HIGH)
2838 return true;
2839
2840 if (is_inflight_io(sbi, type))
2841 return false;
2842
2843 if (sbi->gc_mode == GC_URGENT_MID)
2844 return true;
2845
2846 if (sbi->gc_mode == GC_URGENT_LOW &&
2847 (type == DISCARD_TIME || type == GC_TIME))
2848 return true;
2849
2850 return f2fs_time_over(sbi, type);
2851}
2852
2853static inline void f2fs_radix_tree_insert(struct radix_tree_root *root,
2854 unsigned long index, void *item)
2855{
2856 while (radix_tree_insert(root, index, item))
2857 cond_resched();
2858}
2859
2860#define RAW_IS_INODE(p) ((p)->footer.nid == (p)->footer.ino)
2861
2862static inline bool IS_INODE(struct page *page)
2863{
2864 struct f2fs_node *p = F2FS_NODE(page);
2865
2866 return RAW_IS_INODE(p);
2867}
2868
2869static inline int offset_in_addr(struct f2fs_inode *i)
2870{
2871 return (i->i_inline & F2FS_EXTRA_ATTR) ?
2872 (le16_to_cpu(i->i_extra_isize) / sizeof(__le32)) : 0;
2873}
2874
2875static inline __le32 *blkaddr_in_node(struct f2fs_node *node)
2876{
2877 return RAW_IS_INODE(node) ? node->i.i_addr : node->dn.addr;
2878}
2879
2880static inline int f2fs_has_extra_attr(struct inode *inode);
2881static inline block_t data_blkaddr(struct inode *inode,
2882 struct page *node_page, unsigned int offset)
2883{
2884 struct f2fs_node *raw_node;
2885 __le32 *addr_array;
2886 int base = 0;
2887 bool is_inode = IS_INODE(node_page);
2888
2889 raw_node = F2FS_NODE(node_page);
2890
2891 if (is_inode) {
2892 if (!inode)
2893 /* from GC path only */
2894 base = offset_in_addr(&raw_node->i);
2895 else if (f2fs_has_extra_attr(inode))
2896 base = get_extra_isize(inode);
2897 }
2898
2899 addr_array = blkaddr_in_node(raw_node);
2900 return le32_to_cpu(addr_array[base + offset]);
2901}
2902
2903static inline block_t f2fs_data_blkaddr(struct dnode_of_data *dn)
2904{
2905 return data_blkaddr(dn->inode, dn->node_page, dn->ofs_in_node);
2906}
2907
2908static inline int f2fs_test_bit(unsigned int nr, char *addr)
2909{
2910 int mask;
2911
2912 addr += (nr >> 3);
2913 mask = 1 << (7 - (nr & 0x07));
2914 return mask & *addr;
2915}
2916
2917static inline void f2fs_set_bit(unsigned int nr, char *addr)
2918{
2919 int mask;
2920
2921 addr += (nr >> 3);
2922 mask = 1 << (7 - (nr & 0x07));
2923 *addr |= mask;
2924}
2925
2926static inline void f2fs_clear_bit(unsigned int nr, char *addr)
2927{
2928 int mask;
2929
2930 addr += (nr >> 3);
2931 mask = 1 << (7 - (nr & 0x07));
2932 *addr &= ~mask;
2933}
2934
2935static inline int f2fs_test_and_set_bit(unsigned int nr, char *addr)
2936{
2937 int mask;
2938 int ret;
2939
2940 addr += (nr >> 3);
2941 mask = 1 << (7 - (nr & 0x07));
2942 ret = mask & *addr;
2943 *addr |= mask;
2944 return ret;
2945}
2946
2947static inline int f2fs_test_and_clear_bit(unsigned int nr, char *addr)
2948{
2949 int mask;
2950 int ret;
2951
2952 addr += (nr >> 3);
2953 mask = 1 << (7 - (nr & 0x07));
2954 ret = mask & *addr;
2955 *addr &= ~mask;
2956 return ret;
2957}
2958
2959static inline void f2fs_change_bit(unsigned int nr, char *addr)
2960{
2961 int mask;
2962
2963 addr += (nr >> 3);
2964 mask = 1 << (7 - (nr & 0x07));
2965 *addr ^= mask;
2966}
2967
2968/*
2969 * On-disk inode flags (f2fs_inode::i_flags)
2970 */
2971#define F2FS_COMPR_FL 0x00000004 /* Compress file */
2972#define F2FS_SYNC_FL 0x00000008 /* Synchronous updates */
2973#define F2FS_IMMUTABLE_FL 0x00000010 /* Immutable file */
2974#define F2FS_APPEND_FL 0x00000020 /* writes to file may only append */
2975#define F2FS_NODUMP_FL 0x00000040 /* do not dump file */
2976#define F2FS_NOATIME_FL 0x00000080 /* do not update atime */
2977#define F2FS_NOCOMP_FL 0x00000400 /* Don't compress */
2978#define F2FS_INDEX_FL 0x00001000 /* hash-indexed directory */
2979#define F2FS_DIRSYNC_FL 0x00010000 /* dirsync behaviour (directories only) */
2980#define F2FS_PROJINHERIT_FL 0x20000000 /* Create with parents projid */
2981#define F2FS_CASEFOLD_FL 0x40000000 /* Casefolded file */
2982
2983/* Flags that should be inherited by new inodes from their parent. */
2984#define F2FS_FL_INHERITED (F2FS_SYNC_FL | F2FS_NODUMP_FL | F2FS_NOATIME_FL | \
2985 F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL | \
2986 F2FS_CASEFOLD_FL)
2987
2988/* Flags that are appropriate for regular files (all but dir-specific ones). */
2989#define F2FS_REG_FLMASK (~(F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL | \
2990 F2FS_CASEFOLD_FL))
2991
2992/* Flags that are appropriate for non-directories/regular files. */
2993#define F2FS_OTHER_FLMASK (F2FS_NODUMP_FL | F2FS_NOATIME_FL)
2994
2995static inline __u32 f2fs_mask_flags(umode_t mode, __u32 flags)
2996{
2997 if (S_ISDIR(mode))
2998 return flags;
2999 else if (S_ISREG(mode))
3000 return flags & F2FS_REG_FLMASK;
3001 else
3002 return flags & F2FS_OTHER_FLMASK;
3003}
3004
3005static inline void __mark_inode_dirty_flag(struct inode *inode,
3006 int flag, bool set)
3007{
3008 switch (flag) {
3009 case FI_INLINE_XATTR:
3010 case FI_INLINE_DATA:
3011 case FI_INLINE_DENTRY:
3012 case FI_NEW_INODE:
3013 if (set)
3014 return;
3015 fallthrough;
3016 case FI_DATA_EXIST:
3017 case FI_INLINE_DOTS:
3018 case FI_PIN_FILE:
3019 case FI_COMPRESS_RELEASED:
3020 f2fs_mark_inode_dirty_sync(inode, true);
3021 }
3022}
3023
3024static inline void set_inode_flag(struct inode *inode, int flag)
3025{
3026 set_bit(flag, F2FS_I(inode)->flags);
3027 __mark_inode_dirty_flag(inode, flag, true);
3028}
3029
3030static inline int is_inode_flag_set(struct inode *inode, int flag)
3031{
3032 return test_bit(flag, F2FS_I(inode)->flags);
3033}
3034
3035static inline void clear_inode_flag(struct inode *inode, int flag)
3036{
3037 clear_bit(flag, F2FS_I(inode)->flags);
3038 __mark_inode_dirty_flag(inode, flag, false);
3039}
3040
3041static inline bool f2fs_verity_in_progress(struct inode *inode)
3042{
3043 return IS_ENABLED(CONFIG_FS_VERITY) &&
3044 is_inode_flag_set(inode, FI_VERITY_IN_PROGRESS);
3045}
3046
3047static inline void set_acl_inode(struct inode *inode, umode_t mode)
3048{
3049 F2FS_I(inode)->i_acl_mode = mode;
3050 set_inode_flag(inode, FI_ACL_MODE);
3051 f2fs_mark_inode_dirty_sync(inode, false);
3052}
3053
3054static inline void f2fs_i_links_write(struct inode *inode, bool inc)
3055{
3056 if (inc)
3057 inc_nlink(inode);
3058 else
3059 drop_nlink(inode);
3060 f2fs_mark_inode_dirty_sync(inode, true);
3061}
3062
3063static inline void f2fs_i_blocks_write(struct inode *inode,
3064 block_t diff, bool add, bool claim)
3065{
3066 bool clean = !is_inode_flag_set(inode, FI_DIRTY_INODE);
3067 bool recover = is_inode_flag_set(inode, FI_AUTO_RECOVER);
3068
3069 /* add = 1, claim = 1 should be dquot_reserve_block in pair */
3070 if (add) {
3071 if (claim)
3072 dquot_claim_block(inode, diff);
3073 else
3074 dquot_alloc_block_nofail(inode, diff);
3075 } else {
3076 dquot_free_block(inode, diff);
3077 }
3078
3079 f2fs_mark_inode_dirty_sync(inode, true);
3080 if (clean || recover)
3081 set_inode_flag(inode, FI_AUTO_RECOVER);
3082}
3083
3084static inline bool f2fs_is_atomic_file(struct inode *inode);
3085
3086static inline void f2fs_i_size_write(struct inode *inode, loff_t i_size)
3087{
3088 bool clean = !is_inode_flag_set(inode, FI_DIRTY_INODE);
3089 bool recover = is_inode_flag_set(inode, FI_AUTO_RECOVER);
3090
3091 if (i_size_read(inode) == i_size)
3092 return;
3093
3094 i_size_write(inode, i_size);
3095
3096 if (f2fs_is_atomic_file(inode))
3097 return;
3098
3099 f2fs_mark_inode_dirty_sync(inode, true);
3100 if (clean || recover)
3101 set_inode_flag(inode, FI_AUTO_RECOVER);
3102}
3103
3104static inline void f2fs_i_depth_write(struct inode *inode, unsigned int depth)
3105{
3106 F2FS_I(inode)->i_current_depth = depth;
3107 f2fs_mark_inode_dirty_sync(inode, true);
3108}
3109
3110static inline void f2fs_i_gc_failures_write(struct inode *inode,
3111 unsigned int count)
3112{
3113 F2FS_I(inode)->i_gc_failures[GC_FAILURE_PIN] = count;
3114 f2fs_mark_inode_dirty_sync(inode, true);
3115}
3116
3117static inline void f2fs_i_xnid_write(struct inode *inode, nid_t xnid)
3118{
3119 F2FS_I(inode)->i_xattr_nid = xnid;
3120 f2fs_mark_inode_dirty_sync(inode, true);
3121}
3122
3123static inline void f2fs_i_pino_write(struct inode *inode, nid_t pino)
3124{
3125 F2FS_I(inode)->i_pino = pino;
3126 f2fs_mark_inode_dirty_sync(inode, true);
3127}
3128
3129static inline void get_inline_info(struct inode *inode, struct f2fs_inode *ri)
3130{
3131 struct f2fs_inode_info *fi = F2FS_I(inode);
3132
3133 if (ri->i_inline & F2FS_INLINE_XATTR)
3134 set_bit(FI_INLINE_XATTR, fi->flags);
3135 if (ri->i_inline & F2FS_INLINE_DATA)
3136 set_bit(FI_INLINE_DATA, fi->flags);
3137 if (ri->i_inline & F2FS_INLINE_DENTRY)
3138 set_bit(FI_INLINE_DENTRY, fi->flags);
3139 if (ri->i_inline & F2FS_DATA_EXIST)
3140 set_bit(FI_DATA_EXIST, fi->flags);
3141 if (ri->i_inline & F2FS_INLINE_DOTS)
3142 set_bit(FI_INLINE_DOTS, fi->flags);
3143 if (ri->i_inline & F2FS_EXTRA_ATTR)
3144 set_bit(FI_EXTRA_ATTR, fi->flags);
3145 if (ri->i_inline & F2FS_PIN_FILE)
3146 set_bit(FI_PIN_FILE, fi->flags);
3147 if (ri->i_inline & F2FS_COMPRESS_RELEASED)
3148 set_bit(FI_COMPRESS_RELEASED, fi->flags);
3149}
3150
3151static inline void set_raw_inline(struct inode *inode, struct f2fs_inode *ri)
3152{
3153 ri->i_inline = 0;
3154
3155 if (is_inode_flag_set(inode, FI_INLINE_XATTR))
3156 ri->i_inline |= F2FS_INLINE_XATTR;
3157 if (is_inode_flag_set(inode, FI_INLINE_DATA))
3158 ri->i_inline |= F2FS_INLINE_DATA;
3159 if (is_inode_flag_set(inode, FI_INLINE_DENTRY))
3160 ri->i_inline |= F2FS_INLINE_DENTRY;
3161 if (is_inode_flag_set(inode, FI_DATA_EXIST))
3162 ri->i_inline |= F2FS_DATA_EXIST;
3163 if (is_inode_flag_set(inode, FI_INLINE_DOTS))
3164 ri->i_inline |= F2FS_INLINE_DOTS;
3165 if (is_inode_flag_set(inode, FI_EXTRA_ATTR))
3166 ri->i_inline |= F2FS_EXTRA_ATTR;
3167 if (is_inode_flag_set(inode, FI_PIN_FILE))
3168 ri->i_inline |= F2FS_PIN_FILE;
3169 if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED))
3170 ri->i_inline |= F2FS_COMPRESS_RELEASED;
3171}
3172
3173static inline int f2fs_has_extra_attr(struct inode *inode)
3174{
3175 return is_inode_flag_set(inode, FI_EXTRA_ATTR);
3176}
3177
3178static inline int f2fs_has_inline_xattr(struct inode *inode)
3179{
3180 return is_inode_flag_set(inode, FI_INLINE_XATTR);
3181}
3182
3183static inline int f2fs_compressed_file(struct inode *inode)
3184{
3185 return S_ISREG(inode->i_mode) &&
3186 is_inode_flag_set(inode, FI_COMPRESSED_FILE);
3187}
3188
3189static inline bool f2fs_need_compress_data(struct inode *inode)
3190{
3191 int compress_mode = F2FS_OPTION(F2FS_I_SB(inode)).compress_mode;
3192
3193 if (!f2fs_compressed_file(inode))
3194 return false;
3195
3196 if (compress_mode == COMPR_MODE_FS)
3197 return true;
3198 else if (compress_mode == COMPR_MODE_USER &&
3199 is_inode_flag_set(inode, FI_ENABLE_COMPRESS))
3200 return true;
3201
3202 return false;
3203}
3204
3205static inline unsigned int addrs_per_inode(struct inode *inode)
3206{
3207 unsigned int addrs = CUR_ADDRS_PER_INODE(inode) -
3208 get_inline_xattr_addrs(inode);
3209
3210 if (!f2fs_compressed_file(inode))
3211 return addrs;
3212 return ALIGN_DOWN(addrs, F2FS_I(inode)->i_cluster_size);
3213}
3214
3215static inline unsigned int addrs_per_block(struct inode *inode)
3216{
3217 if (!f2fs_compressed_file(inode))
3218 return DEF_ADDRS_PER_BLOCK;
3219 return ALIGN_DOWN(DEF_ADDRS_PER_BLOCK, F2FS_I(inode)->i_cluster_size);
3220}
3221
3222static inline void *inline_xattr_addr(struct inode *inode, struct page *page)
3223{
3224 struct f2fs_inode *ri = F2FS_INODE(page);
3225
3226 return (void *)&(ri->i_addr[DEF_ADDRS_PER_INODE -
3227 get_inline_xattr_addrs(inode)]);
3228}
3229
3230static inline int inline_xattr_size(struct inode *inode)
3231{
3232 if (f2fs_has_inline_xattr(inode))
3233 return get_inline_xattr_addrs(inode) * sizeof(__le32);
3234 return 0;
3235}
3236
3237/*
3238 * Notice: check inline_data flag without inode page lock is unsafe.
3239 * It could change at any time by f2fs_convert_inline_page().
3240 */
3241static inline int f2fs_has_inline_data(struct inode *inode)
3242{
3243 return is_inode_flag_set(inode, FI_INLINE_DATA);
3244}
3245
3246static inline int f2fs_exist_data(struct inode *inode)
3247{
3248 return is_inode_flag_set(inode, FI_DATA_EXIST);
3249}
3250
3251static inline int f2fs_has_inline_dots(struct inode *inode)
3252{
3253 return is_inode_flag_set(inode, FI_INLINE_DOTS);
3254}
3255
3256static inline int f2fs_is_mmap_file(struct inode *inode)
3257{
3258 return is_inode_flag_set(inode, FI_MMAP_FILE);
3259}
3260
3261static inline bool f2fs_is_pinned_file(struct inode *inode)
3262{
3263 return is_inode_flag_set(inode, FI_PIN_FILE);
3264}
3265
3266static inline bool f2fs_is_atomic_file(struct inode *inode)
3267{
3268 return is_inode_flag_set(inode, FI_ATOMIC_FILE);
3269}
3270
3271static inline bool f2fs_is_cow_file(struct inode *inode)
3272{
3273 return is_inode_flag_set(inode, FI_COW_FILE);
3274}
3275
3276static inline bool f2fs_is_first_block_written(struct inode *inode)
3277{
3278 return is_inode_flag_set(inode, FI_FIRST_BLOCK_WRITTEN);
3279}
3280
3281static inline bool f2fs_is_drop_cache(struct inode *inode)
3282{
3283 return is_inode_flag_set(inode, FI_DROP_CACHE);
3284}
3285
3286static inline void *inline_data_addr(struct inode *inode, struct page *page)
3287{
3288 struct f2fs_inode *ri = F2FS_INODE(page);
3289 int extra_size = get_extra_isize(inode);
3290
3291 return (void *)&(ri->i_addr[extra_size + DEF_INLINE_RESERVED_SIZE]);
3292}
3293
3294static inline int f2fs_has_inline_dentry(struct inode *inode)
3295{
3296 return is_inode_flag_set(inode, FI_INLINE_DENTRY);
3297}
3298
3299static inline int is_file(struct inode *inode, int type)
3300{
3301 return F2FS_I(inode)->i_advise & type;
3302}
3303
3304static inline void set_file(struct inode *inode, int type)
3305{
3306 if (is_file(inode, type))
3307 return;
3308 F2FS_I(inode)->i_advise |= type;
3309 f2fs_mark_inode_dirty_sync(inode, true);
3310}
3311
3312static inline void clear_file(struct inode *inode, int type)
3313{
3314 if (!is_file(inode, type))
3315 return;
3316 F2FS_I(inode)->i_advise &= ~type;
3317 f2fs_mark_inode_dirty_sync(inode, true);
3318}
3319
3320static inline bool f2fs_is_time_consistent(struct inode *inode)
3321{
3322 if (!timespec64_equal(F2FS_I(inode)->i_disk_time, &inode->i_atime))
3323 return false;
3324 if (!timespec64_equal(F2FS_I(inode)->i_disk_time + 1, &inode->i_ctime))
3325 return false;
3326 if (!timespec64_equal(F2FS_I(inode)->i_disk_time + 2, &inode->i_mtime))
3327 return false;
3328 if (!timespec64_equal(F2FS_I(inode)->i_disk_time + 3,
3329 &F2FS_I(inode)->i_crtime))
3330 return false;
3331 return true;
3332}
3333
3334static inline bool f2fs_skip_inode_update(struct inode *inode, int dsync)
3335{
3336 bool ret;
3337
3338 if (dsync) {
3339 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3340
3341 spin_lock(&sbi->inode_lock[DIRTY_META]);
3342 ret = list_empty(&F2FS_I(inode)->gdirty_list);
3343 spin_unlock(&sbi->inode_lock[DIRTY_META]);
3344 return ret;
3345 }
3346 if (!is_inode_flag_set(inode, FI_AUTO_RECOVER) ||
3347 file_keep_isize(inode) ||
3348 i_size_read(inode) & ~PAGE_MASK)
3349 return false;
3350
3351 if (!f2fs_is_time_consistent(inode))
3352 return false;
3353
3354 spin_lock(&F2FS_I(inode)->i_size_lock);
3355 ret = F2FS_I(inode)->last_disk_size == i_size_read(inode);
3356 spin_unlock(&F2FS_I(inode)->i_size_lock);
3357
3358 return ret;
3359}
3360
3361static inline bool f2fs_readonly(struct super_block *sb)
3362{
3363 return sb_rdonly(sb);
3364}
3365
3366static inline bool f2fs_cp_error(struct f2fs_sb_info *sbi)
3367{
3368 return is_set_ckpt_flags(sbi, CP_ERROR_FLAG);
3369}
3370
3371static inline bool is_dot_dotdot(const u8 *name, size_t len)
3372{
3373 if (len == 1 && name[0] == '.')
3374 return true;
3375
3376 if (len == 2 && name[0] == '.' && name[1] == '.')
3377 return true;
3378
3379 return false;
3380}
3381
3382static inline void *f2fs_kmalloc(struct f2fs_sb_info *sbi,
3383 size_t size, gfp_t flags)
3384{
3385 if (time_to_inject(sbi, FAULT_KMALLOC)) {
3386 f2fs_show_injection_info(sbi, FAULT_KMALLOC);
3387 return NULL;
3388 }
3389
3390 return kmalloc(size, flags);
3391}
3392
3393static inline void *f2fs_kzalloc(struct f2fs_sb_info *sbi,
3394 size_t size, gfp_t flags)
3395{
3396 return f2fs_kmalloc(sbi, size, flags | __GFP_ZERO);
3397}
3398
3399static inline void *f2fs_kvmalloc(struct f2fs_sb_info *sbi,
3400 size_t size, gfp_t flags)
3401{
3402 if (time_to_inject(sbi, FAULT_KVMALLOC)) {
3403 f2fs_show_injection_info(sbi, FAULT_KVMALLOC);
3404 return NULL;
3405 }
3406
3407 return kvmalloc(size, flags);
3408}
3409
3410static inline void *f2fs_kvzalloc(struct f2fs_sb_info *sbi,
3411 size_t size, gfp_t flags)
3412{
3413 return f2fs_kvmalloc(sbi, size, flags | __GFP_ZERO);
3414}
3415
3416static inline int get_extra_isize(struct inode *inode)
3417{
3418 return F2FS_I(inode)->i_extra_isize / sizeof(__le32);
3419}
3420
3421static inline int get_inline_xattr_addrs(struct inode *inode)
3422{
3423 return F2FS_I(inode)->i_inline_xattr_size;
3424}
3425
3426#define f2fs_get_inode_mode(i) \
3427 ((is_inode_flag_set(i, FI_ACL_MODE)) ? \
3428 (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))
3429
3430#define F2FS_TOTAL_EXTRA_ATTR_SIZE \
3431 (offsetof(struct f2fs_inode, i_extra_end) - \
3432 offsetof(struct f2fs_inode, i_extra_isize)) \
3433
3434#define F2FS_OLD_ATTRIBUTE_SIZE (offsetof(struct f2fs_inode, i_addr))
3435#define F2FS_FITS_IN_INODE(f2fs_inode, extra_isize, field) \
3436 ((offsetof(typeof(*(f2fs_inode)), field) + \
3437 sizeof((f2fs_inode)->field)) \
3438 <= (F2FS_OLD_ATTRIBUTE_SIZE + (extra_isize))) \
3439
3440#define __is_large_section(sbi) ((sbi)->segs_per_sec > 1)
3441
3442#define __is_meta_io(fio) (PAGE_TYPE_OF_BIO((fio)->type) == META)
3443
3444bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
3445 block_t blkaddr, int type);
3446static inline void verify_blkaddr(struct f2fs_sb_info *sbi,
3447 block_t blkaddr, int type)
3448{
3449 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, type)) {
3450 f2fs_err(sbi, "invalid blkaddr: %u, type: %d, run fsck to fix.",
3451 blkaddr, type);
3452 f2fs_bug_on(sbi, 1);
3453 }
3454}
3455
3456static inline bool __is_valid_data_blkaddr(block_t blkaddr)
3457{
3458 if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR ||
3459 blkaddr == COMPRESS_ADDR)
3460 return false;
3461 return true;
3462}
3463
3464/*
3465 * file.c
3466 */
3467int f2fs_sync_file(struct file *file, loff_t start, loff_t end, int datasync);
3468void f2fs_truncate_data_blocks(struct dnode_of_data *dn);
3469int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock);
3470int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock);
3471int f2fs_truncate(struct inode *inode);
3472int f2fs_getattr(struct user_namespace *mnt_userns, const struct path *path,
3473 struct kstat *stat, u32 request_mask, unsigned int flags);
3474int f2fs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry,
3475 struct iattr *attr);
3476int f2fs_truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end);
3477void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count);
3478int f2fs_precache_extents(struct inode *inode);
3479int f2fs_fileattr_get(struct dentry *dentry, struct fileattr *fa);
3480int f2fs_fileattr_set(struct user_namespace *mnt_userns,
3481 struct dentry *dentry, struct fileattr *fa);
3482long f2fs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
3483long f2fs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
3484int f2fs_transfer_project_quota(struct inode *inode, kprojid_t kprojid);
3485int f2fs_pin_file_control(struct inode *inode, bool inc);
3486
3487/*
3488 * inode.c
3489 */
3490void f2fs_set_inode_flags(struct inode *inode);
3491bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct page *page);
3492void f2fs_inode_chksum_set(struct f2fs_sb_info *sbi, struct page *page);
3493struct inode *f2fs_iget(struct super_block *sb, unsigned long ino);
3494struct inode *f2fs_iget_retry(struct super_block *sb, unsigned long ino);
3495int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink);
3496void f2fs_update_inode(struct inode *inode, struct page *node_page);
3497void f2fs_update_inode_page(struct inode *inode);
3498int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc);
3499void f2fs_evict_inode(struct inode *inode);
3500void f2fs_handle_failed_inode(struct inode *inode);
3501
3502/*
3503 * namei.c
3504 */
3505int f2fs_update_extension_list(struct f2fs_sb_info *sbi, const char *name,
3506 bool hot, bool set);
3507struct dentry *f2fs_get_parent(struct dentry *child);
3508int f2fs_get_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
3509 struct inode **new_inode);
3510
3511/*
3512 * dir.c
3513 */
3514unsigned char f2fs_get_de_type(struct f2fs_dir_entry *de);
3515int f2fs_init_casefolded_name(const struct inode *dir,
3516 struct f2fs_filename *fname);
3517int f2fs_setup_filename(struct inode *dir, const struct qstr *iname,
3518 int lookup, struct f2fs_filename *fname);
3519int f2fs_prepare_lookup(struct inode *dir, struct dentry *dentry,
3520 struct f2fs_filename *fname);
3521void f2fs_free_filename(struct f2fs_filename *fname);
3522struct f2fs_dir_entry *f2fs_find_target_dentry(const struct f2fs_dentry_ptr *d,
3523 const struct f2fs_filename *fname, int *max_slots);
3524int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
3525 unsigned int start_pos, struct fscrypt_str *fstr);
3526void f2fs_do_make_empty_dir(struct inode *inode, struct inode *parent,
3527 struct f2fs_dentry_ptr *d);
3528struct page *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir,
3529 const struct f2fs_filename *fname, struct page *dpage);
3530void f2fs_update_parent_metadata(struct inode *dir, struct inode *inode,
3531 unsigned int current_depth);
3532int f2fs_room_for_filename(const void *bitmap, int slots, int max_slots);
3533void f2fs_drop_nlink(struct inode *dir, struct inode *inode);
3534struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir,
3535 const struct f2fs_filename *fname,
3536 struct page **res_page);
3537struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir,
3538 const struct qstr *child, struct page **res_page);
3539struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, struct page **p);
3540ino_t f2fs_inode_by_name(struct inode *dir, const struct qstr *qstr,
3541 struct page **page);
3542void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de,
3543 struct page *page, struct inode *inode);
3544bool f2fs_has_enough_room(struct inode *dir, struct page *ipage,
3545 const struct f2fs_filename *fname);
3546void f2fs_update_dentry(nid_t ino, umode_t mode, struct f2fs_dentry_ptr *d,
3547 const struct fscrypt_str *name, f2fs_hash_t name_hash,
3548 unsigned int bit_pos);
3549int f2fs_add_regular_entry(struct inode *dir, const struct f2fs_filename *fname,
3550 struct inode *inode, nid_t ino, umode_t mode);
3551int f2fs_add_dentry(struct inode *dir, const struct f2fs_filename *fname,
3552 struct inode *inode, nid_t ino, umode_t mode);
3553int f2fs_do_add_link(struct inode *dir, const struct qstr *name,
3554 struct inode *inode, nid_t ino, umode_t mode);
3555void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page,
3556 struct inode *dir, struct inode *inode);
3557int f2fs_do_tmpfile(struct inode *inode, struct inode *dir);
3558bool f2fs_empty_dir(struct inode *dir);
3559
3560static inline int f2fs_add_link(struct dentry *dentry, struct inode *inode)
3561{
3562 if (fscrypt_is_nokey_name(dentry))
3563 return -ENOKEY;
3564 return f2fs_do_add_link(d_inode(dentry->d_parent), &dentry->d_name,
3565 inode, inode->i_ino, inode->i_mode);
3566}
3567
3568/*
3569 * super.c
3570 */
3571int f2fs_inode_dirtied(struct inode *inode, bool sync);
3572void f2fs_inode_synced(struct inode *inode);
3573int f2fs_dquot_initialize(struct inode *inode);
3574int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly);
3575int f2fs_quota_sync(struct super_block *sb, int type);
3576loff_t max_file_blocks(struct inode *inode);
3577void f2fs_quota_off_umount(struct super_block *sb);
3578void f2fs_handle_stop(struct f2fs_sb_info *sbi, unsigned char reason);
3579void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error);
3580int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover);
3581int f2fs_sync_fs(struct super_block *sb, int sync);
3582int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi);
3583
3584/*
3585 * hash.c
3586 */
3587void f2fs_hash_filename(const struct inode *dir, struct f2fs_filename *fname);
3588
3589/*
3590 * node.c
3591 */
3592struct node_info;
3593
3594int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid);
3595bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type);
3596bool f2fs_in_warm_node_list(struct f2fs_sb_info *sbi, struct page *page);
3597void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi);
3598void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct page *page);
3599void f2fs_reset_fsync_node_info(struct f2fs_sb_info *sbi);
3600int f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid);
3601bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid);
3602bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino);
3603int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid,
3604 struct node_info *ni, bool checkpoint_context);
3605pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs);
3606int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode);
3607int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from);
3608int f2fs_truncate_xattr_node(struct inode *inode);
3609int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi,
3610 unsigned int seq_id);
3611bool f2fs_nat_bitmap_enabled(struct f2fs_sb_info *sbi);
3612int f2fs_remove_inode_page(struct inode *inode);
3613struct page *f2fs_new_inode_page(struct inode *inode);
3614struct page *f2fs_new_node_page(struct dnode_of_data *dn, unsigned int ofs);
3615void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid);
3616struct page *f2fs_get_node_page(struct f2fs_sb_info *sbi, pgoff_t nid);
3617struct page *f2fs_get_node_page_ra(struct page *parent, int start);
3618int f2fs_move_node_page(struct page *node_page, int gc_type);
3619void f2fs_flush_inline_data(struct f2fs_sb_info *sbi);
3620int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode,
3621 struct writeback_control *wbc, bool atomic,
3622 unsigned int *seq_id);
3623int f2fs_sync_node_pages(struct f2fs_sb_info *sbi,
3624 struct writeback_control *wbc,
3625 bool do_balance, enum iostat_type io_type);
3626int f2fs_build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount);
3627bool f2fs_alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid);
3628void f2fs_alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid);
3629void f2fs_alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid);
3630int f2fs_try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink);
3631int f2fs_recover_inline_xattr(struct inode *inode, struct page *page);
3632int f2fs_recover_xattr_data(struct inode *inode, struct page *page);
3633int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct page *page);
3634int f2fs_restore_node_summary(struct f2fs_sb_info *sbi,
3635 unsigned int segno, struct f2fs_summary_block *sum);
3636void f2fs_enable_nat_bits(struct f2fs_sb_info *sbi);
3637int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc);
3638int f2fs_build_node_manager(struct f2fs_sb_info *sbi);
3639void f2fs_destroy_node_manager(struct f2fs_sb_info *sbi);
3640int __init f2fs_create_node_manager_caches(void);
3641void f2fs_destroy_node_manager_caches(void);
3642
3643/*
3644 * segment.c
3645 */
3646bool f2fs_need_SSR(struct f2fs_sb_info *sbi);
3647int f2fs_commit_atomic_write(struct inode *inode);
3648void f2fs_abort_atomic_write(struct inode *inode, bool clean);
3649void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need);
3650void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg);
3651int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino);
3652int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi);
3653int f2fs_flush_device_cache(struct f2fs_sb_info *sbi);
3654void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free);
3655void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr);
3656bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr);
3657int f2fs_start_discard_thread(struct f2fs_sb_info *sbi);
3658void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi);
3659void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi);
3660bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi);
3661void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
3662 struct cp_control *cpc);
3663void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi);
3664block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi);
3665int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable);
3666void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi);
3667int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra);
3668bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno);
3669void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi);
3670void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi);
3671void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi);
3672void f2fs_get_new_segment(struct f2fs_sb_info *sbi,
3673 unsigned int *newseg, bool new_sec, int dir);
3674void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
3675 unsigned int start, unsigned int end);
3676void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force);
3677void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi);
3678int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range);
3679bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
3680 struct cp_control *cpc);
3681struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno);
3682void f2fs_update_meta_page(struct f2fs_sb_info *sbi, void *src,
3683 block_t blk_addr);
3684void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
3685 enum iostat_type io_type);
3686void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio);
3687void f2fs_outplace_write_data(struct dnode_of_data *dn,
3688 struct f2fs_io_info *fio);
3689int f2fs_inplace_write_data(struct f2fs_io_info *fio);
3690void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
3691 block_t old_blkaddr, block_t new_blkaddr,
3692 bool recover_curseg, bool recover_newaddr,
3693 bool from_gc);
3694void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
3695 block_t old_addr, block_t new_addr,
3696 unsigned char version, bool recover_curseg,
3697 bool recover_newaddr);
3698void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
3699 block_t old_blkaddr, block_t *new_blkaddr,
3700 struct f2fs_summary *sum, int type,
3701 struct f2fs_io_info *fio);
3702void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino,
3703 block_t blkaddr, unsigned int blkcnt);
3704void f2fs_wait_on_page_writeback(struct page *page,
3705 enum page_type type, bool ordered, bool locked);
3706void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr);
3707void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
3708 block_t len);
3709void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk);
3710void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk);
3711int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
3712 unsigned int val, int alloc);
3713void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc);
3714int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi);
3715int f2fs_check_write_pointer(struct f2fs_sb_info *sbi);
3716int f2fs_build_segment_manager(struct f2fs_sb_info *sbi);
3717void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi);
3718int __init f2fs_create_segment_manager_caches(void);
3719void f2fs_destroy_segment_manager_caches(void);
3720int f2fs_rw_hint_to_seg_type(enum rw_hint hint);
3721unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
3722 unsigned int segno);
3723unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
3724 unsigned int segno);
3725
3726#define DEF_FRAGMENT_SIZE 4
3727#define MIN_FRAGMENT_SIZE 1
3728#define MAX_FRAGMENT_SIZE 512
3729
3730static inline bool f2fs_need_rand_seg(struct f2fs_sb_info *sbi)
3731{
3732 return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG ||
3733 F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK;
3734}
3735
3736/*
3737 * checkpoint.c
3738 */
3739void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io,
3740 unsigned char reason);
3741void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi);
3742struct page *f2fs_grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index);
3743struct page *f2fs_get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index);
3744struct page *f2fs_get_meta_page_retry(struct f2fs_sb_info *sbi, pgoff_t index);
3745struct page *f2fs_get_tmp_page(struct f2fs_sb_info *sbi, pgoff_t index);
3746bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
3747 block_t blkaddr, int type);
3748int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
3749 int type, bool sync);
3750void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index,
3751 unsigned int ra_blocks);
3752long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type,
3753 long nr_to_write, enum iostat_type io_type);
3754void f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type);
3755void f2fs_remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type);
3756void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all);
3757bool f2fs_exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode);
3758void f2fs_set_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
3759 unsigned int devidx, int type);
3760bool f2fs_is_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
3761 unsigned int devidx, int type);
3762int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi);
3763int f2fs_acquire_orphan_inode(struct f2fs_sb_info *sbi);
3764void f2fs_release_orphan_inode(struct f2fs_sb_info *sbi);
3765void f2fs_add_orphan_inode(struct inode *inode);
3766void f2fs_remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino);
3767int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi);
3768int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi);
3769void f2fs_update_dirty_folio(struct inode *inode, struct folio *folio);
3770void f2fs_remove_dirty_inode(struct inode *inode);
3771int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type,
3772 bool from_cp);
3773void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type);
3774u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi);
3775int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc);
3776void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi);
3777int __init f2fs_create_checkpoint_caches(void);
3778void f2fs_destroy_checkpoint_caches(void);
3779int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi);
3780int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi);
3781void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi);
3782void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi);
3783
3784/*
3785 * data.c
3786 */
3787int __init f2fs_init_bioset(void);
3788void f2fs_destroy_bioset(void);
3789int f2fs_init_bio_entry_cache(void);
3790void f2fs_destroy_bio_entry_cache(void);
3791void f2fs_submit_bio(struct f2fs_sb_info *sbi,
3792 struct bio *bio, enum page_type type);
3793int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi);
3794void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type);
3795void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
3796 struct inode *inode, struct page *page,
3797 nid_t ino, enum page_type type);
3798void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
3799 struct bio **bio, struct page *page);
3800void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi);
3801int f2fs_submit_page_bio(struct f2fs_io_info *fio);
3802int f2fs_merge_page_bio(struct f2fs_io_info *fio);
3803void f2fs_submit_page_write(struct f2fs_io_info *fio);
3804struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
3805 block_t blk_addr, sector_t *sector);
3806int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr);
3807void f2fs_set_data_blkaddr(struct dnode_of_data *dn);
3808void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr);
3809int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count);
3810int f2fs_reserve_new_block(struct dnode_of_data *dn);
3811int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index);
3812int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index);
3813struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
3814 blk_opf_t op_flags, bool for_write, pgoff_t *next_pgofs);
3815struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index,
3816 pgoff_t *next_pgofs);
3817struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index,
3818 bool for_write);
3819struct page *f2fs_get_new_data_page(struct inode *inode,
3820 struct page *ipage, pgoff_t index, bool new_i_size);
3821int f2fs_do_write_data_page(struct f2fs_io_info *fio);
3822void f2fs_do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock);
3823int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
3824 int create, int flag);
3825int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
3826 u64 start, u64 len);
3827int f2fs_encrypt_one_page(struct f2fs_io_info *fio);
3828bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio);
3829bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio);
3830int f2fs_write_single_data_page(struct page *page, int *submitted,
3831 struct bio **bio, sector_t *last_block,
3832 struct writeback_control *wbc,
3833 enum iostat_type io_type,
3834 int compr_blocks, bool allow_balance);
3835void f2fs_write_failed(struct inode *inode, loff_t to);
3836void f2fs_invalidate_folio(struct folio *folio, size_t offset, size_t length);
3837bool f2fs_release_folio(struct folio *folio, gfp_t wait);
3838bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len);
3839void f2fs_clear_page_cache_dirty_tag(struct page *page);
3840int f2fs_init_post_read_processing(void);
3841void f2fs_destroy_post_read_processing(void);
3842int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi);
3843void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi);
3844extern const struct iomap_ops f2fs_iomap_ops;
3845
3846/*
3847 * gc.c
3848 */
3849int f2fs_start_gc_thread(struct f2fs_sb_info *sbi);
3850void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi);
3851block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode);
3852int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control);
3853void f2fs_build_gc_manager(struct f2fs_sb_info *sbi);
3854int f2fs_resize_fs(struct f2fs_sb_info *sbi, __u64 block_count);
3855int __init f2fs_create_garbage_collection_cache(void);
3856void f2fs_destroy_garbage_collection_cache(void);
3857
3858/*
3859 * recovery.c
3860 */
3861int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only);
3862bool f2fs_space_for_roll_forward(struct f2fs_sb_info *sbi);
3863int __init f2fs_create_recovery_cache(void);
3864void f2fs_destroy_recovery_cache(void);
3865
3866/*
3867 * debug.c
3868 */
3869#ifdef CONFIG_F2FS_STAT_FS
3870struct f2fs_stat_info {
3871 struct list_head stat_list;
3872 struct f2fs_sb_info *sbi;
3873 int all_area_segs, sit_area_segs, nat_area_segs, ssa_area_segs;
3874 int main_area_segs, main_area_sections, main_area_zones;
3875 unsigned long long hit_cached[NR_EXTENT_CACHES];
3876 unsigned long long hit_rbtree[NR_EXTENT_CACHES];
3877 unsigned long long total_ext[NR_EXTENT_CACHES];
3878 unsigned long long hit_total[NR_EXTENT_CACHES];
3879 int ext_tree[NR_EXTENT_CACHES];
3880 int zombie_tree[NR_EXTENT_CACHES];
3881 int ext_node[NR_EXTENT_CACHES];
3882 /* to count memory footprint */
3883 unsigned long long ext_mem[NR_EXTENT_CACHES];
3884 /* for read extent cache */
3885 unsigned long long hit_largest;
3886 /* for block age extent cache */
3887 unsigned long long allocated_data_blocks;
3888 int ndirty_node, ndirty_dent, ndirty_meta, ndirty_imeta;
3889 int ndirty_data, ndirty_qdata;
3890 unsigned int ndirty_dirs, ndirty_files, nquota_files, ndirty_all;
3891 int nats, dirty_nats, sits, dirty_sits;
3892 int free_nids, avail_nids, alloc_nids;
3893 int total_count, utilization;
3894 int bg_gc, nr_wb_cp_data, nr_wb_data;
3895 int nr_rd_data, nr_rd_node, nr_rd_meta;
3896 int nr_dio_read, nr_dio_write;
3897 unsigned int io_skip_bggc, other_skip_bggc;
3898 int nr_flushing, nr_flushed, flush_list_empty;
3899 int nr_discarding, nr_discarded;
3900 int nr_discard_cmd;
3901 unsigned int undiscard_blks;
3902 int nr_issued_ckpt, nr_total_ckpt, nr_queued_ckpt;
3903 unsigned int cur_ckpt_time, peak_ckpt_time;
3904 int inline_xattr, inline_inode, inline_dir, append, update, orphans;
3905 int compr_inode, swapfile_inode;
3906 unsigned long long compr_blocks;
3907 int aw_cnt, max_aw_cnt;
3908 unsigned int valid_count, valid_node_count, valid_inode_count, discard_blks;
3909 unsigned int bimodal, avg_vblocks;
3910 int util_free, util_valid, util_invalid;
3911 int rsvd_segs, overp_segs;
3912 int dirty_count, node_pages, meta_pages, compress_pages;
3913 int compress_page_hit;
3914 int prefree_count, call_count, cp_count, bg_cp_count;
3915 int tot_segs, node_segs, data_segs, free_segs, free_secs;
3916 int bg_node_segs, bg_data_segs;
3917 int tot_blks, data_blks, node_blks;
3918 int bg_data_blks, bg_node_blks;
3919 int curseg[NR_CURSEG_TYPE];
3920 int cursec[NR_CURSEG_TYPE];
3921 int curzone[NR_CURSEG_TYPE];
3922 unsigned int dirty_seg[NR_CURSEG_TYPE];
3923 unsigned int full_seg[NR_CURSEG_TYPE];
3924 unsigned int valid_blks[NR_CURSEG_TYPE];
3925
3926 unsigned int meta_count[META_MAX];
3927 unsigned int segment_count[2];
3928 unsigned int block_count[2];
3929 unsigned int inplace_count;
3930 unsigned long long base_mem, cache_mem, page_mem;
3931};
3932
3933static inline struct f2fs_stat_info *F2FS_STAT(struct f2fs_sb_info *sbi)
3934{
3935 return (struct f2fs_stat_info *)sbi->stat_info;
3936}
3937
3938#define stat_inc_cp_count(si) ((si)->cp_count++)
3939#define stat_inc_bg_cp_count(si) ((si)->bg_cp_count++)
3940#define stat_inc_call_count(si) ((si)->call_count++)
3941#define stat_inc_bggc_count(si) ((si)->bg_gc++)
3942#define stat_io_skip_bggc_count(sbi) ((sbi)->io_skip_bggc++)
3943#define stat_other_skip_bggc_count(sbi) ((sbi)->other_skip_bggc++)
3944#define stat_inc_dirty_inode(sbi, type) ((sbi)->ndirty_inode[type]++)
3945#define stat_dec_dirty_inode(sbi, type) ((sbi)->ndirty_inode[type]--)
3946#define stat_inc_total_hit(sbi, type) (atomic64_inc(&(sbi)->total_hit_ext[type]))
3947#define stat_inc_rbtree_node_hit(sbi, type) (atomic64_inc(&(sbi)->read_hit_rbtree[type]))
3948#define stat_inc_largest_node_hit(sbi) (atomic64_inc(&(sbi)->read_hit_largest))
3949#define stat_inc_cached_node_hit(sbi, type) (atomic64_inc(&(sbi)->read_hit_cached[type]))
3950#define stat_inc_inline_xattr(inode) \
3951 do { \
3952 if (f2fs_has_inline_xattr(inode)) \
3953 (atomic_inc(&F2FS_I_SB(inode)->inline_xattr)); \
3954 } while (0)
3955#define stat_dec_inline_xattr(inode) \
3956 do { \
3957 if (f2fs_has_inline_xattr(inode)) \
3958 (atomic_dec(&F2FS_I_SB(inode)->inline_xattr)); \
3959 } while (0)
3960#define stat_inc_inline_inode(inode) \
3961 do { \
3962 if (f2fs_has_inline_data(inode)) \
3963 (atomic_inc(&F2FS_I_SB(inode)->inline_inode)); \
3964 } while (0)
3965#define stat_dec_inline_inode(inode) \
3966 do { \
3967 if (f2fs_has_inline_data(inode)) \
3968 (atomic_dec(&F2FS_I_SB(inode)->inline_inode)); \
3969 } while (0)
3970#define stat_inc_inline_dir(inode) \
3971 do { \
3972 if (f2fs_has_inline_dentry(inode)) \
3973 (atomic_inc(&F2FS_I_SB(inode)->inline_dir)); \
3974 } while (0)
3975#define stat_dec_inline_dir(inode) \
3976 do { \
3977 if (f2fs_has_inline_dentry(inode)) \
3978 (atomic_dec(&F2FS_I_SB(inode)->inline_dir)); \
3979 } while (0)
3980#define stat_inc_compr_inode(inode) \
3981 do { \
3982 if (f2fs_compressed_file(inode)) \
3983 (atomic_inc(&F2FS_I_SB(inode)->compr_inode)); \
3984 } while (0)
3985#define stat_dec_compr_inode(inode) \
3986 do { \
3987 if (f2fs_compressed_file(inode)) \
3988 (atomic_dec(&F2FS_I_SB(inode)->compr_inode)); \
3989 } while (0)
3990#define stat_add_compr_blocks(inode, blocks) \
3991 (atomic64_add(blocks, &F2FS_I_SB(inode)->compr_blocks))
3992#define stat_sub_compr_blocks(inode, blocks) \
3993 (atomic64_sub(blocks, &F2FS_I_SB(inode)->compr_blocks))
3994#define stat_inc_swapfile_inode(inode) \
3995 (atomic_inc(&F2FS_I_SB(inode)->swapfile_inode))
3996#define stat_dec_swapfile_inode(inode) \
3997 (atomic_dec(&F2FS_I_SB(inode)->swapfile_inode))
3998#define stat_inc_atomic_inode(inode) \
3999 (atomic_inc(&F2FS_I_SB(inode)->atomic_files))
4000#define stat_dec_atomic_inode(inode) \
4001 (atomic_dec(&F2FS_I_SB(inode)->atomic_files))
4002#define stat_inc_meta_count(sbi, blkaddr) \
4003 do { \
4004 if (blkaddr < SIT_I(sbi)->sit_base_addr) \
4005 atomic_inc(&(sbi)->meta_count[META_CP]); \
4006 else if (blkaddr < NM_I(sbi)->nat_blkaddr) \
4007 atomic_inc(&(sbi)->meta_count[META_SIT]); \
4008 else if (blkaddr < SM_I(sbi)->ssa_blkaddr) \
4009 atomic_inc(&(sbi)->meta_count[META_NAT]); \
4010 else if (blkaddr < SM_I(sbi)->main_blkaddr) \
4011 atomic_inc(&(sbi)->meta_count[META_SSA]); \
4012 } while (0)
4013#define stat_inc_seg_type(sbi, curseg) \
4014 ((sbi)->segment_count[(curseg)->alloc_type]++)
4015#define stat_inc_block_count(sbi, curseg) \
4016 ((sbi)->block_count[(curseg)->alloc_type]++)
4017#define stat_inc_inplace_blocks(sbi) \
4018 (atomic_inc(&(sbi)->inplace_count))
4019#define stat_update_max_atomic_write(inode) \
4020 do { \
4021 int cur = atomic_read(&F2FS_I_SB(inode)->atomic_files); \
4022 int max = atomic_read(&F2FS_I_SB(inode)->max_aw_cnt); \
4023 if (cur > max) \
4024 atomic_set(&F2FS_I_SB(inode)->max_aw_cnt, cur); \
4025 } while (0)
4026#define stat_inc_seg_count(sbi, type, gc_type) \
4027 do { \
4028 struct f2fs_stat_info *si = F2FS_STAT(sbi); \
4029 si->tot_segs++; \
4030 if ((type) == SUM_TYPE_DATA) { \
4031 si->data_segs++; \
4032 si->bg_data_segs += (gc_type == BG_GC) ? 1 : 0; \
4033 } else { \
4034 si->node_segs++; \
4035 si->bg_node_segs += (gc_type == BG_GC) ? 1 : 0; \
4036 } \
4037 } while (0)
4038
4039#define stat_inc_tot_blk_count(si, blks) \
4040 ((si)->tot_blks += (blks))
4041
4042#define stat_inc_data_blk_count(sbi, blks, gc_type) \
4043 do { \
4044 struct f2fs_stat_info *si = F2FS_STAT(sbi); \
4045 stat_inc_tot_blk_count(si, blks); \
4046 si->data_blks += (blks); \
4047 si->bg_data_blks += ((gc_type) == BG_GC) ? (blks) : 0; \
4048 } while (0)
4049
4050#define stat_inc_node_blk_count(sbi, blks, gc_type) \
4051 do { \
4052 struct f2fs_stat_info *si = F2FS_STAT(sbi); \
4053 stat_inc_tot_blk_count(si, blks); \
4054 si->node_blks += (blks); \
4055 si->bg_node_blks += ((gc_type) == BG_GC) ? (blks) : 0; \
4056 } while (0)
4057
4058int f2fs_build_stats(struct f2fs_sb_info *sbi);
4059void f2fs_destroy_stats(struct f2fs_sb_info *sbi);
4060void __init f2fs_create_root_stats(void);
4061void f2fs_destroy_root_stats(void);
4062void f2fs_update_sit_info(struct f2fs_sb_info *sbi);
4063#else
4064#define stat_inc_cp_count(si) do { } while (0)
4065#define stat_inc_bg_cp_count(si) do { } while (0)
4066#define stat_inc_call_count(si) do { } while (0)
4067#define stat_inc_bggc_count(si) do { } while (0)
4068#define stat_io_skip_bggc_count(sbi) do { } while (0)
4069#define stat_other_skip_bggc_count(sbi) do { } while (0)
4070#define stat_inc_dirty_inode(sbi, type) do { } while (0)
4071#define stat_dec_dirty_inode(sbi, type) do { } while (0)
4072#define stat_inc_total_hit(sbi, type) do { } while (0)
4073#define stat_inc_rbtree_node_hit(sbi, type) do { } while (0)
4074#define stat_inc_largest_node_hit(sbi) do { } while (0)
4075#define stat_inc_cached_node_hit(sbi, type) do { } while (0)
4076#define stat_inc_inline_xattr(inode) do { } while (0)
4077#define stat_dec_inline_xattr(inode) do { } while (0)
4078#define stat_inc_inline_inode(inode) do { } while (0)
4079#define stat_dec_inline_inode(inode) do { } while (0)
4080#define stat_inc_inline_dir(inode) do { } while (0)
4081#define stat_dec_inline_dir(inode) do { } while (0)
4082#define stat_inc_compr_inode(inode) do { } while (0)
4083#define stat_dec_compr_inode(inode) do { } while (0)
4084#define stat_add_compr_blocks(inode, blocks) do { } while (0)
4085#define stat_sub_compr_blocks(inode, blocks) do { } while (0)
4086#define stat_inc_swapfile_inode(inode) do { } while (0)
4087#define stat_dec_swapfile_inode(inode) do { } while (0)
4088#define stat_inc_atomic_inode(inode) do { } while (0)
4089#define stat_dec_atomic_inode(inode) do { } while (0)
4090#define stat_update_max_atomic_write(inode) do { } while (0)
4091#define stat_inc_meta_count(sbi, blkaddr) do { } while (0)
4092#define stat_inc_seg_type(sbi, curseg) do { } while (0)
4093#define stat_inc_block_count(sbi, curseg) do { } while (0)
4094#define stat_inc_inplace_blocks(sbi) do { } while (0)
4095#define stat_inc_seg_count(sbi, type, gc_type) do { } while (0)
4096#define stat_inc_tot_blk_count(si, blks) do { } while (0)
4097#define stat_inc_data_blk_count(sbi, blks, gc_type) do { } while (0)
4098#define stat_inc_node_blk_count(sbi, blks, gc_type) do { } while (0)
4099
4100static inline int f2fs_build_stats(struct f2fs_sb_info *sbi) { return 0; }
4101static inline void f2fs_destroy_stats(struct f2fs_sb_info *sbi) { }
4102static inline void __init f2fs_create_root_stats(void) { }
4103static inline void f2fs_destroy_root_stats(void) { }
4104static inline void f2fs_update_sit_info(struct f2fs_sb_info *sbi) {}
4105#endif
4106
4107extern const struct file_operations f2fs_dir_operations;
4108extern const struct file_operations f2fs_file_operations;
4109extern const struct inode_operations f2fs_file_inode_operations;
4110extern const struct address_space_operations f2fs_dblock_aops;
4111extern const struct address_space_operations f2fs_node_aops;
4112extern const struct address_space_operations f2fs_meta_aops;
4113extern const struct inode_operations f2fs_dir_inode_operations;
4114extern const struct inode_operations f2fs_symlink_inode_operations;
4115extern const struct inode_operations f2fs_encrypted_symlink_inode_operations;
4116extern const struct inode_operations f2fs_special_inode_operations;
4117extern struct kmem_cache *f2fs_inode_entry_slab;
4118
4119/*
4120 * inline.c
4121 */
4122bool f2fs_may_inline_data(struct inode *inode);
4123bool f2fs_sanity_check_inline_data(struct inode *inode);
4124bool f2fs_may_inline_dentry(struct inode *inode);
4125void f2fs_do_read_inline_data(struct page *page, struct page *ipage);
4126void f2fs_truncate_inline_inode(struct inode *inode,
4127 struct page *ipage, u64 from);
4128int f2fs_read_inline_data(struct inode *inode, struct page *page);
4129int f2fs_convert_inline_page(struct dnode_of_data *dn, struct page *page);
4130int f2fs_convert_inline_inode(struct inode *inode);
4131int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry);
4132int f2fs_write_inline_data(struct inode *inode, struct page *page);
4133int f2fs_recover_inline_data(struct inode *inode, struct page *npage);
4134struct f2fs_dir_entry *f2fs_find_in_inline_dir(struct inode *dir,
4135 const struct f2fs_filename *fname,
4136 struct page **res_page);
4137int f2fs_make_empty_inline_dir(struct inode *inode, struct inode *parent,
4138 struct page *ipage);
4139int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname,
4140 struct inode *inode, nid_t ino, umode_t mode);
4141void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry,
4142 struct page *page, struct inode *dir,
4143 struct inode *inode);
4144bool f2fs_empty_inline_dir(struct inode *dir);
4145int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx,
4146 struct fscrypt_str *fstr);
4147int f2fs_inline_data_fiemap(struct inode *inode,
4148 struct fiemap_extent_info *fieinfo,
4149 __u64 start, __u64 len);
4150
4151/*
4152 * shrinker.c
4153 */
4154unsigned long f2fs_shrink_count(struct shrinker *shrink,
4155 struct shrink_control *sc);
4156unsigned long f2fs_shrink_scan(struct shrinker *shrink,
4157 struct shrink_control *sc);
4158void f2fs_join_shrinker(struct f2fs_sb_info *sbi);
4159void f2fs_leave_shrinker(struct f2fs_sb_info *sbi);
4160
4161/*
4162 * extent_cache.c
4163 */
4164struct rb_entry *f2fs_lookup_rb_tree(struct rb_root_cached *root,
4165 struct rb_entry *cached_re, unsigned int ofs);
4166struct rb_node **f2fs_lookup_rb_tree_ext(struct f2fs_sb_info *sbi,
4167 struct rb_root_cached *root,
4168 struct rb_node **parent,
4169 unsigned long long key, bool *left_most);
4170struct rb_node **f2fs_lookup_rb_tree_for_insert(struct f2fs_sb_info *sbi,
4171 struct rb_root_cached *root,
4172 struct rb_node **parent,
4173 unsigned int ofs, bool *leftmost);
4174struct rb_entry *f2fs_lookup_rb_tree_ret(struct rb_root_cached *root,
4175 struct rb_entry *cached_re, unsigned int ofs,
4176 struct rb_entry **prev_entry, struct rb_entry **next_entry,
4177 struct rb_node ***insert_p, struct rb_node **insert_parent,
4178 bool force, bool *leftmost);
4179bool f2fs_check_rb_tree_consistence(struct f2fs_sb_info *sbi,
4180 struct rb_root_cached *root, bool check_key);
4181void f2fs_init_extent_tree(struct inode *inode);
4182void f2fs_drop_extent_tree(struct inode *inode);
4183void f2fs_destroy_extent_node(struct inode *inode);
4184void f2fs_destroy_extent_tree(struct inode *inode);
4185void f2fs_init_extent_cache_info(struct f2fs_sb_info *sbi);
4186int __init f2fs_create_extent_cache(void);
4187void f2fs_destroy_extent_cache(void);
4188
4189/* read extent cache ops */
4190void f2fs_init_read_extent_tree(struct inode *inode, struct page *ipage);
4191bool f2fs_lookup_read_extent_cache(struct inode *inode, pgoff_t pgofs,
4192 struct extent_info *ei);
4193void f2fs_update_read_extent_cache(struct dnode_of_data *dn);
4194void f2fs_update_read_extent_cache_range(struct dnode_of_data *dn,
4195 pgoff_t fofs, block_t blkaddr, unsigned int len);
4196unsigned int f2fs_shrink_read_extent_tree(struct f2fs_sb_info *sbi,
4197 int nr_shrink);
4198
4199/* block age extent cache ops */
4200void f2fs_init_age_extent_tree(struct inode *inode);
4201bool f2fs_lookup_age_extent_cache(struct inode *inode, pgoff_t pgofs,
4202 struct extent_info *ei);
4203void f2fs_update_age_extent_cache(struct dnode_of_data *dn);
4204void f2fs_update_age_extent_cache_range(struct dnode_of_data *dn,
4205 pgoff_t fofs, unsigned int len);
4206unsigned int f2fs_shrink_age_extent_tree(struct f2fs_sb_info *sbi,
4207 int nr_shrink);
4208
4209/*
4210 * sysfs.c
4211 */
4212#define MIN_RA_MUL 2
4213#define MAX_RA_MUL 256
4214
4215int __init f2fs_init_sysfs(void);
4216void f2fs_exit_sysfs(void);
4217int f2fs_register_sysfs(struct f2fs_sb_info *sbi);
4218void f2fs_unregister_sysfs(struct f2fs_sb_info *sbi);
4219
4220/* verity.c */
4221extern const struct fsverity_operations f2fs_verityops;
4222
4223/*
4224 * crypto support
4225 */
4226static inline bool f2fs_encrypted_file(struct inode *inode)
4227{
4228 return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode);
4229}
4230
4231static inline void f2fs_set_encrypted_inode(struct inode *inode)
4232{
4233#ifdef CONFIG_FS_ENCRYPTION
4234 file_set_encrypt(inode);
4235 f2fs_set_inode_flags(inode);
4236#endif
4237}
4238
4239/*
4240 * Returns true if the reads of the inode's data need to undergo some
4241 * postprocessing step, like decryption or authenticity verification.
4242 */
4243static inline bool f2fs_post_read_required(struct inode *inode)
4244{
4245 return f2fs_encrypted_file(inode) || fsverity_active(inode) ||
4246 f2fs_compressed_file(inode);
4247}
4248
4249/*
4250 * compress.c
4251 */
4252#ifdef CONFIG_F2FS_FS_COMPRESSION
4253bool f2fs_is_compressed_page(struct page *page);
4254struct page *f2fs_compress_control_page(struct page *page);
4255int f2fs_prepare_compress_overwrite(struct inode *inode,
4256 struct page **pagep, pgoff_t index, void **fsdata);
4257bool f2fs_compress_write_end(struct inode *inode, void *fsdata,
4258 pgoff_t index, unsigned copied);
4259int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock);
4260void f2fs_compress_write_end_io(struct bio *bio, struct page *page);
4261bool f2fs_is_compress_backend_ready(struct inode *inode);
4262int f2fs_init_compress_mempool(void);
4263void f2fs_destroy_compress_mempool(void);
4264void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task);
4265void f2fs_end_read_compressed_page(struct page *page, bool failed,
4266 block_t blkaddr, bool in_task);
4267bool f2fs_cluster_is_empty(struct compress_ctx *cc);
4268bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index);
4269bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages,
4270 int index, int nr_pages, bool uptodate);
4271bool f2fs_sanity_check_cluster(struct dnode_of_data *dn);
4272void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page);
4273int f2fs_write_multi_pages(struct compress_ctx *cc,
4274 int *submitted,
4275 struct writeback_control *wbc,
4276 enum iostat_type io_type);
4277int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index);
4278void f2fs_update_read_extent_tree_range_compressed(struct inode *inode,
4279 pgoff_t fofs, block_t blkaddr,
4280 unsigned int llen, unsigned int c_len);
4281int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
4282 unsigned nr_pages, sector_t *last_block_in_bio,
4283 bool is_readahead, bool for_write);
4284struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc);
4285void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed,
4286 bool in_task);
4287void f2fs_put_page_dic(struct page *page, bool in_task);
4288unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn);
4289int f2fs_init_compress_ctx(struct compress_ctx *cc);
4290void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse);
4291void f2fs_init_compress_info(struct f2fs_sb_info *sbi);
4292int f2fs_init_compress_inode(struct f2fs_sb_info *sbi);
4293void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi);
4294int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi);
4295void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi);
4296int __init f2fs_init_compress_cache(void);
4297void f2fs_destroy_compress_cache(void);
4298struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi);
4299void f2fs_invalidate_compress_page(struct f2fs_sb_info *sbi, block_t blkaddr);
4300void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, struct page *page,
4301 nid_t ino, block_t blkaddr);
4302bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi, struct page *page,
4303 block_t blkaddr);
4304void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino);
4305#define inc_compr_inode_stat(inode) \
4306 do { \
4307 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); \
4308 sbi->compr_new_inode++; \
4309 } while (0)
4310#define add_compr_block_stat(inode, blocks) \
4311 do { \
4312 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); \
4313 int diff = F2FS_I(inode)->i_cluster_size - blocks; \
4314 sbi->compr_written_block += blocks; \
4315 sbi->compr_saved_block += diff; \
4316 } while (0)
4317#else
4318static inline bool f2fs_is_compressed_page(struct page *page) { return false; }
4319static inline bool f2fs_is_compress_backend_ready(struct inode *inode)
4320{
4321 if (!f2fs_compressed_file(inode))
4322 return true;
4323 /* not support compression */
4324 return false;
4325}
4326static inline struct page *f2fs_compress_control_page(struct page *page)
4327{
4328 WARN_ON_ONCE(1);
4329 return ERR_PTR(-EINVAL);
4330}
4331static inline int f2fs_init_compress_mempool(void) { return 0; }
4332static inline void f2fs_destroy_compress_mempool(void) { }
4333static inline void f2fs_decompress_cluster(struct decompress_io_ctx *dic,
4334 bool in_task) { }
4335static inline void f2fs_end_read_compressed_page(struct page *page,
4336 bool failed, block_t blkaddr, bool in_task)
4337{
4338 WARN_ON_ONCE(1);
4339}
4340static inline void f2fs_put_page_dic(struct page *page, bool in_task)
4341{
4342 WARN_ON_ONCE(1);
4343}
4344static inline unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn) { return 0; }
4345static inline bool f2fs_sanity_check_cluster(struct dnode_of_data *dn) { return false; }
4346static inline int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) { return 0; }
4347static inline void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) { }
4348static inline int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) { return 0; }
4349static inline void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) { }
4350static inline int __init f2fs_init_compress_cache(void) { return 0; }
4351static inline void f2fs_destroy_compress_cache(void) { }
4352static inline void f2fs_invalidate_compress_page(struct f2fs_sb_info *sbi,
4353 block_t blkaddr) { }
4354static inline void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi,
4355 struct page *page, nid_t ino, block_t blkaddr) { }
4356static inline bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi,
4357 struct page *page, block_t blkaddr) { return false; }
4358static inline void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi,
4359 nid_t ino) { }
4360#define inc_compr_inode_stat(inode) do { } while (0)
4361static inline void f2fs_update_read_extent_tree_range_compressed(
4362 struct inode *inode,
4363 pgoff_t fofs, block_t blkaddr,
4364 unsigned int llen, unsigned int c_len) { }
4365#endif
4366
4367static inline int set_compress_context(struct inode *inode)
4368{
4369#ifdef CONFIG_F2FS_FS_COMPRESSION
4370 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
4371
4372 F2FS_I(inode)->i_compress_algorithm =
4373 F2FS_OPTION(sbi).compress_algorithm;
4374 F2FS_I(inode)->i_log_cluster_size =
4375 F2FS_OPTION(sbi).compress_log_size;
4376 F2FS_I(inode)->i_compress_flag =
4377 F2FS_OPTION(sbi).compress_chksum ?
4378 1 << COMPRESS_CHKSUM : 0;
4379 F2FS_I(inode)->i_cluster_size =
4380 1 << F2FS_I(inode)->i_log_cluster_size;
4381 if ((F2FS_I(inode)->i_compress_algorithm == COMPRESS_LZ4 ||
4382 F2FS_I(inode)->i_compress_algorithm == COMPRESS_ZSTD) &&
4383 F2FS_OPTION(sbi).compress_level)
4384 F2FS_I(inode)->i_compress_flag |=
4385 F2FS_OPTION(sbi).compress_level <<
4386 COMPRESS_LEVEL_OFFSET;
4387 F2FS_I(inode)->i_flags |= F2FS_COMPR_FL;
4388 set_inode_flag(inode, FI_COMPRESSED_FILE);
4389 stat_inc_compr_inode(inode);
4390 inc_compr_inode_stat(inode);
4391 f2fs_mark_inode_dirty_sync(inode, true);
4392 return 0;
4393#else
4394 return -EOPNOTSUPP;
4395#endif
4396}
4397
4398static inline bool f2fs_disable_compressed_file(struct inode *inode)
4399{
4400 struct f2fs_inode_info *fi = F2FS_I(inode);
4401
4402 if (!f2fs_compressed_file(inode))
4403 return true;
4404 if (S_ISREG(inode->i_mode) && F2FS_HAS_BLOCKS(inode))
4405 return false;
4406
4407 fi->i_flags &= ~F2FS_COMPR_FL;
4408 stat_dec_compr_inode(inode);
4409 clear_inode_flag(inode, FI_COMPRESSED_FILE);
4410 f2fs_mark_inode_dirty_sync(inode, true);
4411 return true;
4412}
4413
4414#define F2FS_FEATURE_FUNCS(name, flagname) \
4415static inline bool f2fs_sb_has_##name(struct f2fs_sb_info *sbi) \
4416{ \
4417 return F2FS_HAS_FEATURE(sbi, F2FS_FEATURE_##flagname); \
4418}
4419
4420F2FS_FEATURE_FUNCS(encrypt, ENCRYPT);
4421F2FS_FEATURE_FUNCS(blkzoned, BLKZONED);
4422F2FS_FEATURE_FUNCS(extra_attr, EXTRA_ATTR);
4423F2FS_FEATURE_FUNCS(project_quota, PRJQUOTA);
4424F2FS_FEATURE_FUNCS(inode_chksum, INODE_CHKSUM);
4425F2FS_FEATURE_FUNCS(flexible_inline_xattr, FLEXIBLE_INLINE_XATTR);
4426F2FS_FEATURE_FUNCS(quota_ino, QUOTA_INO);
4427F2FS_FEATURE_FUNCS(inode_crtime, INODE_CRTIME);
4428F2FS_FEATURE_FUNCS(lost_found, LOST_FOUND);
4429F2FS_FEATURE_FUNCS(verity, VERITY);
4430F2FS_FEATURE_FUNCS(sb_chksum, SB_CHKSUM);
4431F2FS_FEATURE_FUNCS(casefold, CASEFOLD);
4432F2FS_FEATURE_FUNCS(compression, COMPRESSION);
4433F2FS_FEATURE_FUNCS(readonly, RO);
4434
4435#ifdef CONFIG_BLK_DEV_ZONED
4436static inline bool f2fs_blkz_is_seq(struct f2fs_sb_info *sbi, int devi,
4437 block_t blkaddr)
4438{
4439 unsigned int zno = blkaddr >> sbi->log_blocks_per_blkz;
4440
4441 return test_bit(zno, FDEV(devi).blkz_seq);
4442}
4443#endif
4444
4445static inline bool f2fs_hw_should_discard(struct f2fs_sb_info *sbi)
4446{
4447 return f2fs_sb_has_blkzoned(sbi);
4448}
4449
4450static inline bool f2fs_bdev_support_discard(struct block_device *bdev)
4451{
4452 return bdev_max_discard_sectors(bdev) || bdev_is_zoned(bdev);
4453}
4454
4455static inline bool f2fs_hw_support_discard(struct f2fs_sb_info *sbi)
4456{
4457 int i;
4458
4459 if (!f2fs_is_multi_device(sbi))
4460 return f2fs_bdev_support_discard(sbi->sb->s_bdev);
4461
4462 for (i = 0; i < sbi->s_ndevs; i++)
4463 if (f2fs_bdev_support_discard(FDEV(i).bdev))
4464 return true;
4465 return false;
4466}
4467
4468static inline bool f2fs_realtime_discard_enable(struct f2fs_sb_info *sbi)
4469{
4470 return (test_opt(sbi, DISCARD) && f2fs_hw_support_discard(sbi)) ||
4471 f2fs_hw_should_discard(sbi);
4472}
4473
4474static inline bool f2fs_hw_is_readonly(struct f2fs_sb_info *sbi)
4475{
4476 int i;
4477
4478 if (!f2fs_is_multi_device(sbi))
4479 return bdev_read_only(sbi->sb->s_bdev);
4480
4481 for (i = 0; i < sbi->s_ndevs; i++)
4482 if (bdev_read_only(FDEV(i).bdev))
4483 return true;
4484 return false;
4485}
4486
4487static inline bool f2fs_lfs_mode(struct f2fs_sb_info *sbi)
4488{
4489 return F2FS_OPTION(sbi).fs_mode == FS_MODE_LFS;
4490}
4491
4492static inline bool f2fs_low_mem_mode(struct f2fs_sb_info *sbi)
4493{
4494 return F2FS_OPTION(sbi).memory_mode == MEMORY_MODE_LOW;
4495}
4496
4497static inline bool f2fs_may_compress(struct inode *inode)
4498{
4499 if (IS_SWAPFILE(inode) || f2fs_is_pinned_file(inode) ||
4500 f2fs_is_atomic_file(inode) || f2fs_has_inline_data(inode))
4501 return false;
4502 return S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode);
4503}
4504
4505static inline void f2fs_i_compr_blocks_update(struct inode *inode,
4506 u64 blocks, bool add)
4507{
4508 struct f2fs_inode_info *fi = F2FS_I(inode);
4509 int diff = fi->i_cluster_size - blocks;
4510
4511 /* don't update i_compr_blocks if saved blocks were released */
4512 if (!add && !atomic_read(&fi->i_compr_blocks))
4513 return;
4514
4515 if (add) {
4516 atomic_add(diff, &fi->i_compr_blocks);
4517 stat_add_compr_blocks(inode, diff);
4518 } else {
4519 atomic_sub(diff, &fi->i_compr_blocks);
4520 stat_sub_compr_blocks(inode, diff);
4521 }
4522 f2fs_mark_inode_dirty_sync(inode, true);
4523}
4524
4525static inline bool f2fs_allow_multi_device_dio(struct f2fs_sb_info *sbi,
4526 int flag)
4527{
4528 if (!f2fs_is_multi_device(sbi))
4529 return false;
4530 if (flag != F2FS_GET_BLOCK_DIO)
4531 return false;
4532 return sbi->aligned_blksize;
4533}
4534
4535static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx)
4536{
4537 return fsverity_active(inode) &&
4538 idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
4539}
4540
4541#ifdef CONFIG_F2FS_FAULT_INJECTION
4542extern void f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned int rate,
4543 unsigned int type);
4544#else
4545#define f2fs_build_fault_attr(sbi, rate, type) do { } while (0)
4546#endif
4547
4548static inline bool is_journalled_quota(struct f2fs_sb_info *sbi)
4549{
4550#ifdef CONFIG_QUOTA
4551 if (f2fs_sb_has_quota_ino(sbi))
4552 return true;
4553 if (F2FS_OPTION(sbi).s_qf_names[USRQUOTA] ||
4554 F2FS_OPTION(sbi).s_qf_names[GRPQUOTA] ||
4555 F2FS_OPTION(sbi).s_qf_names[PRJQUOTA])
4556 return true;
4557#endif
4558 return false;
4559}
4560
4561static inline bool f2fs_block_unit_discard(struct f2fs_sb_info *sbi)
4562{
4563 return F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_BLOCK;
4564}
4565
4566static inline void f2fs_io_schedule_timeout(long timeout)
4567{
4568 set_current_state(TASK_UNINTERRUPTIBLE);
4569 io_schedule_timeout(timeout);
4570}
4571
4572static inline void f2fs_handle_page_eio(struct f2fs_sb_info *sbi, pgoff_t ofs,
4573 enum page_type type)
4574{
4575 if (unlikely(f2fs_cp_error(sbi)))
4576 return;
4577
4578 if (ofs == sbi->page_eio_ofs[type]) {
4579 if (sbi->page_eio_cnt[type]++ == MAX_RETRY_PAGE_EIO)
4580 set_ckpt_flags(sbi, CP_ERROR_FLAG);
4581 } else {
4582 sbi->page_eio_ofs[type] = ofs;
4583 sbi->page_eio_cnt[type] = 0;
4584 }
4585}
4586
4587static inline bool f2fs_is_readonly(struct f2fs_sb_info *sbi)
4588{
4589 return f2fs_sb_has_readonly(sbi) || f2fs_readonly(sbi->sb);
4590}
4591
4592#define EFSBADCRC EBADMSG /* Bad CRC detected */
4593#define EFSCORRUPTED EUCLEAN /* Filesystem is corrupted */
4594
4595#endif /* _LINUX_F2FS_H */