1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * (C) 1997 Linus Torvalds
   4 * (C) 1999 Andrea Arcangeli <andrea@suse.de> (dynamic inode allocation)
   5 */
   6#include <linux/export.h>
   7#include <linux/fs.h>
   8#include <linux/mm.h>
   9#include <linux/backing-dev.h>
  10#include <linux/hash.h>
  11#include <linux/swap.h>
  12#include <linux/security.h>
  13#include <linux/cdev.h>
  14#include <linux/memblock.h>
  15#include <linux/fscrypt.h>
  16#include <linux/fsnotify.h>
  17#include <linux/mount.h>
  18#include <linux/posix_acl.h>
  19#include <linux/prefetch.h>
  20#include <linux/buffer_head.h> /* for inode_has_buffers */
  21#include <linux/ratelimit.h>
  22#include <linux/list_lru.h>
  23#include <linux/iversion.h>
  24#include <trace/events/writeback.h>
  25#include "internal.h"
  26
  27/*
  28 * Inode locking rules:
  29 *
  30 * inode->i_lock protects:
  31 *   inode->i_state, inode->i_hash, __iget()
  32 * Inode LRU list locks protect:
  33 *   inode->i_sb->s_inode_lru, inode->i_lru
  34 * inode->i_sb->s_inode_list_lock protects:
  35 *   inode->i_sb->s_inodes, inode->i_sb_list
  36 * bdi->wb.list_lock protects:
  37 *   bdi->wb.b_{dirty,io,more_io,dirty_time}, inode->i_io_list
  38 * inode_hash_lock protects:
  39 *   inode_hashtable, inode->i_hash
  40 *
  41 * Lock ordering:
  42 *
  43 * inode->i_sb->s_inode_list_lock
  44 *   inode->i_lock
  45 *     Inode LRU list locks
  46 *
  47 * bdi->wb.list_lock
  48 *   inode->i_lock
  49 *
  50 * inode_hash_lock
  51 *   inode->i_sb->s_inode_list_lock
  52 *   inode->i_lock
  53 *
  54 * iunique_lock
  55 *   inode_hash_lock
  56 */
  57
  58static unsigned int i_hash_mask __read_mostly;
  59static unsigned int i_hash_shift __read_mostly;
  60static struct hlist_head *inode_hashtable __read_mostly;
  61static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock);
  62
 
 
  63/*
  64 * Empty aops. Can be used for the cases where the user does not
  65 * define any of the address_space operations.
  66 */
  67const struct address_space_operations empty_aops = {
  68};
  69EXPORT_SYMBOL(empty_aops);
  70
  71/*
  72 * Statistics gathering..
  73 */
  74struct inodes_stat_t inodes_stat;
  75
  76static DEFINE_PER_CPU(unsigned long, nr_inodes);
  77static DEFINE_PER_CPU(unsigned long, nr_unused);
  78
  79static struct kmem_cache *inode_cachep __read_mostly;
  80
  81static long get_nr_inodes(void)
  82{
  83	int i;
  84	long sum = 0;
  85	for_each_possible_cpu(i)
  86		sum += per_cpu(nr_inodes, i);
  87	return sum < 0 ? 0 : sum;
  88}
  89
  90static inline long get_nr_inodes_unused(void)
  91{
  92	int i;
  93	long sum = 0;
  94	for_each_possible_cpu(i)
  95		sum += per_cpu(nr_unused, i);
  96	return sum < 0 ? 0 : sum;
  97}
  98
  99long get_nr_dirty_inodes(void)
 100{
 101	/* not actually dirty inodes, but a wild approximation */
 102	long nr_dirty = get_nr_inodes() - get_nr_inodes_unused();
 103	return nr_dirty > 0 ? nr_dirty : 0;
 104}
 105
 106/*
 107 * Handle nr_inode sysctl
 108 */
 109#ifdef CONFIG_SYSCTL
 110int proc_nr_inodes(struct ctl_table *table, int write,
 111		   void *buffer, size_t *lenp, loff_t *ppos)
 112{
 113	inodes_stat.nr_inodes = get_nr_inodes();
 114	inodes_stat.nr_unused = get_nr_inodes_unused();
 115	return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
 116}
 117#endif
 118
 119static int no_open(struct inode *inode, struct file *file)
 120{
 121	return -ENXIO;
 122}
 123
 124/**
 125 * inode_init_always - perform inode structure initialisation
 126 * @sb: superblock inode belongs to
 127 * @inode: inode to initialise
 128 *
 129 * These are initializations that need to be done on every inode
 130 * allocation as the fields are not initialised by slab allocation.
 131 */
 132int inode_init_always(struct super_block *sb, struct inode *inode)
 133{
 134	static const struct inode_operations empty_iops;
 135	static const struct file_operations no_open_fops = {.open = no_open};
 136	struct address_space *const mapping = &inode->i_data;
 137
 138	inode->i_sb = sb;
 139	inode->i_blkbits = sb->s_blocksize_bits;
 140	inode->i_flags = 0;
 141	atomic64_set(&inode->i_sequence, 0);
 142	atomic_set(&inode->i_count, 1);
 143	inode->i_op = &empty_iops;
 144	inode->i_fop = &no_open_fops;
 145	inode->__i_nlink = 1;
 146	inode->i_opflags = 0;
 147	if (sb->s_xattr)
 148		inode->i_opflags |= IOP_XATTR;
 149	i_uid_write(inode, 0);
 150	i_gid_write(inode, 0);
 151	atomic_set(&inode->i_writecount, 0);
 152	inode->i_size = 0;
 153	inode->i_write_hint = WRITE_LIFE_NOT_SET;
 154	inode->i_blocks = 0;
 155	inode->i_bytes = 0;
 156	inode->i_generation = 0;
 
 
 
 157	inode->i_pipe = NULL;
 158	inode->i_bdev = NULL;
 159	inode->i_cdev = NULL;
 160	inode->i_link = NULL;
 161	inode->i_dir_seq = 0;
 162	inode->i_rdev = 0;
 163	inode->dirtied_when = 0;
 164
 165#ifdef CONFIG_CGROUP_WRITEBACK
 166	inode->i_wb_frn_winner = 0;
 167	inode->i_wb_frn_avg_time = 0;
 168	inode->i_wb_frn_history = 0;
 169#endif
 170
 171	if (security_inode_alloc(inode))
 172		goto out;
 173	spin_lock_init(&inode->i_lock);
 174	lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key);
 175
 176	init_rwsem(&inode->i_rwsem);
 177	lockdep_set_class(&inode->i_rwsem, &sb->s_type->i_mutex_key);
 178
 179	atomic_set(&inode->i_dio_count, 0);
 180
 181	mapping->a_ops = &empty_aops;
 182	mapping->host = inode;
 183	mapping->flags = 0;
 184	mapping->wb_err = 0;
 185	atomic_set(&mapping->i_mmap_writable, 0);
 186#ifdef CONFIG_READ_ONLY_THP_FOR_FS
 187	atomic_set(&mapping->nr_thps, 0);
 188#endif
 189	mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE);
 190	mapping->private_data = NULL;
 
 191	mapping->writeback_index = 0;
 
 
 
 
 
 
 
 
 
 
 
 
 192	inode->i_private = NULL;
 193	inode->i_mapping = mapping;
 194	INIT_HLIST_HEAD(&inode->i_dentry);	/* buggered by rcu freeing */
 195#ifdef CONFIG_FS_POSIX_ACL
 196	inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED;
 197#endif
 198
 199#ifdef CONFIG_FSNOTIFY
 200	inode->i_fsnotify_mask = 0;
 201#endif
 202	inode->i_flctx = NULL;
 203	this_cpu_inc(nr_inodes);
 204
 205	return 0;
 206out:
 207	return -ENOMEM;
 208}
 209EXPORT_SYMBOL(inode_init_always);
 210
 211void free_inode_nonrcu(struct inode *inode)
 212{
 213	kmem_cache_free(inode_cachep, inode);
 214}
 215EXPORT_SYMBOL(free_inode_nonrcu);
 216
 217static void i_callback(struct rcu_head *head)
 218{
 219	struct inode *inode = container_of(head, struct inode, i_rcu);
 220	if (inode->free_inode)
 221		inode->free_inode(inode);
 222	else
 223		free_inode_nonrcu(inode);
 224}
 225
 226static struct inode *alloc_inode(struct super_block *sb)
 227{
 228	const struct super_operations *ops = sb->s_op;
 229	struct inode *inode;
 230
 231	if (ops->alloc_inode)
 232		inode = ops->alloc_inode(sb);
 233	else
 234		inode = kmem_cache_alloc(inode_cachep, GFP_KERNEL);
 235
 236	if (!inode)
 237		return NULL;
 238
 239	if (unlikely(inode_init_always(sb, inode))) {
 240		if (ops->destroy_inode) {
 241			ops->destroy_inode(inode);
 242			if (!ops->free_inode)
 243				return NULL;
 244		}
 245		inode->free_inode = ops->free_inode;
 246		i_callback(&inode->i_rcu);
 247		return NULL;
 248	}
 249
 250	return inode;
 251}
 252
 
 
 
 
 
 
 253void __destroy_inode(struct inode *inode)
 254{
 255	BUG_ON(inode_has_buffers(inode));
 256	inode_detach_wb(inode);
 257	security_inode_free(inode);
 258	fsnotify_inode_delete(inode);
 259	locks_free_lock_context(inode);
 260	if (!inode->i_nlink) {
 261		WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0);
 262		atomic_long_dec(&inode->i_sb->s_remove_count);
 263	}
 264
 265#ifdef CONFIG_FS_POSIX_ACL
 266	if (inode->i_acl && !is_uncached_acl(inode->i_acl))
 267		posix_acl_release(inode->i_acl);
 268	if (inode->i_default_acl && !is_uncached_acl(inode->i_default_acl))
 269		posix_acl_release(inode->i_default_acl);
 270#endif
 271	this_cpu_dec(nr_inodes);
 272}
 273EXPORT_SYMBOL(__destroy_inode);
 274
 275static void destroy_inode(struct inode *inode)
 276{
 277	const struct super_operations *ops = inode->i_sb->s_op;
 
 
 278
 
 
 279	BUG_ON(!list_empty(&inode->i_lru));
 280	__destroy_inode(inode);
 281	if (ops->destroy_inode) {
 282		ops->destroy_inode(inode);
 283		if (!ops->free_inode)
 284			return;
 285	}
 286	inode->free_inode = ops->free_inode;
 287	call_rcu(&inode->i_rcu, i_callback);
 288}
 289
 290/**
 291 * drop_nlink - directly drop an inode's link count
 292 * @inode: inode
 293 *
 294 * This is a low-level filesystem helper to replace any
 295 * direct filesystem manipulation of i_nlink.  In cases
 296 * where we are attempting to track writes to the
 297 * filesystem, a decrement to zero means an imminent
 298 * write when the file is truncated and actually unlinked
 299 * on the filesystem.
 300 */
 301void drop_nlink(struct inode *inode)
 302{
 303	WARN_ON(inode->i_nlink == 0);
 304	inode->__i_nlink--;
 305	if (!inode->i_nlink)
 306		atomic_long_inc(&inode->i_sb->s_remove_count);
 307}
 308EXPORT_SYMBOL(drop_nlink);
 309
 310/**
 311 * clear_nlink - directly zero an inode's link count
 312 * @inode: inode
 313 *
 314 * This is a low-level filesystem helper to replace any
 315 * direct filesystem manipulation of i_nlink.  See
 316 * drop_nlink() for why we care about i_nlink hitting zero.
 317 */
 318void clear_nlink(struct inode *inode)
 319{
 320	if (inode->i_nlink) {
 321		inode->__i_nlink = 0;
 322		atomic_long_inc(&inode->i_sb->s_remove_count);
 323	}
 324}
 325EXPORT_SYMBOL(clear_nlink);
 326
 327/**
 328 * set_nlink - directly set an inode's link count
 329 * @inode: inode
 330 * @nlink: new nlink (should be non-zero)
 331 *
 332 * This is a low-level filesystem helper to replace any
 333 * direct filesystem manipulation of i_nlink.
 334 */
 335void set_nlink(struct inode *inode, unsigned int nlink)
 336{
 337	if (!nlink) {
 338		clear_nlink(inode);
 339	} else {
 340		/* Yes, some filesystems do change nlink from zero to one */
 341		if (inode->i_nlink == 0)
 342			atomic_long_dec(&inode->i_sb->s_remove_count);
 343
 344		inode->__i_nlink = nlink;
 345	}
 346}
 347EXPORT_SYMBOL(set_nlink);
 348
 349/**
 350 * inc_nlink - directly increment an inode's link count
 351 * @inode: inode
 352 *
 353 * This is a low-level filesystem helper to replace any
 354 * direct filesystem manipulation of i_nlink.  Currently,
 355 * it is only here for parity with dec_nlink().
 356 */
 357void inc_nlink(struct inode *inode)
 358{
 359	if (unlikely(inode->i_nlink == 0)) {
 360		WARN_ON(!(inode->i_state & I_LINKABLE));
 361		atomic_long_dec(&inode->i_sb->s_remove_count);
 362	}
 363
 364	inode->__i_nlink++;
 365}
 366EXPORT_SYMBOL(inc_nlink);
 367
 368static void __address_space_init_once(struct address_space *mapping)
 369{
 370	xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ | XA_FLAGS_ACCOUNT);
 371	init_rwsem(&mapping->i_mmap_rwsem);
 372	INIT_LIST_HEAD(&mapping->private_list);
 373	spin_lock_init(&mapping->private_lock);
 374	mapping->i_mmap = RB_ROOT_CACHED;
 375}
 376
 377void address_space_init_once(struct address_space *mapping)
 378{
 379	memset(mapping, 0, sizeof(*mapping));
 380	__address_space_init_once(mapping);
 
 
 
 
 
 
 381}
 382EXPORT_SYMBOL(address_space_init_once);
 383
 384/*
 385 * These are initializations that only need to be done
 386 * once, because the fields are idempotent across use
 387 * of the inode, so let the slab aware of that.
 388 */
 389void inode_init_once(struct inode *inode)
 390{
 391	memset(inode, 0, sizeof(*inode));
 392	INIT_HLIST_NODE(&inode->i_hash);
 393	INIT_LIST_HEAD(&inode->i_devices);
 394	INIT_LIST_HEAD(&inode->i_io_list);
 395	INIT_LIST_HEAD(&inode->i_wb_list);
 396	INIT_LIST_HEAD(&inode->i_lru);
 397	__address_space_init_once(&inode->i_data);
 398	i_size_ordered_init(inode);
 
 
 
 399}
 400EXPORT_SYMBOL(inode_init_once);
 401
 402static void init_once(void *foo)
 403{
 404	struct inode *inode = (struct inode *) foo;
 405
 406	inode_init_once(inode);
 407}
 408
 409/*
 410 * inode->i_lock must be held
 411 */
 412void __iget(struct inode *inode)
 413{
 414	atomic_inc(&inode->i_count);
 415}
 416
 417/*
 418 * get additional reference to inode; caller must already hold one.
 419 */
 420void ihold(struct inode *inode)
 421{
 422	WARN_ON(atomic_inc_return(&inode->i_count) < 2);
 423}
 424EXPORT_SYMBOL(ihold);
 425
 426static void inode_lru_list_add(struct inode *inode)
 427{
 428	if (list_lru_add(&inode->i_sb->s_inode_lru, &inode->i_lru))
 
 
 
 429		this_cpu_inc(nr_unused);
 430	else
 431		inode->i_state |= I_REFERENCED;
 432}
 433
 434/*
 435 * Add inode to LRU if needed (inode is unused and clean).
 436 *
 437 * Needs inode->i_lock held.
 438 */
 439void inode_add_lru(struct inode *inode)
 440{
 441	if (!(inode->i_state & (I_DIRTY_ALL | I_SYNC |
 442				I_FREEING | I_WILL_FREE)) &&
 443	    !atomic_read(&inode->i_count) && inode->i_sb->s_flags & SB_ACTIVE)
 444		inode_lru_list_add(inode);
 445}
 446
 447
 448static void inode_lru_list_del(struct inode *inode)
 449{
 450
 451	if (list_lru_del(&inode->i_sb->s_inode_lru, &inode->i_lru))
 
 
 452		this_cpu_dec(nr_unused);
 
 
 453}
 454
 455/**
 456 * inode_sb_list_add - add inode to the superblock list of inodes
 457 * @inode: inode to add
 458 */
 459void inode_sb_list_add(struct inode *inode)
 460{
 461	spin_lock(&inode->i_sb->s_inode_list_lock);
 462	list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
 463	spin_unlock(&inode->i_sb->s_inode_list_lock);
 464}
 465EXPORT_SYMBOL_GPL(inode_sb_list_add);
 466
 467static inline void inode_sb_list_del(struct inode *inode)
 468{
 469	if (!list_empty(&inode->i_sb_list)) {
 470		spin_lock(&inode->i_sb->s_inode_list_lock);
 471		list_del_init(&inode->i_sb_list);
 472		spin_unlock(&inode->i_sb->s_inode_list_lock);
 473	}
 474}
 475
 476static unsigned long hash(struct super_block *sb, unsigned long hashval)
 477{
 478	unsigned long tmp;
 479
 480	tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) /
 481			L1_CACHE_BYTES;
 482	tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> i_hash_shift);
 483	return tmp & i_hash_mask;
 484}
 485
 486/**
 487 *	__insert_inode_hash - hash an inode
 488 *	@inode: unhashed inode
 489 *	@hashval: unsigned long value used to locate this object in the
 490 *		inode_hashtable.
 491 *
 492 *	Add an inode to the inode hash for this superblock.
 493 */
 494void __insert_inode_hash(struct inode *inode, unsigned long hashval)
 495{
 496	struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval);
 497
 498	spin_lock(&inode_hash_lock);
 499	spin_lock(&inode->i_lock);
 500	hlist_add_head_rcu(&inode->i_hash, b);
 501	spin_unlock(&inode->i_lock);
 502	spin_unlock(&inode_hash_lock);
 503}
 504EXPORT_SYMBOL(__insert_inode_hash);
 505
 506/**
 507 *	__remove_inode_hash - remove an inode from the hash
 508 *	@inode: inode to unhash
 509 *
 510 *	Remove an inode from the superblock.
 511 */
 512void __remove_inode_hash(struct inode *inode)
 513{
 514	spin_lock(&inode_hash_lock);
 515	spin_lock(&inode->i_lock);
 516	hlist_del_init_rcu(&inode->i_hash);
 517	spin_unlock(&inode->i_lock);
 518	spin_unlock(&inode_hash_lock);
 519}
 520EXPORT_SYMBOL(__remove_inode_hash);
 521
 522void clear_inode(struct inode *inode)
 523{
 
 524	/*
 525	 * We have to cycle the i_pages lock here because reclaim can be in the
 526	 * process of removing the last page (in __delete_from_page_cache())
 527	 * and we must not free the mapping under it.
 528	 */
 529	xa_lock_irq(&inode->i_data.i_pages);
 530	BUG_ON(inode->i_data.nrpages);
 531	BUG_ON(inode->i_data.nrexceptional);
 532	xa_unlock_irq(&inode->i_data.i_pages);
 533	BUG_ON(!list_empty(&inode->i_data.private_list));
 534	BUG_ON(!(inode->i_state & I_FREEING));
 535	BUG_ON(inode->i_state & I_CLEAR);
 536	BUG_ON(!list_empty(&inode->i_wb_list));
 537	/* don't need i_lock here, no concurrent mods to i_state */
 538	inode->i_state = I_FREEING | I_CLEAR;
 539}
 540EXPORT_SYMBOL(clear_inode);
 541
 542/*
 543 * Free the inode passed in, removing it from the lists it is still connected
 544 * to. We remove any pages still attached to the inode and wait for any IO that
 545 * is still in progress before finally destroying the inode.
 546 *
 547 * An inode must already be marked I_FREEING so that we avoid the inode being
 548 * moved back onto lists if we race with other code that manipulates the lists
 549 * (e.g. writeback_single_inode). The caller is responsible for setting this.
 550 *
 551 * An inode must already be removed from the LRU list before being evicted from
 552 * the cache. This should occur atomically with setting the I_FREEING state
 553 * flag, so no inodes here should ever be on the LRU when being evicted.
 554 */
 555static void evict(struct inode *inode)
 556{
 557	const struct super_operations *op = inode->i_sb->s_op;
 558
 559	BUG_ON(!(inode->i_state & I_FREEING));
 560	BUG_ON(!list_empty(&inode->i_lru));
 561
 562	if (!list_empty(&inode->i_io_list))
 563		inode_io_list_del(inode);
 564
 565	inode_sb_list_del(inode);
 566
 567	/*
 568	 * Wait for flusher thread to be done with the inode so that filesystem
 569	 * does not start destroying it while writeback is still running. Since
 570	 * the inode has I_FREEING set, flusher thread won't start new work on
 571	 * the inode.  We just have to wait for running writeback to finish.
 572	 */
 573	inode_wait_for_writeback(inode);
 574
 575	if (op->evict_inode) {
 576		op->evict_inode(inode);
 577	} else {
 578		truncate_inode_pages_final(&inode->i_data);
 
 579		clear_inode(inode);
 580	}
 581	if (S_ISBLK(inode->i_mode) && inode->i_bdev)
 582		bd_forget(inode);
 583	if (S_ISCHR(inode->i_mode) && inode->i_cdev)
 584		cd_forget(inode);
 585
 586	remove_inode_hash(inode);
 587
 588	spin_lock(&inode->i_lock);
 589	wake_up_bit(&inode->i_state, __I_NEW);
 590	BUG_ON(inode->i_state != (I_FREEING | I_CLEAR));
 591	spin_unlock(&inode->i_lock);
 592
 593	destroy_inode(inode);
 594}
 595
 596/*
 597 * dispose_list - dispose of the contents of a local list
 598 * @head: the head of the list to free
 599 *
 600 * Dispose-list gets a local list with local inodes in it, so it doesn't
 601 * need to worry about list corruption and SMP locks.
 602 */
 603static void dispose_list(struct list_head *head)
 604{
 605	while (!list_empty(head)) {
 606		struct inode *inode;
 607
 608		inode = list_first_entry(head, struct inode, i_lru);
 609		list_del_init(&inode->i_lru);
 610
 611		evict(inode);
 612		cond_resched();
 613	}
 614}
 615
 616/**
 617 * evict_inodes	- evict all evictable inodes for a superblock
 618 * @sb:		superblock to operate on
 619 *
 620 * Make sure that no inodes with zero refcount are retained.  This is
 621 * called by superblock shutdown after having SB_ACTIVE flag removed,
 622 * so any inode reaching zero refcount during or after that call will
 623 * be immediately evicted.
 624 */
 625void evict_inodes(struct super_block *sb)
 626{
 627	struct inode *inode, *next;
 628	LIST_HEAD(dispose);
 629
 630again:
 631	spin_lock(&sb->s_inode_list_lock);
 632	list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
 633		if (atomic_read(&inode->i_count))
 634			continue;
 635
 636		spin_lock(&inode->i_lock);
 637		if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
 638			spin_unlock(&inode->i_lock);
 639			continue;
 640		}
 641
 642		inode->i_state |= I_FREEING;
 643		inode_lru_list_del(inode);
 644		spin_unlock(&inode->i_lock);
 645		list_add(&inode->i_lru, &dispose);
 646
 647		/*
 648		 * We can have a ton of inodes to evict at unmount time given
 649		 * enough memory, check to see if we need to go to sleep for a
 650		 * bit so we don't livelock.
 651		 */
 652		if (need_resched()) {
 653			spin_unlock(&sb->s_inode_list_lock);
 654			cond_resched();
 655			dispose_list(&dispose);
 656			goto again;
 657		}
 658	}
 659	spin_unlock(&sb->s_inode_list_lock);
 660
 661	dispose_list(&dispose);
 662}
 663EXPORT_SYMBOL_GPL(evict_inodes);
 664
 665/**
 666 * invalidate_inodes	- attempt to free all inodes on a superblock
 667 * @sb:		superblock to operate on
 668 * @kill_dirty: flag to guide handling of dirty inodes
 669 *
 670 * Attempts to free all inodes for a given superblock.  If there were any
 671 * busy inodes return a non-zero value, else zero.
 672 * If @kill_dirty is set, discard dirty inodes too, otherwise treat
 673 * them as busy.
 674 */
 675int invalidate_inodes(struct super_block *sb, bool kill_dirty)
 676{
 677	int busy = 0;
 678	struct inode *inode, *next;
 679	LIST_HEAD(dispose);
 680
 681again:
 682	spin_lock(&sb->s_inode_list_lock);
 683	list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
 684		spin_lock(&inode->i_lock);
 685		if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
 686			spin_unlock(&inode->i_lock);
 687			continue;
 688		}
 689		if (inode->i_state & I_DIRTY_ALL && !kill_dirty) {
 690			spin_unlock(&inode->i_lock);
 691			busy = 1;
 692			continue;
 693		}
 694		if (atomic_read(&inode->i_count)) {
 695			spin_unlock(&inode->i_lock);
 696			busy = 1;
 697			continue;
 698		}
 699
 700		inode->i_state |= I_FREEING;
 701		inode_lru_list_del(inode);
 702		spin_unlock(&inode->i_lock);
 703		list_add(&inode->i_lru, &dispose);
 704		if (need_resched()) {
 705			spin_unlock(&sb->s_inode_list_lock);
 706			cond_resched();
 707			dispose_list(&dispose);
 708			goto again;
 709		}
 710	}
 711	spin_unlock(&sb->s_inode_list_lock);
 712
 713	dispose_list(&dispose);
 714
 715	return busy;
 716}
 717
 
 
 
 
 
 
 
 
 
 
 
 
 
 718/*
 719 * Isolate the inode from the LRU in preparation for freeing it.
 
 
 
 720 *
 721 * Any inodes which are pinned purely because of attached pagecache have their
 722 * pagecache removed.  If the inode has metadata buffers attached to
 723 * mapping->private_list then try to remove them.
 724 *
 725 * If the inode has the I_REFERENCED flag set, then it means that it has been
 726 * used recently - the flag is set in iput_final(). When we encounter such an
 727 * inode, clear the flag and move it to the back of the LRU so it gets another
 728 * pass through the LRU before it gets reclaimed. This is necessary because of
 729 * the fact we are doing lazy LRU updates to minimise lock contention so the
 730 * LRU does not have strict ordering. Hence we don't want to reclaim inodes
 731 * with this flag set because they are the inodes that are out of order.
 732 */
 733static enum lru_status inode_lru_isolate(struct list_head *item,
 734		struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
 735{
 736	struct list_head *freeable = arg;
 737	struct inode	*inode = container_of(item, struct inode, i_lru);
 
 738
 739	/*
 740	 * we are inverting the lru lock/inode->i_lock here, so use a trylock.
 741	 * If we fail to get the lock, just skip it.
 742	 */
 743	if (!spin_trylock(&inode->i_lock))
 744		return LRU_SKIP;
 745
 746	/*
 747	 * Referenced or dirty inodes are still in use. Give them another pass
 748	 * through the LRU as we canot reclaim them now.
 749	 */
 750	if (atomic_read(&inode->i_count) ||
 751	    (inode->i_state & ~I_REFERENCED)) {
 752		list_lru_isolate(lru, &inode->i_lru);
 753		spin_unlock(&inode->i_lock);
 754		this_cpu_dec(nr_unused);
 755		return LRU_REMOVED;
 756	}
 757
 758	/* recently referenced inodes get one more pass */
 759	if (inode->i_state & I_REFERENCED) {
 760		inode->i_state &= ~I_REFERENCED;
 761		spin_unlock(&inode->i_lock);
 762		return LRU_ROTATE;
 763	}
 764
 765	if (inode_has_buffers(inode) || inode->i_data.nrpages) {
 766		__iget(inode);
 767		spin_unlock(&inode->i_lock);
 768		spin_unlock(lru_lock);
 769		if (remove_inode_buffers(inode)) {
 770			unsigned long reap;
 771			reap = invalidate_mapping_pages(&inode->i_data, 0, -1);
 772			if (current_is_kswapd())
 773				__count_vm_events(KSWAPD_INODESTEAL, reap);
 774			else
 775				__count_vm_events(PGINODESTEAL, reap);
 776			if (current->reclaim_state)
 777				current->reclaim_state->reclaimed_slab += reap;
 778		}
 779		iput(inode);
 780		spin_lock(lru_lock);
 781		return LRU_RETRY;
 782	}
 783
 784	WARN_ON(inode->i_state & I_NEW);
 785	inode->i_state |= I_FREEING;
 786	list_lru_isolate_move(lru, &inode->i_lru, freeable);
 787	spin_unlock(&inode->i_lock);
 
 
 
 
 
 
 
 
 788
 789	this_cpu_dec(nr_unused);
 790	return LRU_REMOVED;
 791}
 
 
 
 
 
 
 
 
 
 
 
 
 
 792
 793/*
 794 * Walk the superblock inode LRU for freeable inodes and attempt to free them.
 795 * This is called from the superblock shrinker function with a number of inodes
 796 * to trim from the LRU. Inodes to be freed are moved to a temporary list and
 797 * then are freed outside inode_lock by dispose_list().
 798 */
 799long prune_icache_sb(struct super_block *sb, struct shrink_control *sc)
 800{
 801	LIST_HEAD(freeable);
 802	long freed;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 803
 804	freed = list_lru_shrink_walk(&sb->s_inode_lru, sc,
 805				     inode_lru_isolate, &freeable);
 806	dispose_list(&freeable);
 807	return freed;
 808}
 809
 810static void __wait_on_freeing_inode(struct inode *inode);
 811/*
 812 * Called with the inode lock held.
 813 */
 814static struct inode *find_inode(struct super_block *sb,
 815				struct hlist_head *head,
 816				int (*test)(struct inode *, void *),
 817				void *data)
 818{
 
 819	struct inode *inode = NULL;
 820
 821repeat:
 822	hlist_for_each_entry(inode, head, i_hash) {
 823		if (inode->i_sb != sb)
 
 
 824			continue;
 825		if (!test(inode, data))
 
 
 826			continue;
 827		spin_lock(&inode->i_lock);
 828		if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
 829			__wait_on_freeing_inode(inode);
 830			goto repeat;
 831		}
 832		if (unlikely(inode->i_state & I_CREATING)) {
 833			spin_unlock(&inode->i_lock);
 834			return ERR_PTR(-ESTALE);
 835		}
 836		__iget(inode);
 837		spin_unlock(&inode->i_lock);
 838		return inode;
 839	}
 840	return NULL;
 841}
 842
 843/*
 844 * find_inode_fast is the fast path version of find_inode, see the comment at
 845 * iget_locked for details.
 846 */
 847static struct inode *find_inode_fast(struct super_block *sb,
 848				struct hlist_head *head, unsigned long ino)
 849{
 
 850	struct inode *inode = NULL;
 851
 852repeat:
 853	hlist_for_each_entry(inode, head, i_hash) {
 854		if (inode->i_ino != ino)
 
 
 855			continue;
 856		if (inode->i_sb != sb)
 
 
 857			continue;
 858		spin_lock(&inode->i_lock);
 859		if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
 860			__wait_on_freeing_inode(inode);
 861			goto repeat;
 862		}
 863		if (unlikely(inode->i_state & I_CREATING)) {
 864			spin_unlock(&inode->i_lock);
 865			return ERR_PTR(-ESTALE);
 866		}
 867		__iget(inode);
 868		spin_unlock(&inode->i_lock);
 869		return inode;
 870	}
 871	return NULL;
 872}
 873
 874/*
 875 * Each cpu owns a range of LAST_INO_BATCH numbers.
 876 * 'shared_last_ino' is dirtied only once out of LAST_INO_BATCH allocations,
 877 * to renew the exhausted range.
 878 *
 879 * This does not significantly increase overflow rate because every CPU can
 880 * consume at most LAST_INO_BATCH-1 unused inode numbers. So there is
 881 * NR_CPUS*(LAST_INO_BATCH-1) wastage. At 4096 and 1024, this is ~0.1% of the
 882 * 2^32 range, and is a worst-case. Even a 50% wastage would only increase
 883 * overflow rate by 2x, which does not seem too significant.
 884 *
 885 * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW
 886 * error if st_ino won't fit in target struct field. Use 32bit counter
 887 * here to attempt to avoid that.
 888 */
 889#define LAST_INO_BATCH 1024
 890static DEFINE_PER_CPU(unsigned int, last_ino);
 891
 892unsigned int get_next_ino(void)
 893{
 894	unsigned int *p = &get_cpu_var(last_ino);
 895	unsigned int res = *p;
 896
 897#ifdef CONFIG_SMP
 898	if (unlikely((res & (LAST_INO_BATCH-1)) == 0)) {
 899		static atomic_t shared_last_ino;
 900		int next = atomic_add_return(LAST_INO_BATCH, &shared_last_ino);
 901
 902		res = next - LAST_INO_BATCH;
 903	}
 904#endif
 905
 906	res++;
 907	/* get_next_ino should not provide a 0 inode number */
 908	if (unlikely(!res))
 909		res++;
 910	*p = res;
 911	put_cpu_var(last_ino);
 912	return res;
 913}
 914EXPORT_SYMBOL(get_next_ino);
 915
 916/**
 917 *	new_inode_pseudo 	- obtain an inode
 918 *	@sb: superblock
 919 *
 920 *	Allocates a new inode for given superblock.
 921 *	Inode wont be chained in superblock s_inodes list
 922 *	This means :
 923 *	- fs can't be unmount
 924 *	- quotas, fsnotify, writeback can't work
 925 */
 926struct inode *new_inode_pseudo(struct super_block *sb)
 927{
 928	struct inode *inode = alloc_inode(sb);
 929
 930	if (inode) {
 931		spin_lock(&inode->i_lock);
 932		inode->i_state = 0;
 933		spin_unlock(&inode->i_lock);
 934		INIT_LIST_HEAD(&inode->i_sb_list);
 935	}
 936	return inode;
 937}
 938
 939/**
 940 *	new_inode 	- obtain an inode
 941 *	@sb: superblock
 942 *
 943 *	Allocates a new inode for given superblock. The default gfp_mask
 944 *	for allocations related to inode->i_mapping is GFP_HIGHUSER_MOVABLE.
 945 *	If HIGHMEM pages are unsuitable or it is known that pages allocated
 946 *	for the page cache are not reclaimable or migratable,
 947 *	mapping_set_gfp_mask() must be called with suitable flags on the
 948 *	newly created inode's mapping
 949 *
 950 */
 951struct inode *new_inode(struct super_block *sb)
 952{
 953	struct inode *inode;
 954
 955	spin_lock_prefetch(&sb->s_inode_list_lock);
 956
 957	inode = new_inode_pseudo(sb);
 958	if (inode)
 959		inode_sb_list_add(inode);
 960	return inode;
 961}
 962EXPORT_SYMBOL(new_inode);
 963
 964#ifdef CONFIG_DEBUG_LOCK_ALLOC
 965void lockdep_annotate_inode_mutex_key(struct inode *inode)
 966{
 967	if (S_ISDIR(inode->i_mode)) {
 968		struct file_system_type *type = inode->i_sb->s_type;
 969
 970		/* Set new key only if filesystem hasn't already changed it */
 971		if (lockdep_match_class(&inode->i_rwsem, &type->i_mutex_key)) {
 972			/*
 973			 * ensure nobody is actually holding i_mutex
 974			 */
 975			// mutex_destroy(&inode->i_mutex);
 976			init_rwsem(&inode->i_rwsem);
 977			lockdep_set_class(&inode->i_rwsem,
 978					  &type->i_mutex_dir_key);
 979		}
 980	}
 981}
 982EXPORT_SYMBOL(lockdep_annotate_inode_mutex_key);
 983#endif
 984
 985/**
 986 * unlock_new_inode - clear the I_NEW state and wake up any waiters
 987 * @inode:	new inode to unlock
 988 *
 989 * Called when the inode is fully initialised to clear the new state of the
 990 * inode and wake up anyone waiting for the inode to finish initialisation.
 991 */
 992void unlock_new_inode(struct inode *inode)
 993{
 994	lockdep_annotate_inode_mutex_key(inode);
 995	spin_lock(&inode->i_lock);
 996	WARN_ON(!(inode->i_state & I_NEW));
 997	inode->i_state &= ~I_NEW & ~I_CREATING;
 998	smp_mb();
 999	wake_up_bit(&inode->i_state, __I_NEW);
1000	spin_unlock(&inode->i_lock);
1001}
1002EXPORT_SYMBOL(unlock_new_inode);
1003
1004void discard_new_inode(struct inode *inode)
1005{
1006	lockdep_annotate_inode_mutex_key(inode);
1007	spin_lock(&inode->i_lock);
1008	WARN_ON(!(inode->i_state & I_NEW));
1009	inode->i_state &= ~I_NEW;
1010	smp_mb();
1011	wake_up_bit(&inode->i_state, __I_NEW);
1012	spin_unlock(&inode->i_lock);
1013	iput(inode);
1014}
1015EXPORT_SYMBOL(discard_new_inode);
1016
1017/**
1018 * lock_two_nondirectories - take two i_mutexes on non-directory objects
1019 *
1020 * Lock any non-NULL argument that is not a directory.
1021 * Zero, one or two objects may be locked by this function.
1022 *
1023 * @inode1: first inode to lock
1024 * @inode2: second inode to lock
1025 */
1026void lock_two_nondirectories(struct inode *inode1, struct inode *inode2)
1027{
1028	if (inode1 > inode2)
1029		swap(inode1, inode2);
1030
1031	if (inode1 && !S_ISDIR(inode1->i_mode))
1032		inode_lock(inode1);
1033	if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
1034		inode_lock_nested(inode2, I_MUTEX_NONDIR2);
1035}
1036EXPORT_SYMBOL(lock_two_nondirectories);
1037
1038/**
1039 * unlock_two_nondirectories - release locks from lock_two_nondirectories()
1040 * @inode1: first inode to unlock
1041 * @inode2: second inode to unlock
1042 */
1043void unlock_two_nondirectories(struct inode *inode1, struct inode *inode2)
1044{
1045	if (inode1 && !S_ISDIR(inode1->i_mode))
1046		inode_unlock(inode1);
1047	if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
1048		inode_unlock(inode2);
1049}
1050EXPORT_SYMBOL(unlock_two_nondirectories);
1051
1052/**
1053 * inode_insert5 - obtain an inode from a mounted file system
1054 * @inode:	pre-allocated inode to use for insert to cache
1055 * @hashval:	hash value (usually inode number) to get
1056 * @test:	callback used for comparisons between inodes
1057 * @set:	callback used to initialize a new struct inode
1058 * @data:	opaque data pointer to pass to @test and @set
1059 *
1060 * Search for the inode specified by @hashval and @data in the inode cache,
1061 * and if present it is return it with an increased reference count. This is
1062 * a variant of iget5_locked() for callers that don't want to fail on memory
1063 * allocation of inode.
1064 *
1065 * If the inode is not in cache, insert the pre-allocated inode to cache and
1066 * return it locked, hashed, and with the I_NEW flag set. The file system gets
1067 * to fill it in before unlocking it via unlock_new_inode().
1068 *
1069 * Note both @test and @set are called with the inode_hash_lock held, so can't
1070 * sleep.
1071 */
1072struct inode *inode_insert5(struct inode *inode, unsigned long hashval,
1073			    int (*test)(struct inode *, void *),
1074			    int (*set)(struct inode *, void *), void *data)
1075{
1076	struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval);
1077	struct inode *old;
1078	bool creating = inode->i_state & I_CREATING;
1079
1080again:
1081	spin_lock(&inode_hash_lock);
1082	old = find_inode(inode->i_sb, head, test, data);
1083	if (unlikely(old)) {
1084		/*
1085		 * Uhhuh, somebody else created the same inode under us.
1086		 * Use the old inode instead of the preallocated one.
1087		 */
1088		spin_unlock(&inode_hash_lock);
1089		if (IS_ERR(old))
1090			return NULL;
1091		wait_on_inode(old);
1092		if (unlikely(inode_unhashed(old))) {
1093			iput(old);
1094			goto again;
1095		}
1096		return old;
1097	}
1098
1099	if (set && unlikely(set(inode, data))) {
1100		inode = NULL;
1101		goto unlock;
1102	}
1103
1104	/*
1105	 * Return the locked inode with I_NEW set, the
1106	 * caller is responsible for filling in the contents
1107	 */
1108	spin_lock(&inode->i_lock);
1109	inode->i_state |= I_NEW;
1110	hlist_add_head_rcu(&inode->i_hash, head);
1111	spin_unlock(&inode->i_lock);
1112	if (!creating)
1113		inode_sb_list_add(inode);
1114unlock:
1115	spin_unlock(&inode_hash_lock);
1116
1117	return inode;
1118}
1119EXPORT_SYMBOL(inode_insert5);
1120
1121/**
1122 * iget5_locked - obtain an inode from a mounted file system
1123 * @sb:		super block of file system
1124 * @hashval:	hash value (usually inode number) to get
1125 * @test:	callback used for comparisons between inodes
1126 * @set:	callback used to initialize a new struct inode
1127 * @data:	opaque data pointer to pass to @test and @set
1128 *
1129 * Search for the inode specified by @hashval and @data in the inode cache,
1130 * and if present it is return it with an increased reference count. This is
1131 * a generalized version of iget_locked() for file systems where the inode
1132 * number is not sufficient for unique identification of an inode.
1133 *
1134 * If the inode is not in cache, allocate a new inode and return it locked,
1135 * hashed, and with the I_NEW flag set. The file system gets to fill it in
1136 * before unlocking it via unlock_new_inode().
1137 *
1138 * Note both @test and @set are called with the inode_hash_lock held, so can't
1139 * sleep.
1140 */
1141struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
1142		int (*test)(struct inode *, void *),
1143		int (*set)(struct inode *, void *), void *data)
1144{
1145	struct inode *inode = ilookup5(sb, hashval, test, data);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1146
1147	if (!inode) {
1148		struct inode *new = alloc_inode(sb);
 
 
 
 
1149
1150		if (new) {
1151			new->i_state = 0;
1152			inode = inode_insert5(new, hashval, test, set, data);
1153			if (unlikely(inode != new))
1154				destroy_inode(new);
1155		}
 
 
 
 
 
 
 
 
 
 
1156	}
1157	return inode;
 
 
 
 
 
1158}
1159EXPORT_SYMBOL(iget5_locked);
1160
1161/**
1162 * iget_locked - obtain an inode from a mounted file system
1163 * @sb:		super block of file system
1164 * @ino:	inode number to get
1165 *
1166 * Search for the inode specified by @ino in the inode cache and if present
1167 * return it with an increased reference count. This is for file systems
1168 * where the inode number is sufficient for unique identification of an inode.
1169 *
1170 * If the inode is not in cache, allocate a new inode and return it locked,
1171 * hashed, and with the I_NEW flag set.  The file system gets to fill it in
1172 * before unlocking it via unlock_new_inode().
1173 */
1174struct inode *iget_locked(struct super_block *sb, unsigned long ino)
1175{
1176	struct hlist_head *head = inode_hashtable + hash(sb, ino);
1177	struct inode *inode;
1178again:
1179	spin_lock(&inode_hash_lock);
1180	inode = find_inode_fast(sb, head, ino);
1181	spin_unlock(&inode_hash_lock);
1182	if (inode) {
1183		if (IS_ERR(inode))
1184			return NULL;
1185		wait_on_inode(inode);
1186		if (unlikely(inode_unhashed(inode))) {
1187			iput(inode);
1188			goto again;
1189		}
1190		return inode;
1191	}
1192
1193	inode = alloc_inode(sb);
1194	if (inode) {
1195		struct inode *old;
1196
1197		spin_lock(&inode_hash_lock);
1198		/* We released the lock, so.. */
1199		old = find_inode_fast(sb, head, ino);
1200		if (!old) {
1201			inode->i_ino = ino;
1202			spin_lock(&inode->i_lock);
1203			inode->i_state = I_NEW;
1204			hlist_add_head_rcu(&inode->i_hash, head);
1205			spin_unlock(&inode->i_lock);
1206			inode_sb_list_add(inode);
1207			spin_unlock(&inode_hash_lock);
1208
1209			/* Return the locked inode with I_NEW set, the
1210			 * caller is responsible for filling in the contents
1211			 */
1212			return inode;
1213		}
1214
1215		/*
1216		 * Uhhuh, somebody else created the same inode under
1217		 * us. Use the old inode instead of the one we just
1218		 * allocated.
1219		 */
1220		spin_unlock(&inode_hash_lock);
1221		destroy_inode(inode);
1222		if (IS_ERR(old))
1223			return NULL;
1224		inode = old;
1225		wait_on_inode(inode);
1226		if (unlikely(inode_unhashed(inode))) {
1227			iput(inode);
1228			goto again;
1229		}
1230	}
1231	return inode;
1232}
1233EXPORT_SYMBOL(iget_locked);
1234
1235/*
1236 * search the inode cache for a matching inode number.
1237 * If we find one, then the inode number we are trying to
1238 * allocate is not unique and so we should not use it.
1239 *
1240 * Returns 1 if the inode number is unique, 0 if it is not.
1241 */
1242static int test_inode_iunique(struct super_block *sb, unsigned long ino)
1243{
1244	struct hlist_head *b = inode_hashtable + hash(sb, ino);
 
1245	struct inode *inode;
1246
1247	hlist_for_each_entry_rcu(inode, b, i_hash) {
1248		if (inode->i_ino == ino && inode->i_sb == sb)
 
 
1249			return 0;
 
1250	}
 
 
1251	return 1;
1252}
1253
1254/**
1255 *	iunique - get a unique inode number
1256 *	@sb: superblock
1257 *	@max_reserved: highest reserved inode number
1258 *
1259 *	Obtain an inode number that is unique on the system for a given
1260 *	superblock. This is used by file systems that have no natural
1261 *	permanent inode numbering system. An inode number is returned that
1262 *	is higher than the reserved limit but unique.
1263 *
1264 *	BUGS:
1265 *	With a large number of inodes live on the file system this function
1266 *	currently becomes quite slow.
1267 */
1268ino_t iunique(struct super_block *sb, ino_t max_reserved)
1269{
1270	/*
1271	 * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW
1272	 * error if st_ino won't fit in target struct field. Use 32bit counter
1273	 * here to attempt to avoid that.
1274	 */
1275	static DEFINE_SPINLOCK(iunique_lock);
1276	static unsigned int counter;
1277	ino_t res;
1278
1279	rcu_read_lock();
1280	spin_lock(&iunique_lock);
1281	do {
1282		if (counter <= max_reserved)
1283			counter = max_reserved + 1;
1284		res = counter++;
1285	} while (!test_inode_iunique(sb, res));
1286	spin_unlock(&iunique_lock);
1287	rcu_read_unlock();
1288
1289	return res;
1290}
1291EXPORT_SYMBOL(iunique);
1292
1293struct inode *igrab(struct inode *inode)
1294{
1295	spin_lock(&inode->i_lock);
1296	if (!(inode->i_state & (I_FREEING|I_WILL_FREE))) {
1297		__iget(inode);
1298		spin_unlock(&inode->i_lock);
1299	} else {
1300		spin_unlock(&inode->i_lock);
1301		/*
1302		 * Handle the case where s_op->clear_inode is not been
1303		 * called yet, and somebody is calling igrab
1304		 * while the inode is getting freed.
1305		 */
1306		inode = NULL;
1307	}
1308	return inode;
1309}
1310EXPORT_SYMBOL(igrab);
1311
1312/**
1313 * ilookup5_nowait - search for an inode in the inode cache
1314 * @sb:		super block of file system to search
1315 * @hashval:	hash value (usually inode number) to search for
1316 * @test:	callback used for comparisons between inodes
1317 * @data:	opaque data pointer to pass to @test
1318 *
1319 * Search for the inode specified by @hashval and @data in the inode cache.
1320 * If the inode is in the cache, the inode is returned with an incremented
1321 * reference count.
1322 *
1323 * Note: I_NEW is not waited upon so you have to be very careful what you do
1324 * with the returned inode.  You probably should be using ilookup5() instead.
1325 *
1326 * Note2: @test is called with the inode_hash_lock held, so can't sleep.
1327 */
1328struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,
1329		int (*test)(struct inode *, void *), void *data)
1330{
1331	struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1332	struct inode *inode;
1333
1334	spin_lock(&inode_hash_lock);
1335	inode = find_inode(sb, head, test, data);
1336	spin_unlock(&inode_hash_lock);
1337
1338	return IS_ERR(inode) ? NULL : inode;
1339}
1340EXPORT_SYMBOL(ilookup5_nowait);
1341
1342/**
1343 * ilookup5 - search for an inode in the inode cache
1344 * @sb:		super block of file system to search
1345 * @hashval:	hash value (usually inode number) to search for
1346 * @test:	callback used for comparisons between inodes
1347 * @data:	opaque data pointer to pass to @test
1348 *
1349 * Search for the inode specified by @hashval and @data in the inode cache,
1350 * and if the inode is in the cache, return the inode with an incremented
1351 * reference count.  Waits on I_NEW before returning the inode.
1352 * returned with an incremented reference count.
1353 *
1354 * This is a generalized version of ilookup() for file systems where the
1355 * inode number is not sufficient for unique identification of an inode.
1356 *
1357 * Note: @test is called with the inode_hash_lock held, so can't sleep.
1358 */
1359struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
1360		int (*test)(struct inode *, void *), void *data)
1361{
1362	struct inode *inode;
1363again:
1364	inode = ilookup5_nowait(sb, hashval, test, data);
1365	if (inode) {
1366		wait_on_inode(inode);
1367		if (unlikely(inode_unhashed(inode))) {
1368			iput(inode);
1369			goto again;
1370		}
1371	}
1372	return inode;
1373}
1374EXPORT_SYMBOL(ilookup5);
1375
1376/**
1377 * ilookup - search for an inode in the inode cache
1378 * @sb:		super block of file system to search
1379 * @ino:	inode number to search for
1380 *
1381 * Search for the inode @ino in the inode cache, and if the inode is in the
1382 * cache, the inode is returned with an incremented reference count.
1383 */
1384struct inode *ilookup(struct super_block *sb, unsigned long ino)
1385{
1386	struct hlist_head *head = inode_hashtable + hash(sb, ino);
1387	struct inode *inode;
1388again:
1389	spin_lock(&inode_hash_lock);
1390	inode = find_inode_fast(sb, head, ino);
1391	spin_unlock(&inode_hash_lock);
1392
1393	if (inode) {
1394		if (IS_ERR(inode))
1395			return NULL;
1396		wait_on_inode(inode);
1397		if (unlikely(inode_unhashed(inode))) {
1398			iput(inode);
1399			goto again;
1400		}
1401	}
1402	return inode;
1403}
1404EXPORT_SYMBOL(ilookup);
1405
1406/**
1407 * find_inode_nowait - find an inode in the inode cache
1408 * @sb:		super block of file system to search
1409 * @hashval:	hash value (usually inode number) to search for
1410 * @match:	callback used for comparisons between inodes
1411 * @data:	opaque data pointer to pass to @match
1412 *
1413 * Search for the inode specified by @hashval and @data in the inode
1414 * cache, where the helper function @match will return 0 if the inode
1415 * does not match, 1 if the inode does match, and -1 if the search
1416 * should be stopped.  The @match function must be responsible for
1417 * taking the i_lock spin_lock and checking i_state for an inode being
1418 * freed or being initialized, and incrementing the reference count
1419 * before returning 1.  It also must not sleep, since it is called with
1420 * the inode_hash_lock spinlock held.
1421 *
1422 * This is a even more generalized version of ilookup5() when the
1423 * function must never block --- find_inode() can block in
1424 * __wait_on_freeing_inode() --- or when the caller can not increment
1425 * the reference count because the resulting iput() might cause an
1426 * inode eviction.  The tradeoff is that the @match funtion must be
1427 * very carefully implemented.
1428 */
1429struct inode *find_inode_nowait(struct super_block *sb,
1430				unsigned long hashval,
1431				int (*match)(struct inode *, unsigned long,
1432					     void *),
1433				void *data)
1434{
1435	struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1436	struct inode *inode, *ret_inode = NULL;
1437	int mval;
1438
1439	spin_lock(&inode_hash_lock);
1440	hlist_for_each_entry(inode, head, i_hash) {
1441		if (inode->i_sb != sb)
1442			continue;
1443		mval = match(inode, hashval, data);
1444		if (mval == 0)
1445			continue;
1446		if (mval == 1)
1447			ret_inode = inode;
1448		goto out;
1449	}
1450out:
1451	spin_unlock(&inode_hash_lock);
1452	return ret_inode;
1453}
1454EXPORT_SYMBOL(find_inode_nowait);
1455
1456/**
1457 * find_inode_rcu - find an inode in the inode cache
1458 * @sb:		Super block of file system to search
1459 * @hashval:	Key to hash
1460 * @test:	Function to test match on an inode
1461 * @data:	Data for test function
1462 *
1463 * Search for the inode specified by @hashval and @data in the inode cache,
1464 * where the helper function @test will return 0 if the inode does not match
1465 * and 1 if it does.  The @test function must be responsible for taking the
1466 * i_lock spin_lock and checking i_state for an inode being freed or being
1467 * initialized.
1468 *
1469 * If successful, this will return the inode for which the @test function
1470 * returned 1 and NULL otherwise.
1471 *
1472 * The @test function is not permitted to take a ref on any inode presented.
1473 * It is also not permitted to sleep.
1474 *
1475 * The caller must hold the RCU read lock.
1476 */
1477struct inode *find_inode_rcu(struct super_block *sb, unsigned long hashval,
1478			     int (*test)(struct inode *, void *), void *data)
1479{
1480	struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1481	struct inode *inode;
1482
1483	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
1484			 "suspicious find_inode_rcu() usage");
1485
1486	hlist_for_each_entry_rcu(inode, head, i_hash) {
1487		if (inode->i_sb == sb &&
1488		    !(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE)) &&
1489		    test(inode, data))
1490			return inode;
1491	}
1492	return NULL;
1493}
1494EXPORT_SYMBOL(find_inode_rcu);
1495
1496/**
1497 * find_inode_by_rcu - Find an inode in the inode cache
1498 * @sb:		Super block of file system to search
1499 * @ino:	The inode number to match
1500 *
1501 * Search for the inode specified by @hashval and @data in the inode cache,
1502 * where the helper function @test will return 0 if the inode does not match
1503 * and 1 if it does.  The @test function must be responsible for taking the
1504 * i_lock spin_lock and checking i_state for an inode being freed or being
1505 * initialized.
1506 *
1507 * If successful, this will return the inode for which the @test function
1508 * returned 1 and NULL otherwise.
1509 *
1510 * The @test function is not permitted to take a ref on any inode presented.
1511 * It is also not permitted to sleep.
1512 *
1513 * The caller must hold the RCU read lock.
1514 */
1515struct inode *find_inode_by_ino_rcu(struct super_block *sb,
1516				    unsigned long ino)
1517{
1518	struct hlist_head *head = inode_hashtable + hash(sb, ino);
1519	struct inode *inode;
1520
1521	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
1522			 "suspicious find_inode_by_ino_rcu() usage");
1523
1524	hlist_for_each_entry_rcu(inode, head, i_hash) {
1525		if (inode->i_ino == ino &&
1526		    inode->i_sb == sb &&
1527		    !(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE)))
1528		    return inode;
1529	}
1530	return NULL;
1531}
1532EXPORT_SYMBOL(find_inode_by_ino_rcu);
1533
1534int insert_inode_locked(struct inode *inode)
1535{
1536	struct super_block *sb = inode->i_sb;
1537	ino_t ino = inode->i_ino;
1538	struct hlist_head *head = inode_hashtable + hash(sb, ino);
1539
1540	while (1) {
 
1541		struct inode *old = NULL;
1542		spin_lock(&inode_hash_lock);
1543		hlist_for_each_entry(old, head, i_hash) {
1544			if (old->i_ino != ino)
1545				continue;
1546			if (old->i_sb != sb)
1547				continue;
1548			spin_lock(&old->i_lock);
1549			if (old->i_state & (I_FREEING|I_WILL_FREE)) {
1550				spin_unlock(&old->i_lock);
1551				continue;
1552			}
1553			break;
1554		}
1555		if (likely(!old)) {
1556			spin_lock(&inode->i_lock);
1557			inode->i_state |= I_NEW | I_CREATING;
1558			hlist_add_head_rcu(&inode->i_hash, head);
1559			spin_unlock(&inode->i_lock);
1560			spin_unlock(&inode_hash_lock);
1561			return 0;
1562		}
1563		if (unlikely(old->i_state & I_CREATING)) {
1564			spin_unlock(&old->i_lock);
1565			spin_unlock(&inode_hash_lock);
1566			return -EBUSY;
1567		}
1568		__iget(old);
1569		spin_unlock(&old->i_lock);
1570		spin_unlock(&inode_hash_lock);
1571		wait_on_inode(old);
1572		if (unlikely(!inode_unhashed(old))) {
1573			iput(old);
1574			return -EBUSY;
1575		}
1576		iput(old);
1577	}
1578}
1579EXPORT_SYMBOL(insert_inode_locked);
1580
1581int insert_inode_locked4(struct inode *inode, unsigned long hashval,
1582		int (*test)(struct inode *, void *), void *data)
1583{
1584	struct inode *old;
 
1585
1586	inode->i_state |= I_CREATING;
1587	old = inode_insert5(inode, hashval, test, NULL, data);
 
1588
1589	if (old != inode) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1590		iput(old);
1591		return -EBUSY;
1592	}
1593	return 0;
1594}
1595EXPORT_SYMBOL(insert_inode_locked4);
1596
1597
1598int generic_delete_inode(struct inode *inode)
1599{
1600	return 1;
1601}
1602EXPORT_SYMBOL(generic_delete_inode);
1603
1604/*
1605 * Called when we're dropping the last reference
1606 * to an inode.
1607 *
1608 * Call the FS "drop_inode()" function, defaulting to
1609 * the legacy UNIX filesystem behaviour.  If it tells
1610 * us to evict inode, do so.  Otherwise, retain inode
1611 * in cache if fs is alive, sync and evict if fs is
1612 * shutting down.
1613 */
1614static void iput_final(struct inode *inode)
1615{
1616	struct super_block *sb = inode->i_sb;
1617	const struct super_operations *op = inode->i_sb->s_op;
1618	unsigned long state;
1619	int drop;
1620
1621	WARN_ON(inode->i_state & I_NEW);
1622
1623	if (op->drop_inode)
1624		drop = op->drop_inode(inode);
1625	else
1626		drop = generic_drop_inode(inode);
1627
1628	if (!drop && (sb->s_flags & SB_ACTIVE)) {
1629		inode_add_lru(inode);
 
 
1630		spin_unlock(&inode->i_lock);
1631		return;
1632	}
1633
1634	state = inode->i_state;
1635	if (!drop) {
1636		WRITE_ONCE(inode->i_state, state | I_WILL_FREE);
1637		spin_unlock(&inode->i_lock);
1638
1639		write_inode_now(inode, 1);
1640
1641		spin_lock(&inode->i_lock);
1642		state = inode->i_state;
1643		WARN_ON(state & I_NEW);
1644		state &= ~I_WILL_FREE;
1645	}
1646
1647	WRITE_ONCE(inode->i_state, state | I_FREEING);
1648	if (!list_empty(&inode->i_lru))
1649		inode_lru_list_del(inode);
1650	spin_unlock(&inode->i_lock);
1651
1652	evict(inode);
1653}
1654
1655/**
1656 *	iput	- put an inode
1657 *	@inode: inode to put
1658 *
1659 *	Puts an inode, dropping its usage count. If the inode use count hits
1660 *	zero, the inode is then freed and may also be destroyed.
1661 *
1662 *	Consequently, iput() can sleep.
1663 */
1664void iput(struct inode *inode)
1665{
1666	if (!inode)
1667		return;
1668	BUG_ON(inode->i_state & I_CLEAR);
1669retry:
1670	if (atomic_dec_and_lock(&inode->i_count, &inode->i_lock)) {
1671		if (inode->i_nlink && (inode->i_state & I_DIRTY_TIME)) {
1672			atomic_inc(&inode->i_count);
1673			spin_unlock(&inode->i_lock);
1674			trace_writeback_lazytime_iput(inode);
1675			mark_inode_dirty_sync(inode);
1676			goto retry;
1677		}
1678		iput_final(inode);
1679	}
1680}
1681EXPORT_SYMBOL(iput);
1682
1683#ifdef CONFIG_BLOCK
1684/**
1685 *	bmap	- find a block number in a file
1686 *	@inode:  inode owning the block number being requested
1687 *	@block: pointer containing the block to find
1688 *
1689 *	Replaces the value in ``*block`` with the block number on the device holding
1690 *	corresponding to the requested block number in the file.
1691 *	That is, asked for block 4 of inode 1 the function will replace the
1692 *	4 in ``*block``, with disk block relative to the disk start that holds that
1693 *	block of the file.
1694 *
1695 *	Returns -EINVAL in case of error, 0 otherwise. If mapping falls into a
1696 *	hole, returns 0 and ``*block`` is also set to 0.
1697 */
1698int bmap(struct inode *inode, sector_t *block)
1699{
1700	if (!inode->i_mapping->a_ops->bmap)
1701		return -EINVAL;
1702
1703	*block = inode->i_mapping->a_ops->bmap(inode->i_mapping, *block);
1704	return 0;
 
 
1705}
1706EXPORT_SYMBOL(bmap);
1707#endif
1708
1709/*
1710 * With relative atime, only update atime if the previous atime is
1711 * earlier than either the ctime or mtime or if at least a day has
1712 * passed since the last atime update.
1713 */
1714static int relatime_need_update(struct vfsmount *mnt, struct inode *inode,
1715			     struct timespec64 now)
1716{
1717
1718	if (!(mnt->mnt_flags & MNT_RELATIME))
1719		return 1;
1720	/*
1721	 * Is mtime younger than atime? If yes, update atime:
1722	 */
1723	if (timespec64_compare(&inode->i_mtime, &inode->i_atime) >= 0)
1724		return 1;
1725	/*
1726	 * Is ctime younger than atime? If yes, update atime:
1727	 */
1728	if (timespec64_compare(&inode->i_ctime, &inode->i_atime) >= 0)
1729		return 1;
1730
1731	/*
1732	 * Is the previous atime value older than a day? If yes,
1733	 * update atime:
1734	 */
1735	if ((long)(now.tv_sec - inode->i_atime.tv_sec) >= 24*60*60)
1736		return 1;
1737	/*
1738	 * Good, we can skip the atime update:
1739	 */
1740	return 0;
1741}
1742
1743int generic_update_time(struct inode *inode, struct timespec64 *time, int flags)
 
 
 
 
1744{
1745	int iflags = I_DIRTY_TIME;
1746	bool dirty = false;
1747
1748	if (flags & S_ATIME)
1749		inode->i_atime = *time;
1750	if (flags & S_VERSION)
1751		dirty = inode_maybe_inc_iversion(inode, false);
1752	if (flags & S_CTIME)
1753		inode->i_ctime = *time;
1754	if (flags & S_MTIME)
1755		inode->i_mtime = *time;
1756	if ((flags & (S_ATIME | S_CTIME | S_MTIME)) &&
1757	    !(inode->i_sb->s_flags & SB_LAZYTIME))
1758		dirty = true;
1759
1760	if (dirty)
1761		iflags |= I_DIRTY_SYNC;
1762	__mark_inode_dirty(inode, iflags);
1763	return 0;
1764}
1765EXPORT_SYMBOL(generic_update_time);
1766
1767/*
1768 * This does the actual work of updating an inodes time or version.  Must have
1769 * had called mnt_want_write() before calling this.
1770 */
1771static int update_time(struct inode *inode, struct timespec64 *time, int flags)
1772{
1773	if (inode->i_op->update_time)
1774		return inode->i_op->update_time(inode, time, flags);
1775	return generic_update_time(inode, time, flags);
1776}
1777
1778/**
1779 *	touch_atime	-	update the access time
1780 *	@path: the &struct path to update
1781 *	@inode: inode to update
1782 *
1783 *	Update the accessed time on an inode and mark it for writeback.
1784 *	This function automatically handles read only file systems and media,
1785 *	as well as the "noatime" flag and inode specific "noatime" markers.
1786 */
1787bool atime_needs_update(const struct path *path, struct inode *inode)
1788{
1789	struct vfsmount *mnt = path->mnt;
1790	struct timespec64 now;
 
1791
1792	if (inode->i_flags & S_NOATIME)
1793		return false;
1794
1795	/* Atime updates will likely cause i_uid and i_gid to be written
1796	 * back improprely if their true value is unknown to the vfs.
1797	 */
1798	if (HAS_UNMAPPED_ID(inode))
1799		return false;
1800
1801	if (IS_NOATIME(inode))
1802		return false;
1803	if ((inode->i_sb->s_flags & SB_NODIRATIME) && S_ISDIR(inode->i_mode))
1804		return false;
1805
1806	if (mnt->mnt_flags & MNT_NOATIME)
1807		return false;
1808	if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode))
1809		return false;
1810
1811	now = current_time(inode);
1812
1813	if (!relatime_need_update(mnt, inode, now))
1814		return false;
1815
1816	if (timespec64_equal(&inode->i_atime, &now))
1817		return false;
1818
1819	return true;
1820}
1821
1822void touch_atime(const struct path *path)
1823{
1824	struct vfsmount *mnt = path->mnt;
1825	struct inode *inode = d_inode(path->dentry);
1826	struct timespec64 now;
1827
1828	if (!atime_needs_update(path, inode))
1829		return;
1830
1831	if (!sb_start_write_trylock(inode->i_sb))
1832		return;
1833
1834	if (__mnt_want_write(mnt) != 0)
1835		goto skip_update;
1836	/*
1837	 * File systems can error out when updating inodes if they need to
1838	 * allocate new space to modify an inode (such is the case for
1839	 * Btrfs), but since we touch atime while walking down the path we
1840	 * really don't care if we failed to update the atime of the file,
1841	 * so just ignore the return value.
1842	 * We may also fail on filesystems that have the ability to make parts
1843	 * of the fs read only, e.g. subvolumes in Btrfs.
1844	 */
1845	now = current_time(inode);
1846	update_time(inode, &now, S_ATIME);
1847	__mnt_drop_write(mnt);
1848skip_update:
1849	sb_end_write(inode->i_sb);
1850}
1851EXPORT_SYMBOL(touch_atime);
1852
1853/*
1854 * The logic we want is
1855 *
1856 *	if suid or (sgid and xgrp)
1857 *		remove privs
1858 */
1859int should_remove_suid(struct dentry *dentry)
1860{
1861	umode_t mode = d_inode(dentry)->i_mode;
1862	int kill = 0;
1863
1864	/* suid always must be killed */
1865	if (unlikely(mode & S_ISUID))
1866		kill = ATTR_KILL_SUID;
1867
1868	/*
1869	 * sgid without any exec bits is just a mandatory locking mark; leave
1870	 * it alone.  If some exec bits are set, it's a real sgid; kill it.
1871	 */
1872	if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1873		kill |= ATTR_KILL_SGID;
1874
1875	if (unlikely(kill && !capable(CAP_FSETID) && S_ISREG(mode)))
1876		return kill;
1877
1878	return 0;
1879}
1880EXPORT_SYMBOL(should_remove_suid);
1881
1882/*
1883 * Return mask of changes for notify_change() that need to be done as a
1884 * response to write or truncate. Return 0 if nothing has to be changed.
1885 * Negative value on error (change should be denied).
1886 */
1887int dentry_needs_remove_privs(struct dentry *dentry)
1888{
1889	struct inode *inode = d_inode(dentry);
1890	int mask = 0;
1891	int ret;
1892
1893	if (IS_NOSEC(inode))
1894		return 0;
1895
1896	mask = should_remove_suid(dentry);
1897	ret = security_inode_need_killpriv(dentry);
1898	if (ret < 0)
1899		return ret;
1900	if (ret)
1901		mask |= ATTR_KILL_PRIV;
1902	return mask;
1903}
1904
1905static int __remove_privs(struct dentry *dentry, int kill)
1906{
1907	struct iattr newattrs;
1908
1909	newattrs.ia_valid = ATTR_FORCE | kill;
1910	/*
1911	 * Note we call this on write, so notify_change will not
1912	 * encounter any conflicting delegations:
1913	 */
1914	return notify_change(dentry, &newattrs, NULL);
1915}
1916
1917/*
1918 * Remove special file priviledges (suid, capabilities) when file is written
1919 * to or truncated.
1920 */
1921int file_remove_privs(struct file *file)
1922{
1923	struct dentry *dentry = file_dentry(file);
1924	struct inode *inode = file_inode(file);
1925	int kill;
 
1926	int error = 0;
1927
1928	/*
1929	 * Fast path for nothing security related.
1930	 * As well for non-regular files, e.g. blkdev inodes.
1931	 * For example, blkdev_write_iter() might get here
1932	 * trying to remove privs which it is not allowed to.
1933	 */
1934	if (IS_NOSEC(inode) || !S_ISREG(inode->i_mode))
1935		return 0;
1936
1937	kill = dentry_needs_remove_privs(dentry);
1938	if (kill < 0)
1939		return kill;
1940	if (kill)
1941		error = __remove_privs(dentry, kill);
1942	if (!error)
1943		inode_has_no_xattr(inode);
 
 
 
 
1944
1945	return error;
1946}
1947EXPORT_SYMBOL(file_remove_privs);
1948
1949/**
1950 *	file_update_time	-	update mtime and ctime time
1951 *	@file: file accessed
1952 *
1953 *	Update the mtime and ctime members of an inode and mark the inode
1954 *	for writeback.  Note that this function is meant exclusively for
1955 *	usage in the file write path of filesystems, and filesystems may
1956 *	choose to explicitly ignore update via this function with the
1957 *	S_NOCMTIME inode flag, e.g. for network filesystem where these
1958 *	timestamps are handled by the server.  This can return an error for
1959 *	file systems who need to allocate space in order to update an inode.
1960 */
1961
1962int file_update_time(struct file *file)
1963{
1964	struct inode *inode = file_inode(file);
1965	struct timespec64 now;
1966	int sync_it = 0;
1967	int ret;
1968
1969	/* First try to exhaust all avenues to not sync */
1970	if (IS_NOCMTIME(inode))
1971		return 0;
1972
1973	now = current_time(inode);
1974	if (!timespec64_equal(&inode->i_mtime, &now))
1975		sync_it = S_MTIME;
1976
1977	if (!timespec64_equal(&inode->i_ctime, &now))
1978		sync_it |= S_CTIME;
1979
1980	if (IS_I_VERSION(inode) && inode_iversion_need_inc(inode))
1981		sync_it |= S_VERSION;
1982
1983	if (!sync_it)
1984		return 0;
1985
1986	/* Finally allowed to write? Takes lock. */
1987	if (__mnt_want_write_file(file))
1988		return 0;
1989
1990	ret = update_time(inode, &now, sync_it);
1991	__mnt_drop_write_file(file);
1992
1993	return ret;
1994}
1995EXPORT_SYMBOL(file_update_time);
1996
1997/* Caller must hold the file's inode lock */
1998int file_modified(struct file *file)
1999{
2000	int err;
2001
2002	/*
2003	 * Clear the security bits if the process is not being run by root.
2004	 * This keeps people from modifying setuid and setgid binaries.
2005	 */
2006	err = file_remove_privs(file);
2007	if (err)
2008		return err;
2009
2010	if (unlikely(file->f_mode & FMODE_NOCMTIME))
2011		return 0;
2012
2013	return file_update_time(file);
2014}
2015EXPORT_SYMBOL(file_modified);
2016
2017int inode_needs_sync(struct inode *inode)
2018{
2019	if (IS_SYNC(inode))
2020		return 1;
2021	if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode))
2022		return 1;
2023	return 0;
2024}
2025EXPORT_SYMBOL(inode_needs_sync);
2026
 
 
 
 
 
 
 
2027/*
2028 * If we try to find an inode in the inode hash while it is being
2029 * deleted, we have to wait until the filesystem completes its
2030 * deletion before reporting that it isn't found.  This function waits
2031 * until the deletion _might_ have completed.  Callers are responsible
2032 * to recheck inode state.
2033 *
2034 * It doesn't matter if I_NEW is not set initially, a call to
2035 * wake_up_bit(&inode->i_state, __I_NEW) after removing from the hash list
2036 * will DTRT.
2037 */
2038static void __wait_on_freeing_inode(struct inode *inode)
2039{
2040	wait_queue_head_t *wq;
2041	DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW);
2042	wq = bit_waitqueue(&inode->i_state, __I_NEW);
2043	prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
2044	spin_unlock(&inode->i_lock);
2045	spin_unlock(&inode_hash_lock);
2046	schedule();
2047	finish_wait(wq, &wait.wq_entry);
2048	spin_lock(&inode_hash_lock);
2049}
2050
2051static __initdata unsigned long ihash_entries;
2052static int __init set_ihash_entries(char *str)
2053{
2054	if (!str)
2055		return 0;
2056	ihash_entries = simple_strtoul(str, &str, 0);
2057	return 1;
2058}
2059__setup("ihash_entries=", set_ihash_entries);
2060
2061/*
2062 * Initialize the waitqueues and inode hash table.
2063 */
2064void __init inode_init_early(void)
2065{
 
 
2066	/* If hashes are distributed across NUMA nodes, defer
2067	 * hash allocation until vmalloc space is available.
2068	 */
2069	if (hashdist)
2070		return;
2071
2072	inode_hashtable =
2073		alloc_large_system_hash("Inode-cache",
2074					sizeof(struct hlist_head),
2075					ihash_entries,
2076					14,
2077					HASH_EARLY | HASH_ZERO,
2078					&i_hash_shift,
2079					&i_hash_mask,
2080					0,
2081					0);
 
 
 
2082}
2083
2084void __init inode_init(void)
2085{
 
 
2086	/* inode slab cache */
2087	inode_cachep = kmem_cache_create("inode_cache",
2088					 sizeof(struct inode),
2089					 0,
2090					 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
2091					 SLAB_MEM_SPREAD|SLAB_ACCOUNT),
2092					 init_once);
2093
2094	/* Hash may have been set up in inode_init_early */
2095	if (!hashdist)
2096		return;
2097
2098	inode_hashtable =
2099		alloc_large_system_hash("Inode-cache",
2100					sizeof(struct hlist_head),
2101					ihash_entries,
2102					14,
2103					HASH_ZERO,
2104					&i_hash_shift,
2105					&i_hash_mask,
2106					0,
2107					0);
 
 
 
2108}
2109
2110void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev)
2111{
2112	inode->i_mode = mode;
2113	if (S_ISCHR(mode)) {
2114		inode->i_fop = &def_chr_fops;
2115		inode->i_rdev = rdev;
2116	} else if (S_ISBLK(mode)) {
2117		inode->i_fop = &def_blk_fops;
2118		inode->i_rdev = rdev;
2119	} else if (S_ISFIFO(mode))
2120		inode->i_fop = &pipefifo_fops;
2121	else if (S_ISSOCK(mode))
2122		;	/* leave it no_open_fops */
2123	else
2124		printk(KERN_DEBUG "init_special_inode: bogus i_mode (%o) for"
2125				  " inode %s:%lu\n", mode, inode->i_sb->s_id,
2126				  inode->i_ino);
2127}
2128EXPORT_SYMBOL(init_special_inode);
2129
2130/**
2131 * inode_init_owner - Init uid,gid,mode for new inode according to posix standards
2132 * @inode: New inode
2133 * @dir: Directory inode
2134 * @mode: mode of the new inode
2135 */
2136void inode_init_owner(struct inode *inode, const struct inode *dir,
2137			umode_t mode)
2138{
2139	inode->i_uid = current_fsuid();
2140	if (dir && dir->i_mode & S_ISGID) {
2141		inode->i_gid = dir->i_gid;
2142
2143		/* Directories are special, and always inherit S_ISGID */
2144		if (S_ISDIR(mode))
2145			mode |= S_ISGID;
2146		else if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP) &&
2147			 !in_group_p(inode->i_gid) &&
2148			 !capable_wrt_inode_uidgid(dir, CAP_FSETID))
2149			mode &= ~S_ISGID;
2150	} else
2151		inode->i_gid = current_fsgid();
2152	inode->i_mode = mode;
2153}
2154EXPORT_SYMBOL(inode_init_owner);
2155
2156/**
2157 * inode_owner_or_capable - check current task permissions to inode
2158 * @inode: inode being checked
2159 *
2160 * Return true if current either has CAP_FOWNER in a namespace with the
2161 * inode owner uid mapped, or owns the file.
2162 */
2163bool inode_owner_or_capable(const struct inode *inode)
2164{
2165	struct user_namespace *ns;
2166
2167	if (uid_eq(current_fsuid(), inode->i_uid))
2168		return true;
2169
2170	ns = current_user_ns();
2171	if (kuid_has_mapping(ns, inode->i_uid) && ns_capable(ns, CAP_FOWNER))
2172		return true;
2173	return false;
2174}
2175EXPORT_SYMBOL(inode_owner_or_capable);
2176
2177/*
2178 * Direct i/o helper functions
2179 */
2180static void __inode_dio_wait(struct inode *inode)
2181{
2182	wait_queue_head_t *wq = bit_waitqueue(&inode->i_state, __I_DIO_WAKEUP);
2183	DEFINE_WAIT_BIT(q, &inode->i_state, __I_DIO_WAKEUP);
2184
2185	do {
2186		prepare_to_wait(wq, &q.wq_entry, TASK_UNINTERRUPTIBLE);
2187		if (atomic_read(&inode->i_dio_count))
2188			schedule();
2189	} while (atomic_read(&inode->i_dio_count));
2190	finish_wait(wq, &q.wq_entry);
2191}
2192
2193/**
2194 * inode_dio_wait - wait for outstanding DIO requests to finish
2195 * @inode: inode to wait for
2196 *
2197 * Waits for all pending direct I/O requests to finish so that we can
2198 * proceed with a truncate or equivalent operation.
2199 *
2200 * Must be called under a lock that serializes taking new references
2201 * to i_dio_count, usually by inode->i_mutex.
2202 */
2203void inode_dio_wait(struct inode *inode)
2204{
2205	if (atomic_read(&inode->i_dio_count))
2206		__inode_dio_wait(inode);
2207}
2208EXPORT_SYMBOL(inode_dio_wait);
2209
2210/*
2211 * inode_set_flags - atomically set some inode flags
 
2212 *
2213 * Note: the caller should be holding i_mutex, or else be sure that
2214 * they have exclusive access to the inode structure (i.e., while the
2215 * inode is being instantiated).  The reason for the cmpxchg() loop
2216 * --- which wouldn't be necessary if all code paths which modify
2217 * i_flags actually followed this rule, is that there is at least one
2218 * code path which doesn't today so we use cmpxchg() out of an abundance
2219 * of caution.
2220 *
2221 * In the long run, i_mutex is overkill, and we should probably look
2222 * at using the i_lock spinlock to protect i_flags, and then make sure
2223 * it is so documented in include/linux/fs.h and that all code follows
2224 * the locking convention!!
2225 */
2226void inode_set_flags(struct inode *inode, unsigned int flags,
2227		     unsigned int mask)
2228{
2229	WARN_ON_ONCE(flags & ~mask);
2230	set_mask_bits(&inode->i_flags, mask, flags);
2231}
2232EXPORT_SYMBOL(inode_set_flags);
2233
2234void inode_nohighmem(struct inode *inode)
2235{
2236	mapping_set_gfp_mask(inode->i_mapping, GFP_USER);
2237}
2238EXPORT_SYMBOL(inode_nohighmem);
2239
2240/**
2241 * timestamp_truncate - Truncate timespec to a granularity
2242 * @t: Timespec
2243 * @inode: inode being updated
2244 *
2245 * Truncate a timespec to the granularity supported by the fs
2246 * containing the inode. Always rounds down. gran must
2247 * not be 0 nor greater than a second (NSEC_PER_SEC, or 10^9 ns).
2248 */
2249struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode)
2250{
2251	struct super_block *sb = inode->i_sb;
2252	unsigned int gran = sb->s_time_gran;
2253
2254	t.tv_sec = clamp(t.tv_sec, sb->s_time_min, sb->s_time_max);
2255	if (unlikely(t.tv_sec == sb->s_time_max || t.tv_sec == sb->s_time_min))
2256		t.tv_nsec = 0;
2257
2258	/* Avoid division in the common cases 1 ns and 1 s. */
2259	if (gran == 1)
2260		; /* nothing */
2261	else if (gran == NSEC_PER_SEC)
2262		t.tv_nsec = 0;
2263	else if (gran > 1 && gran < NSEC_PER_SEC)
2264		t.tv_nsec -= t.tv_nsec % gran;
2265	else
2266		WARN(1, "invalid file time granularity: %u", gran);
2267	return t;
2268}
2269EXPORT_SYMBOL(timestamp_truncate);
2270
2271/**
2272 * current_time - Return FS time
2273 * @inode: inode.
2274 *
2275 * Return the current time truncated to the time granularity supported by
2276 * the fs.
2277 *
2278 * Note that inode and inode->sb cannot be NULL.
2279 * Otherwise, the function warns and returns time without truncation.
2280 */
2281struct timespec64 current_time(struct inode *inode)
2282{
2283	struct timespec64 now;
2284
2285	ktime_get_coarse_real_ts64(&now);
2286
2287	if (unlikely(!inode->i_sb)) {
2288		WARN(1, "current_time() called with uninitialized super_block in the inode");
2289		return now;
2290	}
2291
2292	return timestamp_truncate(now, inode);
2293}
2294EXPORT_SYMBOL(current_time);
2295
2296/*
2297 * Generic function to check FS_IOC_SETFLAGS values and reject any invalid
2298 * configurations.
2299 *
2300 * Note: the caller should be holding i_mutex, or else be sure that they have
2301 * exclusive access to the inode structure.
2302 */
2303int vfs_ioc_setflags_prepare(struct inode *inode, unsigned int oldflags,
2304			     unsigned int flags)
2305{
2306	/*
2307	 * The IMMUTABLE and APPEND_ONLY flags can only be changed by
2308	 * the relevant capability.
2309	 *
2310	 * This test looks nicer. Thanks to Pauline Middelink
2311	 */
2312	if ((flags ^ oldflags) & (FS_APPEND_FL | FS_IMMUTABLE_FL) &&
2313	    !capable(CAP_LINUX_IMMUTABLE))
2314		return -EPERM;
2315
2316	return fscrypt_prepare_setflags(inode, oldflags, flags);
2317}
2318EXPORT_SYMBOL(vfs_ioc_setflags_prepare);
2319
2320/*
2321 * Generic function to check FS_IOC_FSSETXATTR values and reject any invalid
2322 * configurations.
2323 *
2324 * Note: the caller should be holding i_mutex, or else be sure that they have
2325 * exclusive access to the inode structure.
2326 */
2327int vfs_ioc_fssetxattr_check(struct inode *inode, const struct fsxattr *old_fa,
2328			     struct fsxattr *fa)
2329{
2330	/*
2331	 * Can't modify an immutable/append-only file unless we have
2332	 * appropriate permission.
2333	 */
2334	if ((old_fa->fsx_xflags ^ fa->fsx_xflags) &
2335			(FS_XFLAG_IMMUTABLE | FS_XFLAG_APPEND) &&
2336	    !capable(CAP_LINUX_IMMUTABLE))
2337		return -EPERM;
2338
2339	/*
2340	 * Project Quota ID state is only allowed to change from within the init
2341	 * namespace. Enforce that restriction only if we are trying to change
2342	 * the quota ID state. Everything else is allowed in user namespaces.
2343	 */
2344	if (current_user_ns() != &init_user_ns) {
2345		if (old_fa->fsx_projid != fa->fsx_projid)
2346			return -EINVAL;
2347		if ((old_fa->fsx_xflags ^ fa->fsx_xflags) &
2348				FS_XFLAG_PROJINHERIT)
2349			return -EINVAL;
2350	}
2351
2352	/* Check extent size hints. */
2353	if ((fa->fsx_xflags & FS_XFLAG_EXTSIZE) && !S_ISREG(inode->i_mode))
2354		return -EINVAL;
2355
2356	if ((fa->fsx_xflags & FS_XFLAG_EXTSZINHERIT) &&
2357			!S_ISDIR(inode->i_mode))
2358		return -EINVAL;
2359
2360	if ((fa->fsx_xflags & FS_XFLAG_COWEXTSIZE) &&
2361	    !S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode))
2362		return -EINVAL;
2363
2364	/*
2365	 * It is only valid to set the DAX flag on regular files and
2366	 * directories on filesystems.
2367	 */
2368	if ((fa->fsx_xflags & FS_XFLAG_DAX) &&
2369	    !(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode)))
2370		return -EINVAL;
2371
2372	/* Extent size hints of zero turn off the flags. */
2373	if (fa->fsx_extsize == 0)
2374		fa->fsx_xflags &= ~(FS_XFLAG_EXTSIZE | FS_XFLAG_EXTSZINHERIT);
2375	if (fa->fsx_cowextsize == 0)
2376		fa->fsx_xflags &= ~FS_XFLAG_COWEXTSIZE;
2377
2378	return 0;
2379}
2380EXPORT_SYMBOL(vfs_ioc_fssetxattr_check);