1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *	fs/libfs.c
   4 *	Library for filesystems writers.
   5 */
   6
   7#include <linux/blkdev.h>
   8#include <linux/export.h>
   9#include <linux/pagemap.h>
  10#include <linux/slab.h>
  11#include <linux/cred.h>
  12#include <linux/mount.h>
  13#include <linux/vfs.h>
  14#include <linux/quotaops.h>
  15#include <linux/mutex.h>
  16#include <linux/namei.h>
  17#include <linux/exportfs.h>
  18#include <linux/iversion.h>
  19#include <linux/writeback.h>
  20#include <linux/buffer_head.h> /* sync_mapping_buffers */
  21#include <linux/fs_context.h>
  22#include <linux/pseudo_fs.h>
  23#include <linux/fsnotify.h>
  24#include <linux/unicode.h>
  25#include <linux/fscrypt.h>
  26#include <linux/pidfs.h>
  27
  28#include <linux/uaccess.h>
  29
  30#include "internal.h"
  31
  32int simple_getattr(struct mnt_idmap *idmap, const struct path *path,
  33		   struct kstat *stat, u32 request_mask,
  34		   unsigned int query_flags)
  35{
  36	struct inode *inode = d_inode(path->dentry);
  37	generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat);
  38	stat->blocks = inode->i_mapping->nrpages << (PAGE_SHIFT - 9);
 
 
 
 
  39	return 0;
  40}
  41EXPORT_SYMBOL(simple_getattr);
  42
  43int simple_statfs(struct dentry *dentry, struct kstatfs *buf)
  44{
  45	u64 id = huge_encode_dev(dentry->d_sb->s_dev);
  46
  47	buf->f_fsid = u64_to_fsid(id);
  48	buf->f_type = dentry->d_sb->s_magic;
  49	buf->f_bsize = PAGE_SIZE;
  50	buf->f_namelen = NAME_MAX;
  51	return 0;
  52}
  53EXPORT_SYMBOL(simple_statfs);
  54
  55/*
  56 * Retaining negative dentries for an in-memory filesystem just wastes
  57 * memory and lookup time: arrange for them to be deleted immediately.
  58 */
  59int always_delete_dentry(const struct dentry *dentry)
  60{
  61	return 1;
  62}
  63EXPORT_SYMBOL(always_delete_dentry);
  64
  65const struct dentry_operations simple_dentry_operations = {
  66	.d_delete = always_delete_dentry,
  67};
  68EXPORT_SYMBOL(simple_dentry_operations);
  69
  70/*
  71 * Lookup the data. This is trivial - if the dentry didn't already
  72 * exist, we know it is negative.  Set d_op to delete negative dentries.
  73 */
  74struct dentry *simple_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags)
  75{
 
 
 
 
  76	if (dentry->d_name.len > NAME_MAX)
  77		return ERR_PTR(-ENAMETOOLONG);
  78	if (!dentry->d_sb->s_d_op)
  79		d_set_d_op(dentry, &simple_dentry_operations);
  80
  81	if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir))
  82		return NULL;
  83
  84	d_add(dentry, NULL);
  85	return NULL;
  86}
  87EXPORT_SYMBOL(simple_lookup);
  88
  89int dcache_dir_open(struct inode *inode, struct file *file)
  90{
  91	file->private_data = d_alloc_cursor(file->f_path.dentry);
 
 
  92
  93	return file->private_data ? 0 : -ENOMEM;
  94}
  95EXPORT_SYMBOL(dcache_dir_open);
  96
  97int dcache_dir_close(struct inode *inode, struct file *file)
  98{
  99	dput(file->private_data);
 100	return 0;
 101}
 102EXPORT_SYMBOL(dcache_dir_close);
 103
 104/* parent is locked at least shared */
 105/*
 106 * Returns an element of siblings' list.
 107 * We are looking for <count>th positive after <p>; if
 108 * found, dentry is grabbed and returned to caller.
 109 * If no such element exists, NULL is returned.
 110 */
 111static struct dentry *scan_positives(struct dentry *cursor,
 112					struct hlist_node **p,
 113					loff_t count,
 114					struct dentry *last)
 115{
 116	struct dentry *dentry = cursor->d_parent, *found = NULL;
 117
 118	spin_lock(&dentry->d_lock);
 119	while (*p) {
 120		struct dentry *d = hlist_entry(*p, struct dentry, d_sib);
 121		p = &d->d_sib.next;
 122		// we must at least skip cursors, to avoid livelocks
 123		if (d->d_flags & DCACHE_DENTRY_CURSOR)
 124			continue;
 125		if (simple_positive(d) && !--count) {
 126			spin_lock_nested(&d->d_lock, DENTRY_D_LOCK_NESTED);
 127			if (simple_positive(d))
 128				found = dget_dlock(d);
 129			spin_unlock(&d->d_lock);
 130			if (likely(found))
 131				break;
 132			count = 1;
 133		}
 134		if (need_resched()) {
 135			if (!hlist_unhashed(&cursor->d_sib))
 136				__hlist_del(&cursor->d_sib);
 137			hlist_add_behind(&cursor->d_sib, &d->d_sib);
 138			p = &cursor->d_sib.next;
 139			spin_unlock(&dentry->d_lock);
 140			cond_resched();
 141			spin_lock(&dentry->d_lock);
 142		}
 143	}
 144	spin_unlock(&dentry->d_lock);
 145	dput(last);
 146	return found;
 147}
 148
 149loff_t dcache_dir_lseek(struct file *file, loff_t offset, int whence)
 150{
 151	struct dentry *dentry = file->f_path.dentry;
 152	switch (whence) {
 
 153		case 1:
 154			offset += file->f_pos;
 155			fallthrough;
 156		case 0:
 157			if (offset >= 0)
 158				break;
 159			fallthrough;
 160		default:
 
 161			return -EINVAL;
 162	}
 163	if (offset != file->f_pos) {
 164		struct dentry *cursor = file->private_data;
 165		struct dentry *to = NULL;
 166
 167		inode_lock_shared(dentry->d_inode);
 168
 169		if (offset > 2)
 170			to = scan_positives(cursor, &dentry->d_children.first,
 171					    offset - 2, NULL);
 172		spin_lock(&dentry->d_lock);
 173		hlist_del_init(&cursor->d_sib);
 174		if (to)
 175			hlist_add_behind(&cursor->d_sib, &to->d_sib);
 176		spin_unlock(&dentry->d_lock);
 177		dput(to);
 178
 179		file->f_pos = offset;
 
 
 
 
 180
 181		inode_unlock_shared(dentry->d_inode);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 182	}
 
 183	return offset;
 184}
 185EXPORT_SYMBOL(dcache_dir_lseek);
 
 
 
 
 
 186
 187/*
 188 * Directory is locked and all positive dentries in it are safe, since
 189 * for ramfs-type trees they can't go away without unlink() or rmdir(),
 190 * both impossible due to the lock on directory.
 191 */
 192
 193int dcache_readdir(struct file *file, struct dir_context *ctx)
 194{
 195	struct dentry *dentry = file->f_path.dentry;
 196	struct dentry *cursor = file->private_data;
 197	struct dentry *next = NULL;
 198	struct hlist_node **p;
 
 199
 200	if (!dir_emit_dots(file, ctx))
 201		return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 202
 203	if (ctx->pos == 2)
 204		p = &dentry->d_children.first;
 205	else
 206		p = &cursor->d_sib.next;
 207
 208	while ((next = scan_positives(cursor, p, 1, next)) != NULL) {
 209		if (!dir_emit(ctx, next->d_name.name, next->d_name.len,
 210			      d_inode(next)->i_ino,
 211			      fs_umode_to_dtype(d_inode(next)->i_mode)))
 212			break;
 213		ctx->pos++;
 214		p = &next->d_sib.next;
 
 
 
 
 
 
 
 
 
 
 
 
 
 215	}
 216	spin_lock(&dentry->d_lock);
 217	hlist_del_init(&cursor->d_sib);
 218	if (next)
 219		hlist_add_before(&cursor->d_sib, &next->d_sib);
 220	spin_unlock(&dentry->d_lock);
 221	dput(next);
 222
 223	return 0;
 224}
 225EXPORT_SYMBOL(dcache_readdir);
 226
 227ssize_t generic_read_dir(struct file *filp, char __user *buf, size_t siz, loff_t *ppos)
 228{
 229	return -EISDIR;
 230}
 231EXPORT_SYMBOL(generic_read_dir);
 232
 233const struct file_operations simple_dir_operations = {
 234	.open		= dcache_dir_open,
 235	.release	= dcache_dir_close,
 236	.llseek		= dcache_dir_lseek,
 237	.read		= generic_read_dir,
 238	.iterate_shared	= dcache_readdir,
 239	.fsync		= noop_fsync,
 240};
 241EXPORT_SYMBOL(simple_dir_operations);
 242
 243const struct inode_operations simple_dir_inode_operations = {
 244	.lookup		= simple_lookup,
 245};
 246EXPORT_SYMBOL(simple_dir_inode_operations);
 247
 248/* simple_offset_add() never assigns these to a dentry */
 249enum {
 250	DIR_OFFSET_FIRST	= 2,		/* Find first real entry */
 251	DIR_OFFSET_EOD		= S32_MAX,
 252};
 253
 254/* simple_offset_add() allocation range */
 255enum {
 256	DIR_OFFSET_MIN		= DIR_OFFSET_FIRST + 1,
 257	DIR_OFFSET_MAX		= DIR_OFFSET_EOD - 1,
 258};
 259
 260static void offset_set(struct dentry *dentry, long offset)
 261{
 262	dentry->d_fsdata = (void *)offset;
 263}
 264
 265static long dentry2offset(struct dentry *dentry)
 266{
 267	return (long)dentry->d_fsdata;
 268}
 269
 270static struct lock_class_key simple_offset_lock_class;
 271
 272/**
 273 * simple_offset_init - initialize an offset_ctx
 274 * @octx: directory offset map to be initialized
 275 *
 276 */
 277void simple_offset_init(struct offset_ctx *octx)
 278{
 279	mt_init_flags(&octx->mt, MT_FLAGS_ALLOC_RANGE);
 280	lockdep_set_class(&octx->mt.ma_lock, &simple_offset_lock_class);
 281	octx->next_offset = DIR_OFFSET_MIN;
 282}
 283
 284/**
 285 * simple_offset_add - Add an entry to a directory's offset map
 286 * @octx: directory offset ctx to be updated
 287 * @dentry: new dentry being added
 288 *
 289 * Returns zero on success. @octx and the dentry's offset are updated.
 290 * Otherwise, a negative errno value is returned.
 291 */
 292int simple_offset_add(struct offset_ctx *octx, struct dentry *dentry)
 293{
 294	unsigned long offset;
 295	int ret;
 296
 297	if (dentry2offset(dentry) != 0)
 298		return -EBUSY;
 299
 300	ret = mtree_alloc_cyclic(&octx->mt, &offset, dentry, DIR_OFFSET_MIN,
 301				 DIR_OFFSET_MAX, &octx->next_offset,
 302				 GFP_KERNEL);
 303	if (unlikely(ret < 0))
 304		return ret == -EBUSY ? -ENOSPC : ret;
 305
 306	offset_set(dentry, offset);
 307	return 0;
 308}
 309
 310static int simple_offset_replace(struct offset_ctx *octx, struct dentry *dentry,
 311				 long offset)
 312{
 313	int ret;
 314
 315	ret = mtree_store(&octx->mt, offset, dentry, GFP_KERNEL);
 316	if (ret)
 317		return ret;
 318	offset_set(dentry, offset);
 319	return 0;
 320}
 321
 322/**
 323 * simple_offset_remove - Remove an entry to a directory's offset map
 324 * @octx: directory offset ctx to be updated
 325 * @dentry: dentry being removed
 326 *
 327 */
 328void simple_offset_remove(struct offset_ctx *octx, struct dentry *dentry)
 329{
 330	long offset;
 331
 332	offset = dentry2offset(dentry);
 333	if (offset == 0)
 334		return;
 335
 336	mtree_erase(&octx->mt, offset);
 337	offset_set(dentry, 0);
 338}
 339
 340/**
 341 * simple_offset_rename - handle directory offsets for rename
 342 * @old_dir: parent directory of source entry
 343 * @old_dentry: dentry of source entry
 344 * @new_dir: parent_directory of destination entry
 345 * @new_dentry: dentry of destination
 346 *
 347 * Caller provides appropriate serialization.
 348 *
 349 * User space expects the directory offset value of the replaced
 350 * (new) directory entry to be unchanged after a rename.
 351 *
 352 * Returns zero on success, a negative errno value on failure.
 353 */
 354int simple_offset_rename(struct inode *old_dir, struct dentry *old_dentry,
 355			 struct inode *new_dir, struct dentry *new_dentry)
 356{
 357	struct offset_ctx *old_ctx = old_dir->i_op->get_offset_ctx(old_dir);
 358	struct offset_ctx *new_ctx = new_dir->i_op->get_offset_ctx(new_dir);
 359	long new_offset = dentry2offset(new_dentry);
 360
 361	simple_offset_remove(old_ctx, old_dentry);
 362
 363	if (new_offset) {
 364		offset_set(new_dentry, 0);
 365		return simple_offset_replace(new_ctx, old_dentry, new_offset);
 366	}
 367	return simple_offset_add(new_ctx, old_dentry);
 368}
 369
 370/**
 371 * simple_offset_rename_exchange - exchange rename with directory offsets
 372 * @old_dir: parent of dentry being moved
 373 * @old_dentry: dentry being moved
 374 * @new_dir: destination parent
 375 * @new_dentry: destination dentry
 376 *
 377 * This API preserves the directory offset values. Caller provides
 378 * appropriate serialization.
 379 *
 380 * Returns zero on success. Otherwise a negative errno is returned and the
 381 * rename is rolled back.
 382 */
 383int simple_offset_rename_exchange(struct inode *old_dir,
 384				  struct dentry *old_dentry,
 385				  struct inode *new_dir,
 386				  struct dentry *new_dentry)
 387{
 388	struct offset_ctx *old_ctx = old_dir->i_op->get_offset_ctx(old_dir);
 389	struct offset_ctx *new_ctx = new_dir->i_op->get_offset_ctx(new_dir);
 390	long old_index = dentry2offset(old_dentry);
 391	long new_index = dentry2offset(new_dentry);
 392	int ret;
 393
 394	simple_offset_remove(old_ctx, old_dentry);
 395	simple_offset_remove(new_ctx, new_dentry);
 396
 397	ret = simple_offset_replace(new_ctx, old_dentry, new_index);
 398	if (ret)
 399		goto out_restore;
 400
 401	ret = simple_offset_replace(old_ctx, new_dentry, old_index);
 402	if (ret) {
 403		simple_offset_remove(new_ctx, old_dentry);
 404		goto out_restore;
 405	}
 406
 407	ret = simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry);
 408	if (ret) {
 409		simple_offset_remove(new_ctx, old_dentry);
 410		simple_offset_remove(old_ctx, new_dentry);
 411		goto out_restore;
 412	}
 413	return 0;
 414
 415out_restore:
 416	(void)simple_offset_replace(old_ctx, old_dentry, old_index);
 417	(void)simple_offset_replace(new_ctx, new_dentry, new_index);
 418	return ret;
 419}
 420
 421/**
 422 * simple_offset_destroy - Release offset map
 423 * @octx: directory offset ctx that is about to be destroyed
 424 *
 425 * During fs teardown (eg. umount), a directory's offset map might still
 426 * contain entries. xa_destroy() cleans out anything that remains.
 427 */
 428void simple_offset_destroy(struct offset_ctx *octx)
 429{
 430	mtree_destroy(&octx->mt);
 431}
 432
 433/**
 434 * offset_dir_llseek - Advance the read position of a directory descriptor
 435 * @file: an open directory whose position is to be updated
 436 * @offset: a byte offset
 437 * @whence: enumerator describing the starting position for this update
 438 *
 439 * SEEK_END, SEEK_DATA, and SEEK_HOLE are not supported for directories.
 440 *
 441 * Returns the updated read position if successful; otherwise a
 442 * negative errno is returned and the read position remains unchanged.
 443 */
 444static loff_t offset_dir_llseek(struct file *file, loff_t offset, int whence)
 445{
 446	switch (whence) {
 447	case SEEK_CUR:
 448		offset += file->f_pos;
 449		fallthrough;
 450	case SEEK_SET:
 451		if (offset >= 0)
 452			break;
 453		fallthrough;
 454	default:
 455		return -EINVAL;
 456	}
 457
 458	return vfs_setpos(file, offset, LONG_MAX);
 459}
 460
 461static struct dentry *find_positive_dentry(struct dentry *parent,
 462					   struct dentry *dentry,
 463					   bool next)
 464{
 465	struct dentry *found = NULL;
 466
 467	spin_lock(&parent->d_lock);
 468	if (next)
 469		dentry = d_next_sibling(dentry);
 470	else if (!dentry)
 471		dentry = d_first_child(parent);
 472	hlist_for_each_entry_from(dentry, d_sib) {
 473		if (!simple_positive(dentry))
 474			continue;
 475		spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
 476		if (simple_positive(dentry))
 477			found = dget_dlock(dentry);
 478		spin_unlock(&dentry->d_lock);
 479		if (likely(found))
 480			break;
 481	}
 482	spin_unlock(&parent->d_lock);
 483	return found;
 484}
 485
 486static noinline_for_stack struct dentry *
 487offset_dir_lookup(struct dentry *parent, loff_t offset)
 488{
 489	struct inode *inode = d_inode(parent);
 490	struct offset_ctx *octx = inode->i_op->get_offset_ctx(inode);
 491	struct dentry *child, *found = NULL;
 492
 493	MA_STATE(mas, &octx->mt, offset, offset);
 494
 495	if (offset == DIR_OFFSET_FIRST)
 496		found = find_positive_dentry(parent, NULL, false);
 497	else {
 498		rcu_read_lock();
 499		child = mas_find(&mas, DIR_OFFSET_MAX);
 500		found = find_positive_dentry(parent, child, false);
 501		rcu_read_unlock();
 502	}
 503	return found;
 504}
 505
 506static bool offset_dir_emit(struct dir_context *ctx, struct dentry *dentry)
 507{
 508	struct inode *inode = d_inode(dentry);
 509
 510	return dir_emit(ctx, dentry->d_name.name, dentry->d_name.len,
 511			inode->i_ino, fs_umode_to_dtype(inode->i_mode));
 512}
 513
 514static void offset_iterate_dir(struct file *file, struct dir_context *ctx)
 515{
 516	struct dentry *dir = file->f_path.dentry;
 517	struct dentry *dentry;
 518
 519	dentry = offset_dir_lookup(dir, ctx->pos);
 520	if (!dentry)
 521		goto out_eod;
 522	while (true) {
 523		struct dentry *next;
 524
 525		ctx->pos = dentry2offset(dentry);
 526		if (!offset_dir_emit(ctx, dentry))
 527			break;
 528
 529		next = find_positive_dentry(dir, dentry, true);
 530		dput(dentry);
 531
 532		if (!next)
 533			goto out_eod;
 534		dentry = next;
 535	}
 536	dput(dentry);
 537	return;
 538
 539out_eod:
 540	ctx->pos = DIR_OFFSET_EOD;
 541}
 542
 543/**
 544 * offset_readdir - Emit entries starting at offset @ctx->pos
 545 * @file: an open directory to iterate over
 546 * @ctx: directory iteration context
 547 *
 548 * Caller must hold @file's i_rwsem to prevent insertion or removal of
 549 * entries during this call.
 550 *
 551 * On entry, @ctx->pos contains an offset that represents the first entry
 552 * to be read from the directory.
 553 *
 554 * The operation continues until there are no more entries to read, or
 555 * until the ctx->actor indicates there is no more space in the caller's
 556 * output buffer.
 557 *
 558 * On return, @ctx->pos contains an offset that will read the next entry
 559 * in this directory when offset_readdir() is called again with @ctx.
 560 * Caller places this value in the d_off field of the last entry in the
 561 * user's buffer.
 562 *
 563 * Return values:
 564 *   %0 - Complete
 565 */
 566static int offset_readdir(struct file *file, struct dir_context *ctx)
 567{
 568	struct dentry *dir = file->f_path.dentry;
 569
 570	lockdep_assert_held(&d_inode(dir)->i_rwsem);
 571
 572	if (!dir_emit_dots(file, ctx))
 573		return 0;
 574	if (ctx->pos != DIR_OFFSET_EOD)
 575		offset_iterate_dir(file, ctx);
 576	return 0;
 577}
 578
 579const struct file_operations simple_offset_dir_operations = {
 580	.llseek		= offset_dir_llseek,
 581	.iterate_shared	= offset_readdir,
 582	.read		= generic_read_dir,
 583	.fsync		= noop_fsync,
 584};
 585
 586static struct dentry *find_next_child(struct dentry *parent, struct dentry *prev)
 587{
 588	struct dentry *child = NULL, *d;
 589
 590	spin_lock(&parent->d_lock);
 591	d = prev ? d_next_sibling(prev) : d_first_child(parent);
 592	hlist_for_each_entry_from(d, d_sib) {
 593		if (simple_positive(d)) {
 594			spin_lock_nested(&d->d_lock, DENTRY_D_LOCK_NESTED);
 595			if (simple_positive(d))
 596				child = dget_dlock(d);
 597			spin_unlock(&d->d_lock);
 598			if (likely(child))
 599				break;
 600		}
 601	}
 602	spin_unlock(&parent->d_lock);
 603	dput(prev);
 604	return child;
 605}
 606
 607void simple_recursive_removal(struct dentry *dentry,
 608                              void (*callback)(struct dentry *))
 609{
 610	struct dentry *this = dget(dentry);
 611	while (true) {
 612		struct dentry *victim = NULL, *child;
 613		struct inode *inode = this->d_inode;
 614
 615		inode_lock(inode);
 616		if (d_is_dir(this))
 617			inode->i_flags |= S_DEAD;
 618		while ((child = find_next_child(this, victim)) == NULL) {
 619			// kill and ascend
 620			// update metadata while it's still locked
 621			inode_set_ctime_current(inode);
 622			clear_nlink(inode);
 623			inode_unlock(inode);
 624			victim = this;
 625			this = this->d_parent;
 626			inode = this->d_inode;
 627			inode_lock(inode);
 628			if (simple_positive(victim)) {
 629				d_invalidate(victim);	// avoid lost mounts
 630				if (d_is_dir(victim))
 631					fsnotify_rmdir(inode, victim);
 632				else
 633					fsnotify_unlink(inode, victim);
 634				if (callback)
 635					callback(victim);
 636				dput(victim);		// unpin it
 637			}
 638			if (victim == dentry) {
 639				inode_set_mtime_to_ts(inode,
 640						      inode_set_ctime_current(inode));
 641				if (d_is_dir(dentry))
 642					drop_nlink(inode);
 643				inode_unlock(inode);
 644				dput(dentry);
 645				return;
 646			}
 647		}
 648		inode_unlock(inode);
 649		this = child;
 650	}
 651}
 652EXPORT_SYMBOL(simple_recursive_removal);
 653
 654static const struct super_operations simple_super_operations = {
 655	.statfs		= simple_statfs,
 656};
 657
 658static int pseudo_fs_fill_super(struct super_block *s, struct fs_context *fc)
 
 
 
 
 
 
 659{
 660	struct pseudo_fs_context *ctx = fc->fs_private;
 
 661	struct inode *root;
 
 662
 
 
 
 
 663	s->s_maxbytes = MAX_LFS_FILESIZE;
 664	s->s_blocksize = PAGE_SIZE;
 665	s->s_blocksize_bits = PAGE_SHIFT;
 666	s->s_magic = ctx->magic;
 667	s->s_op = ctx->ops ?: &simple_super_operations;
 668	s->s_xattr = ctx->xattr;
 669	s->s_time_gran = 1;
 670	root = new_inode(s);
 671	if (!root)
 672		return -ENOMEM;
 673
 674	/*
 675	 * since this is the first inode, make it number 1. New inodes created
 676	 * after this must take care not to collide with it (by passing
 677	 * max_reserved of 1 to iunique).
 678	 */
 679	root->i_ino = 1;
 680	root->i_mode = S_IFDIR | S_IRUSR | S_IWUSR;
 681	simple_inode_init_ts(root);
 682	s->s_root = d_make_root(root);
 683	if (!s->s_root)
 684		return -ENOMEM;
 685	s->s_d_op = ctx->dops;
 686	return 0;
 687}
 688
 689static int pseudo_fs_get_tree(struct fs_context *fc)
 690{
 691	return get_tree_nodev(fc, pseudo_fs_fill_super);
 692}
 693
 694static void pseudo_fs_free(struct fs_context *fc)
 695{
 696	kfree(fc->fs_private);
 697}
 698
 699static const struct fs_context_operations pseudo_fs_context_ops = {
 700	.free		= pseudo_fs_free,
 701	.get_tree	= pseudo_fs_get_tree,
 702};
 703
 704/*
 705 * Common helper for pseudo-filesystems (sockfs, pipefs, bdev - stuff that
 706 * will never be mountable)
 707 */
 708struct pseudo_fs_context *init_pseudo(struct fs_context *fc,
 709					unsigned long magic)
 710{
 711	struct pseudo_fs_context *ctx;
 712
 713	ctx = kzalloc(sizeof(struct pseudo_fs_context), GFP_KERNEL);
 714	if (likely(ctx)) {
 715		ctx->magic = magic;
 716		fc->fs_private = ctx;
 717		fc->ops = &pseudo_fs_context_ops;
 718		fc->sb_flags |= SB_NOUSER;
 719		fc->global = true;
 720	}
 721	return ctx;
 
 
 
 
 
 
 
 
 722}
 723EXPORT_SYMBOL(init_pseudo);
 724
 725int simple_open(struct inode *inode, struct file *file)
 726{
 727	if (inode->i_private)
 728		file->private_data = inode->i_private;
 729	return 0;
 730}
 731EXPORT_SYMBOL(simple_open);
 732
 733int simple_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
 734{
 735	struct inode *inode = d_inode(old_dentry);
 736
 737	inode_set_mtime_to_ts(dir,
 738			      inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode)));
 739	inc_nlink(inode);
 740	ihold(inode);
 741	dget(dentry);
 742	d_instantiate(dentry, inode);
 743	return 0;
 744}
 745EXPORT_SYMBOL(simple_link);
 746
 747int simple_empty(struct dentry *dentry)
 748{
 749	struct dentry *child;
 750	int ret = 0;
 751
 752	spin_lock(&dentry->d_lock);
 753	hlist_for_each_entry(child, &dentry->d_children, d_sib) {
 754		spin_lock_nested(&child->d_lock, DENTRY_D_LOCK_NESTED);
 755		if (simple_positive(child)) {
 756			spin_unlock(&child->d_lock);
 757			goto out;
 758		}
 759		spin_unlock(&child->d_lock);
 760	}
 761	ret = 1;
 762out:
 763	spin_unlock(&dentry->d_lock);
 764	return ret;
 765}
 766EXPORT_SYMBOL(simple_empty);
 767
 768int simple_unlink(struct inode *dir, struct dentry *dentry)
 769{
 770	struct inode *inode = d_inode(dentry);
 771
 772	inode_set_mtime_to_ts(dir,
 773			      inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode)));
 774	drop_nlink(inode);
 775	dput(dentry);
 776	return 0;
 777}
 778EXPORT_SYMBOL(simple_unlink);
 779
 780int simple_rmdir(struct inode *dir, struct dentry *dentry)
 781{
 782	if (!simple_empty(dentry))
 783		return -ENOTEMPTY;
 784
 785	drop_nlink(d_inode(dentry));
 786	simple_unlink(dir, dentry);
 787	drop_nlink(dir);
 788	return 0;
 789}
 790EXPORT_SYMBOL(simple_rmdir);
 791
 792/**
 793 * simple_rename_timestamp - update the various inode timestamps for rename
 794 * @old_dir: old parent directory
 795 * @old_dentry: dentry that is being renamed
 796 * @new_dir: new parent directory
 797 * @new_dentry: target for rename
 798 *
 799 * POSIX mandates that the old and new parent directories have their ctime and
 800 * mtime updated, and that inodes of @old_dentry and @new_dentry (if any), have
 801 * their ctime updated.
 802 */
 803void simple_rename_timestamp(struct inode *old_dir, struct dentry *old_dentry,
 804			     struct inode *new_dir, struct dentry *new_dentry)
 805{
 806	struct inode *newino = d_inode(new_dentry);
 807
 808	inode_set_mtime_to_ts(old_dir, inode_set_ctime_current(old_dir));
 809	if (new_dir != old_dir)
 810		inode_set_mtime_to_ts(new_dir,
 811				      inode_set_ctime_current(new_dir));
 812	inode_set_ctime_current(d_inode(old_dentry));
 813	if (newino)
 814		inode_set_ctime_current(newino);
 815}
 816EXPORT_SYMBOL_GPL(simple_rename_timestamp);
 817
 818int simple_rename_exchange(struct inode *old_dir, struct dentry *old_dentry,
 819			   struct inode *new_dir, struct dentry *new_dentry)
 820{
 821	bool old_is_dir = d_is_dir(old_dentry);
 822	bool new_is_dir = d_is_dir(new_dentry);
 823
 824	if (old_dir != new_dir && old_is_dir != new_is_dir) {
 825		if (old_is_dir) {
 826			drop_nlink(old_dir);
 827			inc_nlink(new_dir);
 828		} else {
 829			drop_nlink(new_dir);
 830			inc_nlink(old_dir);
 831		}
 832	}
 833	simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);
 834	return 0;
 835}
 836EXPORT_SYMBOL_GPL(simple_rename_exchange);
 837
 838int simple_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 839		  struct dentry *old_dentry, struct inode *new_dir,
 840		  struct dentry *new_dentry, unsigned int flags)
 841{
 842	int they_are_dirs = d_is_dir(old_dentry);
 843
 844	if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE))
 845		return -EINVAL;
 846
 847	if (flags & RENAME_EXCHANGE)
 848		return simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry);
 849
 850	if (!simple_empty(new_dentry))
 851		return -ENOTEMPTY;
 852
 853	if (d_really_is_positive(new_dentry)) {
 854		simple_unlink(new_dir, new_dentry);
 855		if (they_are_dirs) {
 856			drop_nlink(d_inode(new_dentry));
 857			drop_nlink(old_dir);
 858		}
 859	} else if (they_are_dirs) {
 860		drop_nlink(old_dir);
 861		inc_nlink(new_dir);
 862	}
 863
 864	simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);
 
 
 865	return 0;
 866}
 867EXPORT_SYMBOL(simple_rename);
 868
 869/**
 870 * simple_setattr - setattr for simple filesystem
 871 * @idmap: idmap of the target mount
 872 * @dentry: dentry
 873 * @iattr: iattr structure
 874 *
 875 * Returns 0 on success, -error on failure.
 876 *
 877 * simple_setattr is a simple ->setattr implementation without a proper
 878 * implementation of size changes.
 879 *
 880 * It can either be used for in-memory filesystems or special files
 881 * on simple regular filesystems.  Anything that needs to change on-disk
 882 * or wire state on size changes needs its own setattr method.
 883 */
 884int simple_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 885		   struct iattr *iattr)
 886{
 887	struct inode *inode = d_inode(dentry);
 888	int error;
 889
 890	error = setattr_prepare(idmap, dentry, iattr);
 
 
 891	if (error)
 892		return error;
 893
 894	if (iattr->ia_valid & ATTR_SIZE)
 895		truncate_setsize(inode, iattr->ia_size);
 896	setattr_copy(idmap, inode, iattr);
 897	mark_inode_dirty(inode);
 898	return 0;
 899}
 900EXPORT_SYMBOL(simple_setattr);
 901
 902static int simple_read_folio(struct file *file, struct folio *folio)
 903{
 904	folio_zero_range(folio, 0, folio_size(folio));
 905	flush_dcache_folio(folio);
 906	folio_mark_uptodate(folio);
 907	folio_unlock(folio);
 908	return 0;
 909}
 910
 911int simple_write_begin(struct file *file, struct address_space *mapping,
 912			loff_t pos, unsigned len,
 913			struct folio **foliop, void **fsdata)
 914{
 915	struct folio *folio;
 
 916
 917	folio = __filemap_get_folio(mapping, pos / PAGE_SIZE, FGP_WRITEBEGIN,
 918			mapping_gfp_mask(mapping));
 919	if (IS_ERR(folio))
 920		return PTR_ERR(folio);
 
 921
 922	*foliop = folio;
 923
 924	if (!folio_test_uptodate(folio) && (len != folio_size(folio))) {
 925		size_t from = offset_in_folio(folio, pos);
 926
 927		folio_zero_segments(folio, 0, from,
 928				from + len, folio_size(folio));
 929	}
 930	return 0;
 931}
 932EXPORT_SYMBOL(simple_write_begin);
 933
 934/**
 935 * simple_write_end - .write_end helper for non-block-device FSes
 936 * @file: See .write_end of address_space_operations
 
 937 * @mapping: 		"
 938 * @pos: 		"
 939 * @len: 		"
 940 * @copied: 		"
 941 * @folio: 		"
 942 * @fsdata: 		"
 943 *
 944 * simple_write_end does the minimum needed for updating a folio after
 945 * writing is done. It has the same API signature as the .write_end of
 946 * address_space_operations vector. So it can just be set onto .write_end for
 947 * FSes that don't need any other processing. i_mutex is assumed to be held.
 948 * Block based filesystems should use generic_write_end().
 949 * NOTE: Even though i_size might get updated by this function, mark_inode_dirty
 950 * is not called, so a filesystem that actually does store data in .write_inode
 951 * should extend on what's done here with a call to mark_inode_dirty() in the
 952 * case that i_size has changed.
 953 *
 954 * Use *ONLY* with simple_read_folio()
 955 */
 956static int simple_write_end(struct file *file, struct address_space *mapping,
 957			loff_t pos, unsigned len, unsigned copied,
 958			struct folio *folio, void *fsdata)
 959{
 960	struct inode *inode = folio->mapping->host;
 961	loff_t last_pos = pos + copied;
 962
 963	/* zero the stale part of the folio if we did a short copy */
 964	if (!folio_test_uptodate(folio)) {
 965		if (copied < len) {
 966			size_t from = offset_in_folio(folio, pos);
 967
 968			folio_zero_range(folio, from + copied, len - copied);
 969		}
 970		folio_mark_uptodate(folio);
 971	}
 
 
 
 972	/*
 973	 * No need to use i_size_read() here, the i_size
 974	 * cannot change under us because we hold the i_mutex.
 975	 */
 976	if (last_pos > inode->i_size)
 977		i_size_write(inode, last_pos);
 978
 979	folio_mark_dirty(folio);
 980	folio_unlock(folio);
 981	folio_put(folio);
 982
 983	return copied;
 984}
 985
 986/*
 987 * Provides ramfs-style behavior: data in the pagecache, but no writeback.
 988 */
 989const struct address_space_operations ram_aops = {
 990	.read_folio	= simple_read_folio,
 991	.write_begin	= simple_write_begin,
 992	.write_end	= simple_write_end,
 993	.dirty_folio	= noop_dirty_folio,
 994};
 995EXPORT_SYMBOL(ram_aops);
 996
 997/*
 998 * the inodes created here are not hashed. If you use iunique to generate
 999 * unique inode values later for this filesystem, then you must take care
1000 * to pass it an appropriate max_reserved value to avoid collisions.
1001 */
1002int simple_fill_super(struct super_block *s, unsigned long magic,
1003		      const struct tree_descr *files)
1004{
1005	struct inode *inode;
 
1006	struct dentry *dentry;
1007	int i;
1008
1009	s->s_blocksize = PAGE_SIZE;
1010	s->s_blocksize_bits = PAGE_SHIFT;
1011	s->s_magic = magic;
1012	s->s_op = &simple_super_operations;
1013	s->s_time_gran = 1;
1014
1015	inode = new_inode(s);
1016	if (!inode)
1017		return -ENOMEM;
1018	/*
1019	 * because the root inode is 1, the files array must not contain an
1020	 * entry at index 1
1021	 */
1022	inode->i_ino = 1;
1023	inode->i_mode = S_IFDIR | 0755;
1024	simple_inode_init_ts(inode);
1025	inode->i_op = &simple_dir_inode_operations;
1026	inode->i_fop = &simple_dir_operations;
1027	set_nlink(inode, 2);
1028	s->s_root = d_make_root(inode);
1029	if (!s->s_root)
1030		return -ENOMEM;
1031	for (i = 0; !files->name || files->name[0]; i++, files++) {
1032		if (!files->name)
1033			continue;
1034
1035		/* warn if it tries to conflict with the root inode */
1036		if (unlikely(i == 1))
1037			printk(KERN_WARNING "%s: %s passed in a files array"
1038				"with an index of 1!\n", __func__,
1039				s->s_type->name);
1040
1041		dentry = d_alloc_name(s->s_root, files->name);
1042		if (!dentry)
1043			return -ENOMEM;
1044		inode = new_inode(s);
1045		if (!inode) {
1046			dput(dentry);
1047			return -ENOMEM;
1048		}
1049		inode->i_mode = S_IFREG | files->mode;
1050		simple_inode_init_ts(inode);
1051		inode->i_fop = files->ops;
1052		inode->i_ino = i;
1053		d_add(dentry, inode);
1054	}
 
1055	return 0;
 
 
 
 
 
1056}
1057EXPORT_SYMBOL(simple_fill_super);
1058
1059static DEFINE_SPINLOCK(pin_fs_lock);
1060
1061int simple_pin_fs(struct file_system_type *type, struct vfsmount **mount, int *count)
1062{
1063	struct vfsmount *mnt = NULL;
1064	spin_lock(&pin_fs_lock);
1065	if (unlikely(!*mount)) {
1066		spin_unlock(&pin_fs_lock);
1067		mnt = vfs_kern_mount(type, SB_KERNMOUNT, type->name, NULL);
1068		if (IS_ERR(mnt))
1069			return PTR_ERR(mnt);
1070		spin_lock(&pin_fs_lock);
1071		if (!*mount)
1072			*mount = mnt;
1073	}
1074	mntget(*mount);
1075	++*count;
1076	spin_unlock(&pin_fs_lock);
1077	mntput(mnt);
1078	return 0;
1079}
1080EXPORT_SYMBOL(simple_pin_fs);
1081
1082void simple_release_fs(struct vfsmount **mount, int *count)
1083{
1084	struct vfsmount *mnt;
1085	spin_lock(&pin_fs_lock);
1086	mnt = *mount;
1087	if (!--*count)
1088		*mount = NULL;
1089	spin_unlock(&pin_fs_lock);
1090	mntput(mnt);
1091}
1092EXPORT_SYMBOL(simple_release_fs);
1093
1094/**
1095 * simple_read_from_buffer - copy data from the buffer to user space
1096 * @to: the user space buffer to read to
1097 * @count: the maximum number of bytes to read
1098 * @ppos: the current position in the buffer
1099 * @from: the buffer to read from
1100 * @available: the size of the buffer
1101 *
1102 * The simple_read_from_buffer() function reads up to @count bytes from the
1103 * buffer @from at offset @ppos into the user space address starting at @to.
1104 *
1105 * On success, the number of bytes read is returned and the offset @ppos is
1106 * advanced by this number, or negative value is returned on error.
1107 **/
1108ssize_t simple_read_from_buffer(void __user *to, size_t count, loff_t *ppos,
1109				const void *from, size_t available)
1110{
1111	loff_t pos = *ppos;
1112	size_t ret;
1113
1114	if (pos < 0)
1115		return -EINVAL;
1116	if (pos >= available || !count)
1117		return 0;
1118	if (count > available - pos)
1119		count = available - pos;
1120	ret = copy_to_user(to, from + pos, count);
1121	if (ret == count)
1122		return -EFAULT;
1123	count -= ret;
1124	*ppos = pos + count;
1125	return count;
1126}
1127EXPORT_SYMBOL(simple_read_from_buffer);
1128
1129/**
1130 * simple_write_to_buffer - copy data from user space to the buffer
1131 * @to: the buffer to write to
1132 * @available: the size of the buffer
1133 * @ppos: the current position in the buffer
1134 * @from: the user space buffer to read from
1135 * @count: the maximum number of bytes to read
1136 *
1137 * The simple_write_to_buffer() function reads up to @count bytes from the user
1138 * space address starting at @from into the buffer @to at offset @ppos.
1139 *
1140 * On success, the number of bytes written is returned and the offset @ppos is
1141 * advanced by this number, or negative value is returned on error.
1142 **/
1143ssize_t simple_write_to_buffer(void *to, size_t available, loff_t *ppos,
1144		const void __user *from, size_t count)
1145{
1146	loff_t pos = *ppos;
1147	size_t res;
1148
1149	if (pos < 0)
1150		return -EINVAL;
1151	if (pos >= available || !count)
1152		return 0;
1153	if (count > available - pos)
1154		count = available - pos;
1155	res = copy_from_user(to + pos, from, count);
1156	if (res == count)
1157		return -EFAULT;
1158	count -= res;
1159	*ppos = pos + count;
1160	return count;
1161}
1162EXPORT_SYMBOL(simple_write_to_buffer);
1163
1164/**
1165 * memory_read_from_buffer - copy data from the buffer
1166 * @to: the kernel space buffer to read to
1167 * @count: the maximum number of bytes to read
1168 * @ppos: the current position in the buffer
1169 * @from: the buffer to read from
1170 * @available: the size of the buffer
1171 *
1172 * The memory_read_from_buffer() function reads up to @count bytes from the
1173 * buffer @from at offset @ppos into the kernel space address starting at @to.
1174 *
1175 * On success, the number of bytes read is returned and the offset @ppos is
1176 * advanced by this number, or negative value is returned on error.
1177 **/
1178ssize_t memory_read_from_buffer(void *to, size_t count, loff_t *ppos,
1179				const void *from, size_t available)
1180{
1181	loff_t pos = *ppos;
1182
1183	if (pos < 0)
1184		return -EINVAL;
1185	if (pos >= available)
1186		return 0;
1187	if (count > available - pos)
1188		count = available - pos;
1189	memcpy(to, from + pos, count);
1190	*ppos = pos + count;
1191
1192	return count;
1193}
1194EXPORT_SYMBOL(memory_read_from_buffer);
1195
1196/*
1197 * Transaction based IO.
1198 * The file expects a single write which triggers the transaction, and then
1199 * possibly a read which collects the result - which is stored in a
1200 * file-local buffer.
1201 */
1202
1203void simple_transaction_set(struct file *file, size_t n)
1204{
1205	struct simple_transaction_argresp *ar = file->private_data;
1206
1207	BUG_ON(n > SIMPLE_TRANSACTION_LIMIT);
1208
1209	/*
1210	 * The barrier ensures that ar->size will really remain zero until
1211	 * ar->data is ready for reading.
1212	 */
1213	smp_mb();
1214	ar->size = n;
1215}
1216EXPORT_SYMBOL(simple_transaction_set);
1217
1218char *simple_transaction_get(struct file *file, const char __user *buf, size_t size)
1219{
1220	struct simple_transaction_argresp *ar;
1221	static DEFINE_SPINLOCK(simple_transaction_lock);
1222
1223	if (size > SIMPLE_TRANSACTION_LIMIT - 1)
1224		return ERR_PTR(-EFBIG);
1225
1226	ar = (struct simple_transaction_argresp *)get_zeroed_page(GFP_KERNEL);
1227	if (!ar)
1228		return ERR_PTR(-ENOMEM);
1229
1230	spin_lock(&simple_transaction_lock);
1231
1232	/* only one write allowed per open */
1233	if (file->private_data) {
1234		spin_unlock(&simple_transaction_lock);
1235		free_page((unsigned long)ar);
1236		return ERR_PTR(-EBUSY);
1237	}
1238
1239	file->private_data = ar;
1240
1241	spin_unlock(&simple_transaction_lock);
1242
1243	if (copy_from_user(ar->data, buf, size))
1244		return ERR_PTR(-EFAULT);
1245
1246	return ar->data;
1247}
1248EXPORT_SYMBOL(simple_transaction_get);
1249
1250ssize_t simple_transaction_read(struct file *file, char __user *buf, size_t size, loff_t *pos)
1251{
1252	struct simple_transaction_argresp *ar = file->private_data;
1253
1254	if (!ar)
1255		return 0;
1256	return simple_read_from_buffer(buf, size, pos, ar->data, ar->size);
1257}
1258EXPORT_SYMBOL(simple_transaction_read);
1259
1260int simple_transaction_release(struct inode *inode, struct file *file)
1261{
1262	free_page((unsigned long)file->private_data);
1263	return 0;
1264}
1265EXPORT_SYMBOL(simple_transaction_release);
1266
1267/* Simple attribute files */
1268
1269struct simple_attr {
1270	int (*get)(void *, u64 *);
1271	int (*set)(void *, u64);
1272	char get_buf[24];	/* enough to store a u64 and "\n\0" */
1273	char set_buf[24];
1274	void *data;
1275	const char *fmt;	/* format for read operation */
1276	struct mutex mutex;	/* protects access to these buffers */
1277};
1278
1279/* simple_attr_open is called by an actual attribute open file operation
1280 * to set the attribute specific access operations. */
1281int simple_attr_open(struct inode *inode, struct file *file,
1282		     int (*get)(void *, u64 *), int (*set)(void *, u64),
1283		     const char *fmt)
1284{
1285	struct simple_attr *attr;
1286
1287	attr = kzalloc(sizeof(*attr), GFP_KERNEL);
1288	if (!attr)
1289		return -ENOMEM;
1290
1291	attr->get = get;
1292	attr->set = set;
1293	attr->data = inode->i_private;
1294	attr->fmt = fmt;
1295	mutex_init(&attr->mutex);
1296
1297	file->private_data = attr;
1298
1299	return nonseekable_open(inode, file);
1300}
1301EXPORT_SYMBOL_GPL(simple_attr_open);
1302
1303int simple_attr_release(struct inode *inode, struct file *file)
1304{
1305	kfree(file->private_data);
1306	return 0;
1307}
1308EXPORT_SYMBOL_GPL(simple_attr_release);	/* GPL-only?  This?  Really? */
1309
1310/* read from the buffer that is filled with the get function */
1311ssize_t simple_attr_read(struct file *file, char __user *buf,
1312			 size_t len, loff_t *ppos)
1313{
1314	struct simple_attr *attr;
1315	size_t size;
1316	ssize_t ret;
1317
1318	attr = file->private_data;
1319
1320	if (!attr->get)
1321		return -EACCES;
1322
1323	ret = mutex_lock_interruptible(&attr->mutex);
1324	if (ret)
1325		return ret;
1326
1327	if (*ppos && attr->get_buf[0]) {
1328		/* continued read */
1329		size = strlen(attr->get_buf);
1330	} else {
1331		/* first read */
1332		u64 val;
1333		ret = attr->get(attr->data, &val);
1334		if (ret)
1335			goto out;
1336
1337		size = scnprintf(attr->get_buf, sizeof(attr->get_buf),
1338				 attr->fmt, (unsigned long long)val);
1339	}
1340
1341	ret = simple_read_from_buffer(buf, len, ppos, attr->get_buf, size);
1342out:
1343	mutex_unlock(&attr->mutex);
1344	return ret;
1345}
1346EXPORT_SYMBOL_GPL(simple_attr_read);
1347
1348/* interpret the buffer as a number to call the set function with */
1349static ssize_t simple_attr_write_xsigned(struct file *file, const char __user *buf,
1350			  size_t len, loff_t *ppos, bool is_signed)
1351{
1352	struct simple_attr *attr;
1353	unsigned long long val;
1354	size_t size;
1355	ssize_t ret;
1356
1357	attr = file->private_data;
1358	if (!attr->set)
1359		return -EACCES;
1360
1361	ret = mutex_lock_interruptible(&attr->mutex);
1362	if (ret)
1363		return ret;
1364
1365	ret = -EFAULT;
1366	size = min(sizeof(attr->set_buf) - 1, len);
1367	if (copy_from_user(attr->set_buf, buf, size))
1368		goto out;
1369
1370	attr->set_buf[size] = '\0';
1371	if (is_signed)
1372		ret = kstrtoll(attr->set_buf, 0, &val);
1373	else
1374		ret = kstrtoull(attr->set_buf, 0, &val);
1375	if (ret)
1376		goto out;
1377	ret = attr->set(attr->data, val);
1378	if (ret == 0)
1379		ret = len; /* on success, claim we got the whole input */
1380out:
1381	mutex_unlock(&attr->mutex);
1382	return ret;
1383}
1384
1385ssize_t simple_attr_write(struct file *file, const char __user *buf,
1386			  size_t len, loff_t *ppos)
1387{
1388	return simple_attr_write_xsigned(file, buf, len, ppos, false);
1389}
1390EXPORT_SYMBOL_GPL(simple_attr_write);
1391
1392ssize_t simple_attr_write_signed(struct file *file, const char __user *buf,
1393			  size_t len, loff_t *ppos)
1394{
1395	return simple_attr_write_xsigned(file, buf, len, ppos, true);
1396}
1397EXPORT_SYMBOL_GPL(simple_attr_write_signed);
1398
1399/**
1400 * generic_encode_ino32_fh - generic export_operations->encode_fh function
1401 * @inode:   the object to encode
1402 * @fh:      where to store the file handle fragment
1403 * @max_len: maximum length to store there (in 4 byte units)
1404 * @parent:  parent directory inode, if wanted
1405 *
1406 * This generic encode_fh function assumes that the 32 inode number
1407 * is suitable for locating an inode, and that the generation number
1408 * can be used to check that it is still valid.  It places them in the
1409 * filehandle fragment where export_decode_fh expects to find them.
1410 */
1411int generic_encode_ino32_fh(struct inode *inode, __u32 *fh, int *max_len,
1412			    struct inode *parent)
1413{
1414	struct fid *fid = (void *)fh;
1415	int len = *max_len;
1416	int type = FILEID_INO32_GEN;
1417
1418	if (parent && (len < 4)) {
1419		*max_len = 4;
1420		return FILEID_INVALID;
1421	} else if (len < 2) {
1422		*max_len = 2;
1423		return FILEID_INVALID;
1424	}
1425
1426	len = 2;
1427	fid->i32.ino = inode->i_ino;
1428	fid->i32.gen = inode->i_generation;
1429	if (parent) {
1430		fid->i32.parent_ino = parent->i_ino;
1431		fid->i32.parent_gen = parent->i_generation;
1432		len = 4;
1433		type = FILEID_INO32_GEN_PARENT;
1434	}
1435	*max_len = len;
1436	return type;
1437}
1438EXPORT_SYMBOL_GPL(generic_encode_ino32_fh);
1439
1440/**
1441 * generic_fh_to_dentry - generic helper for the fh_to_dentry export operation
1442 * @sb:		filesystem to do the file handle conversion on
1443 * @fid:	file handle to convert
1444 * @fh_len:	length of the file handle in bytes
1445 * @fh_type:	type of file handle
1446 * @get_inode:	filesystem callback to retrieve inode
1447 *
1448 * This function decodes @fid as long as it has one of the well-known
1449 * Linux filehandle types and calls @get_inode on it to retrieve the
1450 * inode for the object specified in the file handle.
1451 */
1452struct dentry *generic_fh_to_dentry(struct super_block *sb, struct fid *fid,
1453		int fh_len, int fh_type, struct inode *(*get_inode)
1454			(struct super_block *sb, u64 ino, u32 gen))
1455{
1456	struct inode *inode = NULL;
1457
1458	if (fh_len < 2)
1459		return NULL;
1460
1461	switch (fh_type) {
1462	case FILEID_INO32_GEN:
1463	case FILEID_INO32_GEN_PARENT:
1464		inode = get_inode(sb, fid->i32.ino, fid->i32.gen);
1465		break;
1466	}
1467
1468	return d_obtain_alias(inode);
1469}
1470EXPORT_SYMBOL_GPL(generic_fh_to_dentry);
1471
1472/**
1473 * generic_fh_to_parent - generic helper for the fh_to_parent export operation
1474 * @sb:		filesystem to do the file handle conversion on
1475 * @fid:	file handle to convert
1476 * @fh_len:	length of the file handle in bytes
1477 * @fh_type:	type of file handle
1478 * @get_inode:	filesystem callback to retrieve inode
1479 *
1480 * This function decodes @fid as long as it has one of the well-known
1481 * Linux filehandle types and calls @get_inode on it to retrieve the
1482 * inode for the _parent_ object specified in the file handle if it
1483 * is specified in the file handle, or NULL otherwise.
1484 */
1485struct dentry *generic_fh_to_parent(struct super_block *sb, struct fid *fid,
1486		int fh_len, int fh_type, struct inode *(*get_inode)
1487			(struct super_block *sb, u64 ino, u32 gen))
1488{
1489	struct inode *inode = NULL;
1490
1491	if (fh_len <= 2)
1492		return NULL;
1493
1494	switch (fh_type) {
1495	case FILEID_INO32_GEN_PARENT:
1496		inode = get_inode(sb, fid->i32.parent_ino,
1497				  (fh_len > 3 ? fid->i32.parent_gen : 0));
1498		break;
1499	}
1500
1501	return d_obtain_alias(inode);
1502}
1503EXPORT_SYMBOL_GPL(generic_fh_to_parent);
1504
1505/**
1506 * __generic_file_fsync - generic fsync implementation for simple filesystems
1507 *
1508 * @file:	file to synchronize
1509 * @start:	start offset in bytes
1510 * @end:	end offset in bytes (inclusive)
1511 * @datasync:	only synchronize essential metadata if true
1512 *
1513 * This is a generic implementation of the fsync method for simple
1514 * filesystems which track all non-inode metadata in the buffers list
1515 * hanging off the address_space structure.
1516 */
1517int __generic_file_fsync(struct file *file, loff_t start, loff_t end,
1518				 int datasync)
1519{
1520	struct inode *inode = file->f_mapping->host;
1521	int err;
1522	int ret;
1523
1524	err = file_write_and_wait_range(file, start, end);
1525	if (err)
1526		return err;
1527
1528	inode_lock(inode);
1529	ret = sync_mapping_buffers(inode->i_mapping);
1530	if (!(inode->i_state & I_DIRTY_ALL))
1531		goto out;
1532	if (datasync && !(inode->i_state & I_DIRTY_DATASYNC))
1533		goto out;
1534
1535	err = sync_inode_metadata(inode, 1);
1536	if (ret == 0)
1537		ret = err;
1538
1539out:
1540	inode_unlock(inode);
1541	/* check and advance again to catch errors after syncing out buffers */
1542	err = file_check_and_advance_wb_err(file);
1543	if (ret == 0)
1544		ret = err;
1545	return ret;
1546}
1547EXPORT_SYMBOL(__generic_file_fsync);
1548
1549/**
1550 * generic_file_fsync - generic fsync implementation for simple filesystems
1551 *			with flush
1552 * @file:	file to synchronize
1553 * @start:	start offset in bytes
1554 * @end:	end offset in bytes (inclusive)
1555 * @datasync:	only synchronize essential metadata if true
1556 *
1557 */
1558
1559int generic_file_fsync(struct file *file, loff_t start, loff_t end,
1560		       int datasync)
1561{
1562	struct inode *inode = file->f_mapping->host;
1563	int err;
1564
1565	err = __generic_file_fsync(file, start, end, datasync);
1566	if (err)
1567		return err;
1568	return blkdev_issue_flush(inode->i_sb->s_bdev);
1569}
1570EXPORT_SYMBOL(generic_file_fsync);
1571
1572/**
1573 * generic_check_addressable - Check addressability of file system
1574 * @blocksize_bits:	log of file system block size
1575 * @num_blocks:		number of blocks in file system
1576 *
1577 * Determine whether a file system with @num_blocks blocks (and a
1578 * block size of 2**@blocksize_bits) is addressable by the sector_t
1579 * and page cache of the system.  Return 0 if so and -EFBIG otherwise.
1580 */
1581int generic_check_addressable(unsigned blocksize_bits, u64 num_blocks)
1582{
1583	u64 last_fs_block = num_blocks - 1;
1584	u64 last_fs_page =
1585		last_fs_block >> (PAGE_SHIFT - blocksize_bits);
1586
1587	if (unlikely(num_blocks == 0))
1588		return 0;
1589
1590	if ((blocksize_bits < 9) || (blocksize_bits > PAGE_SHIFT))
1591		return -EINVAL;
1592
1593	if ((last_fs_block > (sector_t)(~0ULL) >> (blocksize_bits - 9)) ||
1594	    (last_fs_page > (pgoff_t)(~0ULL))) {
1595		return -EFBIG;
1596	}
1597	return 0;
1598}
1599EXPORT_SYMBOL(generic_check_addressable);
1600
1601/*
1602 * No-op implementation of ->fsync for in-memory filesystems.
1603 */
1604int noop_fsync(struct file *file, loff_t start, loff_t end, int datasync)
1605{
1606	return 0;
1607}
1608EXPORT_SYMBOL(noop_fsync);
1609
1610ssize_t noop_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
1611{
1612	/*
1613	 * iomap based filesystems support direct I/O without need for
1614	 * this callback. However, it still needs to be set in
1615	 * inode->a_ops so that open/fcntl know that direct I/O is
1616	 * generally supported.
1617	 */
1618	return -EINVAL;
1619}
1620EXPORT_SYMBOL_GPL(noop_direct_IO);
1621
1622/* Because kfree isn't assignment-compatible with void(void*) ;-/ */
1623void kfree_link(void *p)
1624{
1625	kfree(p);
1626}
1627EXPORT_SYMBOL(kfree_link);
1628
1629struct inode *alloc_anon_inode(struct super_block *s)
1630{
1631	static const struct address_space_operations anon_aops = {
1632		.dirty_folio	= noop_dirty_folio,
1633	};
1634	struct inode *inode = new_inode_pseudo(s);
1635
1636	if (!inode)
1637		return ERR_PTR(-ENOMEM);
1638
1639	inode->i_ino = get_next_ino();
1640	inode->i_mapping->a_ops = &anon_aops;
1641
1642	/*
1643	 * Mark the inode dirty from the very beginning,
1644	 * that way it will never be moved to the dirty
1645	 * list because mark_inode_dirty() will think
1646	 * that it already _is_ on the dirty list.
1647	 */
1648	inode->i_state = I_DIRTY;
1649	inode->i_mode = S_IRUSR | S_IWUSR;
1650	inode->i_uid = current_fsuid();
1651	inode->i_gid = current_fsgid();
1652	inode->i_flags |= S_PRIVATE;
1653	simple_inode_init_ts(inode);
1654	return inode;
1655}
1656EXPORT_SYMBOL(alloc_anon_inode);
1657
1658/**
1659 * simple_nosetlease - generic helper for prohibiting leases
1660 * @filp: file pointer
1661 * @arg: type of lease to obtain
1662 * @flp: new lease supplied for insertion
1663 * @priv: private data for lm_setup operation
1664 *
1665 * Generic helper for filesystems that do not wish to allow leases to be set.
1666 * All arguments are ignored and it just returns -EINVAL.
1667 */
1668int
1669simple_nosetlease(struct file *filp, int arg, struct file_lease **flp,
1670		  void **priv)
1671{
1672	return -EINVAL;
1673}
1674EXPORT_SYMBOL(simple_nosetlease);
1675
1676/**
1677 * simple_get_link - generic helper to get the target of "fast" symlinks
1678 * @dentry: not used here
1679 * @inode: the symlink inode
1680 * @done: not used here
1681 *
1682 * Generic helper for filesystems to use for symlink inodes where a pointer to
1683 * the symlink target is stored in ->i_link.  NOTE: this isn't normally called,
1684 * since as an optimization the path lookup code uses any non-NULL ->i_link
1685 * directly, without calling ->get_link().  But ->get_link() still must be set,
1686 * to mark the inode_operations as being for a symlink.
1687 *
1688 * Return: the symlink target
1689 */
1690const char *simple_get_link(struct dentry *dentry, struct inode *inode,
1691			    struct delayed_call *done)
1692{
1693	return inode->i_link;
1694}
1695EXPORT_SYMBOL(simple_get_link);
1696
1697const struct inode_operations simple_symlink_inode_operations = {
1698	.get_link = simple_get_link,
1699};
1700EXPORT_SYMBOL(simple_symlink_inode_operations);
1701
1702/*
1703 * Operations for a permanently empty directory.
1704 */
1705static struct dentry *empty_dir_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags)
1706{
1707	return ERR_PTR(-ENOENT);
1708}
1709
1710static int empty_dir_setattr(struct mnt_idmap *idmap,
1711			     struct dentry *dentry, struct iattr *attr)
1712{
1713	return -EPERM;
1714}
1715
1716static ssize_t empty_dir_listxattr(struct dentry *dentry, char *list, size_t size)
1717{
1718	return -EOPNOTSUPP;
1719}
1720
1721static const struct inode_operations empty_dir_inode_operations = {
1722	.lookup		= empty_dir_lookup,
1723	.setattr	= empty_dir_setattr,
1724	.listxattr	= empty_dir_listxattr,
1725};
1726
1727static loff_t empty_dir_llseek(struct file *file, loff_t offset, int whence)
1728{
1729	/* An empty directory has two entries . and .. at offsets 0 and 1 */
1730	return generic_file_llseek_size(file, offset, whence, 2, 2);
1731}
1732
1733static int empty_dir_readdir(struct file *file, struct dir_context *ctx)
1734{
1735	dir_emit_dots(file, ctx);
1736	return 0;
1737}
1738
1739static const struct file_operations empty_dir_operations = {
1740	.llseek		= empty_dir_llseek,
1741	.read		= generic_read_dir,
1742	.iterate_shared	= empty_dir_readdir,
1743	.fsync		= noop_fsync,
1744};
1745
1746
1747void make_empty_dir_inode(struct inode *inode)
1748{
1749	set_nlink(inode, 2);
1750	inode->i_mode = S_IFDIR | S_IRUGO | S_IXUGO;
1751	inode->i_uid = GLOBAL_ROOT_UID;
1752	inode->i_gid = GLOBAL_ROOT_GID;
1753	inode->i_rdev = 0;
1754	inode->i_size = 0;
1755	inode->i_blkbits = PAGE_SHIFT;
1756	inode->i_blocks = 0;
1757
1758	inode->i_op = &empty_dir_inode_operations;
1759	inode->i_opflags &= ~IOP_XATTR;
1760	inode->i_fop = &empty_dir_operations;
1761}
1762
1763bool is_empty_dir_inode(struct inode *inode)
1764{
1765	return (inode->i_fop == &empty_dir_operations) &&
1766		(inode->i_op == &empty_dir_inode_operations);
1767}
1768
1769#if IS_ENABLED(CONFIG_UNICODE)
1770/**
1771 * generic_ci_d_compare - generic d_compare implementation for casefolding filesystems
1772 * @dentry:	dentry whose name we are checking against
1773 * @len:	len of name of dentry
1774 * @str:	str pointer to name of dentry
1775 * @name:	Name to compare against
1776 *
1777 * Return: 0 if names match, 1 if mismatch, or -ERRNO
1778 */
1779int generic_ci_d_compare(const struct dentry *dentry, unsigned int len,
1780			 const char *str, const struct qstr *name)
1781{
1782	const struct dentry *parent;
1783	const struct inode *dir;
1784	char strbuf[DNAME_INLINE_LEN];
1785	struct qstr qstr;
1786
1787	/*
1788	 * Attempt a case-sensitive match first. It is cheaper and
1789	 * should cover most lookups, including all the sane
1790	 * applications that expect a case-sensitive filesystem.
1791	 *
1792	 * This comparison is safe under RCU because the caller
1793	 * guarantees the consistency between str and len. See
1794	 * __d_lookup_rcu_op_compare() for details.
1795	 */
1796	if (len == name->len && !memcmp(str, name->name, len))
1797		return 0;
1798
1799	parent = READ_ONCE(dentry->d_parent);
1800	dir = READ_ONCE(parent->d_inode);
1801	if (!dir || !IS_CASEFOLDED(dir))
1802		return 1;
1803
1804	/*
1805	 * If the dentry name is stored in-line, then it may be concurrently
1806	 * modified by a rename.  If this happens, the VFS will eventually retry
1807	 * the lookup, so it doesn't matter what ->d_compare() returns.
1808	 * However, it's unsafe to call utf8_strncasecmp() with an unstable
1809	 * string.  Therefore, we have to copy the name into a temporary buffer.
1810	 */
1811	if (len <= DNAME_INLINE_LEN - 1) {
1812		memcpy(strbuf, str, len);
1813		strbuf[len] = 0;
1814		str = strbuf;
1815		/* prevent compiler from optimizing out the temporary buffer */
1816		barrier();
1817	}
1818	qstr.len = len;
1819	qstr.name = str;
1820
1821	return utf8_strncasecmp(dentry->d_sb->s_encoding, name, &qstr);
1822}
1823EXPORT_SYMBOL(generic_ci_d_compare);
1824
1825/**
1826 * generic_ci_d_hash - generic d_hash implementation for casefolding filesystems
1827 * @dentry:	dentry of the parent directory
1828 * @str:	qstr of name whose hash we should fill in
1829 *
1830 * Return: 0 if hash was successful or unchanged, and -EINVAL on error
1831 */
1832int generic_ci_d_hash(const struct dentry *dentry, struct qstr *str)
1833{
1834	const struct inode *dir = READ_ONCE(dentry->d_inode);
1835	struct super_block *sb = dentry->d_sb;
1836	const struct unicode_map *um = sb->s_encoding;
1837	int ret;
1838
1839	if (!dir || !IS_CASEFOLDED(dir))
1840		return 0;
1841
1842	ret = utf8_casefold_hash(um, dentry, str);
1843	if (ret < 0 && sb_has_strict_encoding(sb))
1844		return -EINVAL;
1845	return 0;
1846}
1847EXPORT_SYMBOL(generic_ci_d_hash);
1848
1849static const struct dentry_operations generic_ci_dentry_ops = {
1850	.d_hash = generic_ci_d_hash,
1851	.d_compare = generic_ci_d_compare,
1852#ifdef CONFIG_FS_ENCRYPTION
1853	.d_revalidate = fscrypt_d_revalidate,
1854#endif
1855};
1856
1857/**
1858 * generic_ci_match() - Match a name (case-insensitively) with a dirent.
1859 * This is a filesystem helper for comparison with directory entries.
1860 * generic_ci_d_compare should be used in VFS' ->d_compare instead.
1861 *
1862 * @parent: Inode of the parent of the dirent under comparison
1863 * @name: name under lookup.
1864 * @folded_name: Optional pre-folded name under lookup
1865 * @de_name: Dirent name.
1866 * @de_name_len: dirent name length.
1867 *
1868 * Test whether a case-insensitive directory entry matches the filename
1869 * being searched.  If @folded_name is provided, it is used instead of
1870 * recalculating the casefold of @name.
1871 *
1872 * Return: > 0 if the directory entry matches, 0 if it doesn't match, or
1873 * < 0 on error.
1874 */
1875int generic_ci_match(const struct inode *parent,
1876		     const struct qstr *name,
1877		     const struct qstr *folded_name,
1878		     const u8 *de_name, u32 de_name_len)
1879{
1880	const struct super_block *sb = parent->i_sb;
1881	const struct unicode_map *um = sb->s_encoding;
1882	struct fscrypt_str decrypted_name = FSTR_INIT(NULL, de_name_len);
1883	struct qstr dirent = QSTR_INIT(de_name, de_name_len);
1884	int res = 0;
1885
1886	if (IS_ENCRYPTED(parent)) {
1887		const struct fscrypt_str encrypted_name =
1888			FSTR_INIT((u8 *) de_name, de_name_len);
1889
1890		if (WARN_ON_ONCE(!fscrypt_has_encryption_key(parent)))
1891			return -EINVAL;
1892
1893		decrypted_name.name = kmalloc(de_name_len, GFP_KERNEL);
1894		if (!decrypted_name.name)
1895			return -ENOMEM;
1896		res = fscrypt_fname_disk_to_usr(parent, 0, 0, &encrypted_name,
1897						&decrypted_name);
1898		if (res < 0) {
1899			kfree(decrypted_name.name);
1900			return res;
1901		}
1902		dirent.name = decrypted_name.name;
1903		dirent.len = decrypted_name.len;
1904	}
1905
1906	/*
1907	 * Attempt a case-sensitive match first. It is cheaper and
1908	 * should cover most lookups, including all the sane
1909	 * applications that expect a case-sensitive filesystem.
1910	 */
1911
1912	if (dirent.len == name->len &&
1913	    !memcmp(name->name, dirent.name, dirent.len))
1914		goto out;
1915
1916	if (folded_name->name)
1917		res = utf8_strncasecmp_folded(um, folded_name, &dirent);
1918	else
1919		res = utf8_strncasecmp(um, name, &dirent);
1920
1921out:
1922	kfree(decrypted_name.name);
1923	if (res < 0 && sb_has_strict_encoding(sb)) {
1924		pr_err_ratelimited("Directory contains filename that is invalid UTF-8");
1925		return 0;
1926	}
1927	return !res;
1928}
1929EXPORT_SYMBOL(generic_ci_match);
1930#endif
1931
1932#ifdef CONFIG_FS_ENCRYPTION
1933static const struct dentry_operations generic_encrypted_dentry_ops = {
1934	.d_revalidate = fscrypt_d_revalidate,
1935};
1936#endif
1937
1938/**
1939 * generic_set_sb_d_ops - helper for choosing the set of
1940 * filesystem-wide dentry operations for the enabled features
1941 * @sb: superblock to be configured
1942 *
1943 * Filesystems supporting casefolding and/or fscrypt can call this
1944 * helper at mount-time to configure sb->s_d_op to best set of dentry
1945 * operations required for the enabled features. The helper must be
1946 * called after these have been configured, but before the root dentry
1947 * is created.
1948 */
1949void generic_set_sb_d_ops(struct super_block *sb)
1950{
1951#if IS_ENABLED(CONFIG_UNICODE)
1952	if (sb->s_encoding) {
1953		sb->s_d_op = &generic_ci_dentry_ops;
1954		return;
1955	}
1956#endif
1957#ifdef CONFIG_FS_ENCRYPTION
1958	if (sb->s_cop) {
1959		sb->s_d_op = &generic_encrypted_dentry_ops;
1960		return;
1961	}
1962#endif
1963}
1964EXPORT_SYMBOL(generic_set_sb_d_ops);
1965
1966/**
1967 * inode_maybe_inc_iversion - increments i_version
1968 * @inode: inode with the i_version that should be updated
1969 * @force: increment the counter even if it's not necessary?
1970 *
1971 * Every time the inode is modified, the i_version field must be seen to have
1972 * changed by any observer.
1973 *
1974 * If "force" is set or the QUERIED flag is set, then ensure that we increment
1975 * the value, and clear the queried flag.
1976 *
1977 * In the common case where neither is set, then we can return "false" without
1978 * updating i_version.
1979 *
1980 * If this function returns false, and no other metadata has changed, then we
1981 * can avoid logging the metadata.
1982 */
1983bool inode_maybe_inc_iversion(struct inode *inode, bool force)
1984{
1985	u64 cur, new;
1986
1987	/*
1988	 * The i_version field is not strictly ordered with any other inode
1989	 * information, but the legacy inode_inc_iversion code used a spinlock
1990	 * to serialize increments.
1991	 *
1992	 * We add a full memory barrier to ensure that any de facto ordering
1993	 * with other state is preserved (either implicitly coming from cmpxchg
1994	 * or explicitly from smp_mb if we don't know upfront if we will execute
1995	 * the former).
1996	 *
1997	 * These barriers pair with inode_query_iversion().
1998	 */
1999	cur = inode_peek_iversion_raw(inode);
2000	if (!force && !(cur & I_VERSION_QUERIED)) {
2001		smp_mb();
2002		cur = inode_peek_iversion_raw(inode);
2003	}
2004
2005	do {
2006		/* If flag is clear then we needn't do anything */
2007		if (!force && !(cur & I_VERSION_QUERIED))
2008			return false;
2009
2010		/* Since lowest bit is flag, add 2 to avoid it */
2011		new = (cur & ~I_VERSION_QUERIED) + I_VERSION_INCREMENT;
2012	} while (!atomic64_try_cmpxchg(&inode->i_version, &cur, new));
2013	return true;
2014}
2015EXPORT_SYMBOL(inode_maybe_inc_iversion);
2016
2017/**
2018 * inode_query_iversion - read i_version for later use
2019 * @inode: inode from which i_version should be read
2020 *
2021 * Read the inode i_version counter. This should be used by callers that wish
2022 * to store the returned i_version for later comparison. This will guarantee
2023 * that a later query of the i_version will result in a different value if
2024 * anything has changed.
2025 *
2026 * In this implementation, we fetch the current value, set the QUERIED flag and
2027 * then try to swap it into place with a cmpxchg, if it wasn't already set. If
2028 * that fails, we try again with the newly fetched value from the cmpxchg.
2029 */
2030u64 inode_query_iversion(struct inode *inode)
2031{
2032	u64 cur, new;
2033	bool fenced = false;
2034
2035	/*
2036	 * Memory barriers (implicit in cmpxchg, explicit in smp_mb) pair with
2037	 * inode_maybe_inc_iversion(), see that routine for more details.
2038	 */
2039	cur = inode_peek_iversion_raw(inode);
2040	do {
2041		/* If flag is already set, then no need to swap */
2042		if (cur & I_VERSION_QUERIED) {
2043			if (!fenced)
2044				smp_mb();
2045			break;
2046		}
2047
2048		fenced = true;
2049		new = cur | I_VERSION_QUERIED;
2050	} while (!atomic64_try_cmpxchg(&inode->i_version, &cur, new));
2051	return cur >> I_VERSION_QUERIED_SHIFT;
2052}
2053EXPORT_SYMBOL(inode_query_iversion);
2054
2055ssize_t direct_write_fallback(struct kiocb *iocb, struct iov_iter *iter,
2056		ssize_t direct_written, ssize_t buffered_written)
2057{
2058	struct address_space *mapping = iocb->ki_filp->f_mapping;
2059	loff_t pos = iocb->ki_pos - buffered_written;
2060	loff_t end = iocb->ki_pos - 1;
2061	int err;
2062
2063	/*
2064	 * If the buffered write fallback returned an error, we want to return
2065	 * the number of bytes which were written by direct I/O, or the error
2066	 * code if that was zero.
2067	 *
2068	 * Note that this differs from normal direct-io semantics, which will
2069	 * return -EFOO even if some bytes were written.
2070	 */
2071	if (unlikely(buffered_written < 0)) {
2072		if (direct_written)
2073			return direct_written;
2074		return buffered_written;
2075	}
2076
2077	/*
2078	 * We need to ensure that the page cache pages are written to disk and
2079	 * invalidated to preserve the expected O_DIRECT semantics.
2080	 */
2081	err = filemap_write_and_wait_range(mapping, pos, end);
2082	if (err < 0) {
2083		/*
2084		 * We don't know how much we wrote, so just return the number of
2085		 * bytes which were direct-written
2086		 */
2087		iocb->ki_pos -= buffered_written;
2088		if (direct_written)
2089			return direct_written;
2090		return err;
2091	}
2092	invalidate_mapping_pages(mapping, pos >> PAGE_SHIFT, end >> PAGE_SHIFT);
2093	return direct_written + buffered_written;
2094}
2095EXPORT_SYMBOL_GPL(direct_write_fallback);
2096
2097/**
2098 * simple_inode_init_ts - initialize the timestamps for a new inode
2099 * @inode: inode to be initialized
2100 *
2101 * When a new inode is created, most filesystems set the timestamps to the
2102 * current time. Add a helper to do this.
2103 */
2104struct timespec64 simple_inode_init_ts(struct inode *inode)
2105{
2106	struct timespec64 ts = inode_set_ctime_current(inode);
2107
2108	inode_set_atime_to_ts(inode, ts);
2109	inode_set_mtime_to_ts(inode, ts);
2110	return ts;
2111}
2112EXPORT_SYMBOL(simple_inode_init_ts);
2113
2114static inline struct dentry *get_stashed_dentry(struct dentry **stashed)
2115{
2116	struct dentry *dentry;
2117
2118	guard(rcu)();
2119	dentry = rcu_dereference(*stashed);
2120	if (!dentry)
2121		return NULL;
2122	if (!lockref_get_not_dead(&dentry->d_lockref))
2123		return NULL;
2124	return dentry;
2125}
2126
2127static struct dentry *prepare_anon_dentry(struct dentry **stashed,
2128					  struct super_block *sb,
2129					  void *data)
2130{
2131	struct dentry *dentry;
2132	struct inode *inode;
2133	const struct stashed_operations *sops = sb->s_fs_info;
2134	int ret;
2135
2136	inode = new_inode_pseudo(sb);
2137	if (!inode) {
2138		sops->put_data(data);
2139		return ERR_PTR(-ENOMEM);
2140	}
2141
2142	inode->i_flags |= S_IMMUTABLE;
2143	inode->i_mode = S_IFREG;
2144	simple_inode_init_ts(inode);
2145
2146	ret = sops->init_inode(inode, data);
2147	if (ret < 0) {
2148		iput(inode);
2149		return ERR_PTR(ret);
2150	}
2151
2152	/* Notice when this is changed. */
2153	WARN_ON_ONCE(!S_ISREG(inode->i_mode));
2154	WARN_ON_ONCE(!IS_IMMUTABLE(inode));
2155
2156	dentry = d_alloc_anon(sb);
2157	if (!dentry) {
2158		iput(inode);
2159		return ERR_PTR(-ENOMEM);
2160	}
2161
2162	/* Store address of location where dentry's supposed to be stashed. */
2163	dentry->d_fsdata = stashed;
2164
2165	/* @data is now owned by the fs */
2166	d_instantiate(dentry, inode);
2167	return dentry;
2168}
2169
2170static struct dentry *stash_dentry(struct dentry **stashed,
2171				   struct dentry *dentry)
2172{
2173	guard(rcu)();
2174	for (;;) {
2175		struct dentry *old;
2176
2177		/* Assume any old dentry was cleared out. */
2178		old = cmpxchg(stashed, NULL, dentry);
2179		if (likely(!old))
2180			return dentry;
2181
2182		/* Check if somebody else installed a reusable dentry. */
2183		if (lockref_get_not_dead(&old->d_lockref))
2184			return old;
2185
2186		/* There's an old dead dentry there, try to take it over. */
2187		if (likely(try_cmpxchg(stashed, &old, dentry)))
2188			return dentry;
2189	}
2190}
2191
2192/**
2193 * path_from_stashed - create path from stashed or new dentry
2194 * @stashed:    where to retrieve or stash dentry
2195 * @mnt:        mnt of the filesystems to use
2196 * @data:       data to store in inode->i_private
2197 * @path:       path to create
2198 *
2199 * The function tries to retrieve a stashed dentry from @stashed. If the dentry
2200 * is still valid then it will be reused. If the dentry isn't able the function
2201 * will allocate a new dentry and inode. It will then check again whether it
2202 * can reuse an existing dentry in case one has been added in the meantime or
2203 * update @stashed with the newly added dentry.
2204 *
2205 * Special-purpose helper for nsfs and pidfs.
2206 *
2207 * Return: On success zero and on failure a negative error is returned.
2208 */
2209int path_from_stashed(struct dentry **stashed, struct vfsmount *mnt, void *data,
2210		      struct path *path)
2211{
2212	struct dentry *dentry;
2213	const struct stashed_operations *sops = mnt->mnt_sb->s_fs_info;
2214
2215	/* See if dentry can be reused. */
2216	path->dentry = get_stashed_dentry(stashed);
2217	if (path->dentry) {
2218		sops->put_data(data);
2219		goto out_path;
2220	}
2221
2222	/* Allocate a new dentry. */
2223	dentry = prepare_anon_dentry(stashed, mnt->mnt_sb, data);
2224	if (IS_ERR(dentry))
2225		return PTR_ERR(dentry);
2226
2227	/* Added a new dentry. @data is now owned by the filesystem. */
2228	path->dentry = stash_dentry(stashed, dentry);
2229	if (path->dentry != dentry)
2230		dput(dentry);
2231
2232out_path:
2233	WARN_ON_ONCE(path->dentry->d_fsdata != stashed);
2234	WARN_ON_ONCE(d_inode(path->dentry)->i_private != data);
2235	path->mnt = mntget(mnt);
2236	return 0;
2237}
2238
2239void stashed_dentry_prune(struct dentry *dentry)
2240{
2241	struct dentry **stashed = dentry->d_fsdata;
2242	struct inode *inode = d_inode(dentry);
2243
2244	if (WARN_ON_ONCE(!stashed))
2245		return;
2246
2247	if (!inode)
2248		return;
2249
2250	/*
2251	 * Only replace our own @dentry as someone else might've
2252	 * already cleared out @dentry and stashed their own
2253	 * dentry in there.
2254	 */
2255	cmpxchg(stashed, dentry, NULL);
2256}