1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
   4 * All Rights Reserved.
   5 */
   6#include "xfs.h"
   7#include "xfs_fs.h"
   8#include "xfs_shared.h"
   9#include "xfs_format.h"
  10#include "xfs_log_format.h"
  11#include "xfs_trans_resv.h"
  12#include "xfs_mount.h"
  13#include "xfs_inode.h"
  14#include "xfs_trans.h"
  15#include "xfs_inode_item.h"
  16#include "xfs_bmap.h"
  17#include "xfs_bmap_util.h"
  18#include "xfs_dir2.h"
  19#include "xfs_dir2_priv.h"
  20#include "xfs_ioctl.h"
  21#include "xfs_trace.h"
  22#include "xfs_log.h"
  23#include "xfs_icache.h"
  24#include "xfs_pnfs.h"
  25#include "xfs_iomap.h"
  26#include "xfs_reflink.h"
  27
  28#include <linux/dax.h>
  29#include <linux/falloc.h>
  30#include <linux/backing-dev.h>
  31#include <linux/mman.h>
  32#include <linux/fadvise.h>
  33#include <linux/mount.h>
  34
  35static const struct vm_operations_struct xfs_file_vm_ops;
  36
  37/*
  38 * Decide if the given file range is aligned to the size of the fundamental
  39 * allocation unit for the file.
  40 */
  41static bool
  42xfs_is_falloc_aligned(
  43	struct xfs_inode	*ip,
  44	loff_t			pos,
  45	long long int		len)
  46{
  47	struct xfs_mount	*mp = ip->i_mount;
  48	uint64_t		mask;
  49
  50	if (XFS_IS_REALTIME_INODE(ip)) {
  51		if (!is_power_of_2(mp->m_sb.sb_rextsize)) {
  52			u64	rextbytes;
  53			u32	mod;
  54
  55			rextbytes = XFS_FSB_TO_B(mp, mp->m_sb.sb_rextsize);
  56			div_u64_rem(pos, rextbytes, &mod);
  57			if (mod)
  58				return false;
  59			div_u64_rem(len, rextbytes, &mod);
  60			return mod == 0;
  61		}
  62		mask = XFS_FSB_TO_B(mp, mp->m_sb.sb_rextsize) - 1;
  63	} else {
  64		mask = mp->m_sb.sb_blocksize - 1;
  65	}
  66
  67	return !((pos | len) & mask);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  68}
  69
  70/*
  71 * Fsync operations on directories are much simpler than on regular files,
  72 * as there is no file data to flush, and thus also no need for explicit
  73 * cache flush operations, and there are no non-transaction metadata updates
  74 * on directories either.
  75 */
  76STATIC int
  77xfs_dir_fsync(
  78	struct file		*file,
  79	loff_t			start,
  80	loff_t			end,
  81	int			datasync)
  82{
  83	struct xfs_inode	*ip = XFS_I(file->f_mapping->host);
  84
  85	trace_xfs_dir_fsync(ip);
  86	return xfs_log_force_inode(ip);
  87}
  88
  89static xfs_csn_t
  90xfs_fsync_seq(
  91	struct xfs_inode	*ip,
  92	bool			datasync)
  93{
  94	if (!xfs_ipincount(ip))
  95		return 0;
  96	if (datasync && !(ip->i_itemp->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP))
  97		return 0;
  98	return ip->i_itemp->ili_commit_seq;
  99}
 100
 101/*
 102 * All metadata updates are logged, which means that we just have to flush the
 103 * log up to the latest LSN that touched the inode.
 104 *
 105 * If we have concurrent fsync/fdatasync() calls, we need them to all block on
 106 * the log force before we clear the ili_fsync_fields field. This ensures that
 107 * we don't get a racing sync operation that does not wait for the metadata to
 108 * hit the journal before returning.  If we race with clearing ili_fsync_fields,
 109 * then all that will happen is the log force will do nothing as the lsn will
 110 * already be on disk.  We can't race with setting ili_fsync_fields because that
 111 * is done under XFS_ILOCK_EXCL, and that can't happen because we hold the lock
 112 * shared until after the ili_fsync_fields is cleared.
 113 */
 114static  int
 115xfs_fsync_flush_log(
 116	struct xfs_inode	*ip,
 117	bool			datasync,
 118	int			*log_flushed)
 119{
 120	int			error = 0;
 121	xfs_csn_t		seq;
 122
 123	xfs_ilock(ip, XFS_ILOCK_SHARED);
 124	seq = xfs_fsync_seq(ip, datasync);
 125	if (seq) {
 126		error = xfs_log_force_seq(ip->i_mount, seq, XFS_LOG_SYNC,
 127					  log_flushed);
 128
 129		spin_lock(&ip->i_itemp->ili_lock);
 130		ip->i_itemp->ili_fsync_fields = 0;
 131		spin_unlock(&ip->i_itemp->ili_lock);
 132	}
 133	xfs_iunlock(ip, XFS_ILOCK_SHARED);
 134	return error;
 135}
 136
 137STATIC int
 138xfs_file_fsync(
 139	struct file		*file,
 140	loff_t			start,
 141	loff_t			end,
 142	int			datasync)
 143{
 144	struct xfs_inode	*ip = XFS_I(file->f_mapping->host);
 
 
 145	struct xfs_mount	*mp = ip->i_mount;
 146	int			error, err2;
 147	int			log_flushed = 0;
 
 148
 149	trace_xfs_file_fsync(ip);
 150
 151	error = file_write_and_wait_range(file, start, end);
 152	if (error)
 153		return error;
 154
 155	if (xfs_is_shutdown(mp))
 156		return -EIO;
 157
 158	xfs_iflags_clear(ip, XFS_ITRUNCATED);
 159
 160	/*
 161	 * If we have an RT and/or log subvolume we need to make sure to flush
 162	 * the write cache the device used for file data first.  This is to
 163	 * ensure newly written file data make it to disk before logging the new
 164	 * inode size in case of an extending write.
 165	 */
 166	if (XFS_IS_REALTIME_INODE(ip))
 167		error = blkdev_issue_flush(mp->m_rtdev_targp->bt_bdev);
 168	else if (mp->m_logdev_targp != mp->m_ddev_targp)
 169		error = blkdev_issue_flush(mp->m_ddev_targp->bt_bdev);
 170
 171	/*
 172	 * Any inode that has dirty modifications in the log is pinned.  The
 173	 * racy check here for a pinned inode will not catch modifications
 174	 * that happen concurrently to the fsync call, but fsync semantics
 175	 * only require to sync previously completed I/O.
 
 
 
 
 
 
 
 176	 */
 
 177	if (xfs_ipincount(ip)) {
 178		err2 = xfs_fsync_flush_log(ip, datasync, &log_flushed);
 179		if (err2 && !error)
 180			error = err2;
 181	}
 182
 
 
 
 
 
 
 
 
 183	/*
 184	 * If we only have a single device, and the log force about was
 185	 * a no-op we might have to flush the data device cache here.
 186	 * This can only happen for fdatasync/O_DSYNC if we were overwriting
 187	 * an already allocated file and thus do not have any metadata to
 188	 * commit.
 189	 */
 190	if (!log_flushed && !XFS_IS_REALTIME_INODE(ip) &&
 191	    mp->m_logdev_targp == mp->m_ddev_targp) {
 192		err2 = blkdev_issue_flush(mp->m_ddev_targp->bt_bdev);
 193		if (err2 && !error)
 194			error = err2;
 195	}
 196
 197	return error;
 198}
 199
 200static int
 201xfs_ilock_iocb(
 202	struct kiocb		*iocb,
 203	unsigned int		lock_mode)
 204{
 205	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
 206
 207	if (iocb->ki_flags & IOCB_NOWAIT) {
 208		if (!xfs_ilock_nowait(ip, lock_mode))
 209			return -EAGAIN;
 210	} else {
 211		xfs_ilock(ip, lock_mode);
 212	}
 213
 214	return 0;
 215}
 216
 217static int
 218xfs_ilock_iocb_for_write(
 219	struct kiocb		*iocb,
 220	unsigned int		*lock_mode)
 221{
 222	ssize_t			ret;
 223	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
 224
 225	ret = xfs_ilock_iocb(iocb, *lock_mode);
 226	if (ret)
 227		return ret;
 228
 229	if (*lock_mode == XFS_IOLOCK_EXCL)
 230		return 0;
 231	if (!xfs_iflags_test(ip, XFS_IREMAPPING))
 232		return 0;
 233
 234	xfs_iunlock(ip, *lock_mode);
 235	*lock_mode = XFS_IOLOCK_EXCL;
 236	return xfs_ilock_iocb(iocb, *lock_mode);
 237}
 238
 239static unsigned int
 240xfs_ilock_for_write_fault(
 241	struct xfs_inode	*ip)
 242{
 243	/* get a shared lock if no remapping in progress */
 244	xfs_ilock(ip, XFS_MMAPLOCK_SHARED);
 245	if (!xfs_iflags_test(ip, XFS_IREMAPPING))
 246		return XFS_MMAPLOCK_SHARED;
 247
 248	/* wait for remapping to complete */
 249	xfs_iunlock(ip, XFS_MMAPLOCK_SHARED);
 250	xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
 251	return XFS_MMAPLOCK_EXCL;
 252}
 253
 254STATIC ssize_t
 255xfs_file_dio_read(
 256	struct kiocb		*iocb,
 257	struct iov_iter		*to)
 258{
 259	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
 
 260	ssize_t			ret;
 261
 262	trace_xfs_file_direct_read(iocb, to);
 263
 264	if (!iov_iter_count(to))
 265		return 0; /* skip atime */
 266
 267	file_accessed(iocb->ki_filp);
 268
 269	ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED);
 270	if (ret)
 271		return ret;
 272	ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, 0, NULL, 0);
 
 
 
 
 273	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
 274
 275	return ret;
 276}
 277
 278static noinline ssize_t
 279xfs_file_dax_read(
 280	struct kiocb		*iocb,
 281	struct iov_iter		*to)
 282{
 283	struct xfs_inode	*ip = XFS_I(iocb->ki_filp->f_mapping->host);
 
 284	ssize_t			ret = 0;
 285
 286	trace_xfs_file_dax_read(iocb, to);
 287
 288	if (!iov_iter_count(to))
 289		return 0; /* skip atime */
 290
 291	ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED);
 292	if (ret)
 293		return ret;
 
 
 
 
 294	ret = dax_iomap_rw(iocb, to, &xfs_read_iomap_ops);
 295	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
 296
 297	file_accessed(iocb->ki_filp);
 298	return ret;
 299}
 300
 301STATIC ssize_t
 302xfs_file_buffered_read(
 303	struct kiocb		*iocb,
 304	struct iov_iter		*to)
 305{
 306	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
 307	ssize_t			ret;
 308
 309	trace_xfs_file_buffered_read(iocb, to);
 310
 311	ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED);
 312	if (ret)
 313		return ret;
 
 
 
 314	ret = generic_file_read_iter(iocb, to);
 315	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
 316
 317	return ret;
 318}
 319
 320STATIC ssize_t
 321xfs_file_read_iter(
 322	struct kiocb		*iocb,
 323	struct iov_iter		*to)
 324{
 325	struct inode		*inode = file_inode(iocb->ki_filp);
 326	struct xfs_mount	*mp = XFS_I(inode)->i_mount;
 327	ssize_t			ret = 0;
 328
 329	XFS_STATS_INC(mp, xs_read_calls);
 330
 331	if (xfs_is_shutdown(mp))
 332		return -EIO;
 333
 334	if (IS_DAX(inode))
 335		ret = xfs_file_dax_read(iocb, to);
 336	else if (iocb->ki_flags & IOCB_DIRECT)
 337		ret = xfs_file_dio_read(iocb, to);
 338	else
 339		ret = xfs_file_buffered_read(iocb, to);
 340
 341	if (ret > 0)
 342		XFS_STATS_ADD(mp, xs_read_bytes, ret);
 343	return ret;
 344}
 345
 346STATIC ssize_t
 347xfs_file_splice_read(
 348	struct file		*in,
 349	loff_t			*ppos,
 350	struct pipe_inode_info	*pipe,
 351	size_t			len,
 352	unsigned int		flags)
 353{
 354	struct inode		*inode = file_inode(in);
 355	struct xfs_inode	*ip = XFS_I(inode);
 356	struct xfs_mount	*mp = ip->i_mount;
 357	ssize_t			ret = 0;
 358
 359	XFS_STATS_INC(mp, xs_read_calls);
 360
 361	if (xfs_is_shutdown(mp))
 362		return -EIO;
 363
 364	trace_xfs_file_splice_read(ip, *ppos, len);
 365
 366	xfs_ilock(ip, XFS_IOLOCK_SHARED);
 367	ret = filemap_splice_read(in, ppos, pipe, len, flags);
 368	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
 369	if (ret > 0)
 370		XFS_STATS_ADD(mp, xs_read_bytes, ret);
 371	return ret;
 372}
 373
 374/*
 375 * Common pre-write limit and setup checks.
 376 *
 377 * Called with the iolocked held either shared and exclusive according to
 378 * @iolock, and returns with it held.  Might upgrade the iolock to exclusive
 379 * if called for a direct write beyond i_size.
 380 */
 381STATIC ssize_t
 382xfs_file_write_checks(
 383	struct kiocb		*iocb,
 384	struct iov_iter		*from,
 385	unsigned int		*iolock)
 386{
 387	struct file		*file = iocb->ki_filp;
 388	struct inode		*inode = file->f_mapping->host;
 389	struct xfs_inode	*ip = XFS_I(inode);
 390	ssize_t			error = 0;
 391	size_t			count = iov_iter_count(from);
 392	bool			drained_dio = false;
 393	loff_t			isize;
 394
 395restart:
 396	error = generic_write_checks(iocb, from);
 397	if (error <= 0)
 398		return error;
 399
 400	if (iocb->ki_flags & IOCB_NOWAIT) {
 401		error = break_layout(inode, false);
 402		if (error == -EWOULDBLOCK)
 403			error = -EAGAIN;
 404	} else {
 405		error = xfs_break_layouts(inode, iolock, BREAK_WRITE);
 406	}
 407
 408	if (error)
 409		return error;
 410
 411	/*
 412	 * For changing security info in file_remove_privs() we need i_rwsem
 413	 * exclusively.
 414	 */
 415	if (*iolock == XFS_IOLOCK_SHARED && !IS_NOSEC(inode)) {
 416		xfs_iunlock(ip, *iolock);
 417		*iolock = XFS_IOLOCK_EXCL;
 418		error = xfs_ilock_iocb(iocb, *iolock);
 419		if (error) {
 420			*iolock = 0;
 421			return error;
 422		}
 423		goto restart;
 424	}
 425
 426	/*
 427	 * If the offset is beyond the size of the file, we need to zero any
 428	 * blocks that fall between the existing EOF and the start of this
 429	 * write.  If zeroing is needed and we are currently holding the iolock
 430	 * shared, we need to update it to exclusive which implies having to
 431	 * redo all checks before.
 432	 *
 433	 * We need to serialise against EOF updates that occur in IO completions
 434	 * here. We want to make sure that nobody is changing the size while we
 435	 * do this check until we have placed an IO barrier (i.e.  hold the
 436	 * XFS_IOLOCK_EXCL) that prevents new IO from being dispatched.  The
 437	 * spinlock effectively forms a memory barrier once we have the
 438	 * XFS_IOLOCK_EXCL so we are guaranteed to see the latest EOF value and
 439	 * hence be able to correctly determine if we need to run zeroing.
 440	 *
 441	 * We can do an unlocked check here safely as IO completion can only
 442	 * extend EOF. Truncate is locked out at this point, so the EOF can
 443	 * not move backwards, only forwards. Hence we only need to take the
 444	 * slow path and spin locks when we are at or beyond the current EOF.
 445	 */
 446	if (iocb->ki_pos <= i_size_read(inode))
 447		goto out;
 448
 449	spin_lock(&ip->i_flags_lock);
 450	isize = i_size_read(inode);
 451	if (iocb->ki_pos > isize) {
 452		spin_unlock(&ip->i_flags_lock);
 453
 454		if (iocb->ki_flags & IOCB_NOWAIT)
 455			return -EAGAIN;
 456
 457		if (!drained_dio) {
 458			if (*iolock == XFS_IOLOCK_SHARED) {
 459				xfs_iunlock(ip, *iolock);
 460				*iolock = XFS_IOLOCK_EXCL;
 461				xfs_ilock(ip, *iolock);
 462				iov_iter_reexpand(from, count);
 463			}
 464			/*
 465			 * We now have an IO submission barrier in place, but
 466			 * AIO can do EOF updates during IO completion and hence
 467			 * we now need to wait for all of them to drain. Non-AIO
 468			 * DIO will have drained before we are given the
 469			 * XFS_IOLOCK_EXCL, and so for most cases this wait is a
 470			 * no-op.
 471			 */
 472			inode_dio_wait(inode);
 473			drained_dio = true;
 474			goto restart;
 475		}
 476
 477		trace_xfs_zero_eof(ip, isize, iocb->ki_pos - isize);
 478		error = xfs_zero_range(ip, isize, iocb->ki_pos - isize, NULL);
 
 479		if (error)
 480			return error;
 481	} else
 482		spin_unlock(&ip->i_flags_lock);
 483
 484out:
 485	return kiocb_modified(iocb);
 
 
 
 
 
 486}
 487
 488static int
 489xfs_dio_write_end_io(
 490	struct kiocb		*iocb,
 491	ssize_t			size,
 492	int			error,
 493	unsigned		flags)
 494{
 495	struct inode		*inode = file_inode(iocb->ki_filp);
 496	struct xfs_inode	*ip = XFS_I(inode);
 497	loff_t			offset = iocb->ki_pos;
 498	unsigned int		nofs_flag;
 499
 500	trace_xfs_end_io_direct_write(ip, offset, size);
 501
 502	if (xfs_is_shutdown(ip->i_mount))
 503		return -EIO;
 504
 505	if (error)
 506		return error;
 507	if (!size)
 508		return 0;
 509
 510	/*
 511	 * Capture amount written on completion as we can't reliably account
 512	 * for it on submission.
 513	 */
 514	XFS_STATS_ADD(ip->i_mount, xs_write_bytes, size);
 515
 516	/*
 517	 * We can allocate memory here while doing writeback on behalf of
 518	 * memory reclaim.  To avoid memory allocation deadlocks set the
 519	 * task-wide nofs context for the following operations.
 520	 */
 521	nofs_flag = memalloc_nofs_save();
 522
 523	if (flags & IOMAP_DIO_COW) {
 524		error = xfs_reflink_end_cow(ip, offset, size);
 525		if (error)
 526			goto out;
 527	}
 528
 529	/*
 530	 * Unwritten conversion updates the in-core isize after extent
 531	 * conversion but before updating the on-disk size. Updating isize any
 532	 * earlier allows a racing dio read to find unwritten extents before
 533	 * they are converted.
 534	 */
 535	if (flags & IOMAP_DIO_UNWRITTEN) {
 536		error = xfs_iomap_write_unwritten(ip, offset, size, true);
 537		goto out;
 538	}
 539
 540	/*
 541	 * We need to update the in-core inode size here so that we don't end up
 542	 * with the on-disk inode size being outside the in-core inode size. We
 543	 * have no other method of updating EOF for AIO, so always do it here
 544	 * if necessary.
 545	 *
 546	 * We need to lock the test/set EOF update as we can be racing with
 547	 * other IO completions here to update the EOF. Failing to serialise
 548	 * here can result in EOF moving backwards and Bad Things Happen when
 549	 * that occurs.
 550	 *
 551	 * As IO completion only ever extends EOF, we can do an unlocked check
 552	 * here to avoid taking the spinlock. If we land within the current EOF,
 553	 * then we do not need to do an extending update at all, and we don't
 554	 * need to take the lock to check this. If we race with an update moving
 555	 * EOF, then we'll either still be beyond EOF and need to take the lock,
 556	 * or we'll be within EOF and we don't need to take it at all.
 557	 */
 558	if (offset + size <= i_size_read(inode))
 559		goto out;
 560
 561	spin_lock(&ip->i_flags_lock);
 562	if (offset + size > i_size_read(inode)) {
 563		i_size_write(inode, offset + size);
 564		spin_unlock(&ip->i_flags_lock);
 565		error = xfs_setfilesize(ip, offset, size);
 566	} else {
 567		spin_unlock(&ip->i_flags_lock);
 568	}
 569
 570out:
 571	memalloc_nofs_restore(nofs_flag);
 572	return error;
 573}
 574
 575static const struct iomap_dio_ops xfs_dio_write_ops = {
 576	.end_io		= xfs_dio_write_end_io,
 577};
 578
 579/*
 580 * Handle block aligned direct I/O writes
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 581 */
 582static noinline ssize_t
 583xfs_file_dio_write_aligned(
 584	struct xfs_inode	*ip,
 585	struct kiocb		*iocb,
 586	struct iov_iter		*from)
 587{
 588	unsigned int		iolock = XFS_IOLOCK_SHARED;
 589	ssize_t			ret;
 
 
 
 
 
 
 
 
 590
 591	ret = xfs_ilock_iocb_for_write(iocb, &iolock);
 592	if (ret)
 593		return ret;
 594	ret = xfs_file_write_checks(iocb, from, &iolock);
 595	if (ret)
 596		goto out_unlock;
 597
 598	/*
 599	 * We don't need to hold the IOLOCK exclusively across the IO, so demote
 600	 * the iolock back to shared if we had to take the exclusive lock in
 601	 * xfs_file_write_checks() for other reasons.
 602	 */
 603	if (iolock == XFS_IOLOCK_EXCL) {
 604		xfs_ilock_demote(ip, XFS_IOLOCK_EXCL);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 605		iolock = XFS_IOLOCK_SHARED;
 606	}
 607	trace_xfs_file_direct_write(iocb, from);
 608	ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
 609			   &xfs_dio_write_ops, 0, NULL, 0);
 610out_unlock:
 611	if (iolock)
 612		xfs_iunlock(ip, iolock);
 613	return ret;
 614}
 615
 616/*
 617 * Handle block unaligned direct I/O writes
 618 *
 619 * In most cases direct I/O writes will be done holding IOLOCK_SHARED, allowing
 620 * them to be done in parallel with reads and other direct I/O writes.  However,
 621 * if the I/O is not aligned to filesystem blocks, the direct I/O layer may need
 622 * to do sub-block zeroing and that requires serialisation against other direct
 623 * I/O to the same block.  In this case we need to serialise the submission of
 624 * the unaligned I/O so that we don't get racing block zeroing in the dio layer.
 625 * In the case where sub-block zeroing is not required, we can do concurrent
 626 * sub-block dios to the same block successfully.
 627 *
 628 * Optimistically submit the I/O using the shared lock first, but use the
 629 * IOMAP_DIO_OVERWRITE_ONLY flag to tell the lower layers to return -EAGAIN
 630 * if block allocation or partial block zeroing would be required.  In that case
 631 * we try again with the exclusive lock.
 632 */
 633static noinline ssize_t
 634xfs_file_dio_write_unaligned(
 635	struct xfs_inode	*ip,
 636	struct kiocb		*iocb,
 637	struct iov_iter		*from)
 638{
 639	size_t			isize = i_size_read(VFS_I(ip));
 640	size_t			count = iov_iter_count(from);
 641	unsigned int		iolock = XFS_IOLOCK_SHARED;
 642	unsigned int		flags = IOMAP_DIO_OVERWRITE_ONLY;
 643	ssize_t			ret;
 644
 645	/*
 646	 * Extending writes need exclusivity because of the sub-block zeroing
 647	 * that the DIO code always does for partial tail blocks beyond EOF, so
 648	 * don't even bother trying the fast path in this case.
 649	 */
 650	if (iocb->ki_pos > isize || iocb->ki_pos + count >= isize) {
 651		if (iocb->ki_flags & IOCB_NOWAIT)
 652			return -EAGAIN;
 653retry_exclusive:
 654		iolock = XFS_IOLOCK_EXCL;
 655		flags = IOMAP_DIO_FORCE_WAIT;
 
 656	}
 657
 658	ret = xfs_ilock_iocb_for_write(iocb, &iolock);
 659	if (ret)
 660		return ret;
 
 661
 662	/*
 663	 * We can't properly handle unaligned direct I/O to reflink files yet,
 664	 * as we can't unshare a partial block.
 665	 */
 666	if (xfs_is_cow_inode(ip)) {
 667		trace_xfs_reflink_bounce_dio_write(iocb, from);
 668		ret = -ENOTBLK;
 669		goto out_unlock;
 
 
 
 
 670	}
 671
 672	ret = xfs_file_write_checks(iocb, from, &iolock);
 673	if (ret)
 674		goto out_unlock;
 675
 676	/*
 677	 * If we are doing exclusive unaligned I/O, this must be the only I/O
 678	 * in-flight.  Otherwise we risk data corruption due to unwritten extent
 679	 * conversions from the AIO end_io handler.  Wait for all other I/O to
 680	 * drain first.
 681	 */
 682	if (flags & IOMAP_DIO_FORCE_WAIT)
 683		inode_dio_wait(VFS_I(ip));
 684
 685	trace_xfs_file_direct_write(iocb, from);
 686	ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
 687			   &xfs_dio_write_ops, flags, NULL, 0);
 
 
 
 688
 689	/*
 690	 * Retry unaligned I/O with exclusive blocking semantics if the DIO
 691	 * layer rejected it for mapping or locking reasons. If we are doing
 692	 * nonblocking user I/O, propagate the error.
 693	 */
 694	if (ret == -EAGAIN && !(iocb->ki_flags & IOCB_NOWAIT)) {
 695		ASSERT(flags & IOMAP_DIO_OVERWRITE_ONLY);
 696		xfs_iunlock(ip, iolock);
 697		goto retry_exclusive;
 698	}
 699
 700out_unlock:
 701	if (iolock)
 702		xfs_iunlock(ip, iolock);
 703	return ret;
 704}
 705
 706static ssize_t
 707xfs_file_dio_write(
 708	struct kiocb		*iocb,
 709	struct iov_iter		*from)
 710{
 711	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
 712	struct xfs_buftarg      *target = xfs_inode_buftarg(ip);
 713	size_t			count = iov_iter_count(from);
 714
 715	/* direct I/O must be aligned to device logical sector size */
 716	if ((iocb->ki_pos | count) & target->bt_logical_sectormask)
 717		return -EINVAL;
 718	if ((iocb->ki_pos | count) & ip->i_mount->m_blockmask)
 719		return xfs_file_dio_write_unaligned(ip, iocb, from);
 720	return xfs_file_dio_write_aligned(ip, iocb, from);
 721}
 722
 723static noinline ssize_t
 724xfs_file_dax_write(
 725	struct kiocb		*iocb,
 726	struct iov_iter		*from)
 727{
 728	struct inode		*inode = iocb->ki_filp->f_mapping->host;
 729	struct xfs_inode	*ip = XFS_I(inode);
 730	unsigned int		iolock = XFS_IOLOCK_EXCL;
 731	ssize_t			ret, error = 0;
 
 732	loff_t			pos;
 733
 734	ret = xfs_ilock_iocb(iocb, iolock);
 735	if (ret)
 736		return ret;
 737	ret = xfs_file_write_checks(iocb, from, &iolock);
 
 
 
 
 738	if (ret)
 739		goto out;
 740
 741	pos = iocb->ki_pos;
 
 742
 743	trace_xfs_file_dax_write(iocb, from);
 744	ret = dax_iomap_rw(iocb, from, &xfs_dax_write_iomap_ops);
 745	if (ret > 0 && iocb->ki_pos > i_size_read(inode)) {
 746		i_size_write(inode, iocb->ki_pos);
 747		error = xfs_setfilesize(ip, pos, ret);
 748	}
 749out:
 750	if (iolock)
 751		xfs_iunlock(ip, iolock);
 752	if (error)
 753		return error;
 754
 755	if (ret > 0) {
 756		XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret);
 757
 758		/* Handle various SYNC-type writes */
 759		ret = generic_write_sync(iocb, ret);
 760	}
 761	return ret;
 762}
 763
 764STATIC ssize_t
 765xfs_file_buffered_write(
 766	struct kiocb		*iocb,
 767	struct iov_iter		*from)
 768{
 769	struct inode		*inode = iocb->ki_filp->f_mapping->host;
 
 
 770	struct xfs_inode	*ip = XFS_I(inode);
 771	ssize_t			ret;
 772	bool			cleared_space = false;
 773	unsigned int		iolock;
 
 
 
 774
 775write_retry:
 776	iolock = XFS_IOLOCK_EXCL;
 777	ret = xfs_ilock_iocb(iocb, iolock);
 778	if (ret)
 779		return ret;
 780
 781	ret = xfs_file_write_checks(iocb, from, &iolock);
 782	if (ret)
 783		goto out;
 784
 785	trace_xfs_file_buffered_write(iocb, from);
 
 
 
 786	ret = iomap_file_buffered_write(iocb, from,
 787			&xfs_buffered_write_iomap_ops);
 
 
 788
 789	/*
 790	 * If we hit a space limit, try to free up some lingering preallocated
 791	 * space before returning an error. In the case of ENOSPC, first try to
 792	 * write back all dirty inodes to free up some of the excess reserved
 793	 * metadata space. This reduces the chances that the eofblocks scan
 794	 * waits on dirty mappings. Since xfs_flush_inodes() is serialized, this
 795	 * also behaves as a filter to prevent too many eofblocks scans from
 796	 * running at the same time.  Use a synchronous scan to increase the
 797	 * effectiveness of the scan.
 798	 */
 799	if (ret == -EDQUOT && !cleared_space) {
 800		xfs_iunlock(ip, iolock);
 801		xfs_blockgc_free_quota(ip, XFS_ICWALK_FLAG_SYNC);
 802		cleared_space = true;
 803		goto write_retry;
 804	} else if (ret == -ENOSPC && !cleared_space) {
 805		struct xfs_icwalk	icw = {0};
 
 
 
 
 806
 807		cleared_space = true;
 808		xfs_flush_inodes(ip->i_mount);
 809
 810		xfs_iunlock(ip, iolock);
 811		icw.icw_flags = XFS_ICWALK_FLAG_SYNC;
 812		xfs_blockgc_free_space(ip->i_mount, &icw);
 
 813		goto write_retry;
 814	}
 815
 
 816out:
 817	if (iolock)
 818		xfs_iunlock(ip, iolock);
 819
 820	if (ret > 0) {
 821		XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret);
 822		/* Handle various SYNC-type writes */
 823		ret = generic_write_sync(iocb, ret);
 824	}
 825	return ret;
 826}
 827
 828STATIC ssize_t
 829xfs_file_write_iter(
 830	struct kiocb		*iocb,
 831	struct iov_iter		*from)
 832{
 833	struct inode		*inode = iocb->ki_filp->f_mapping->host;
 
 
 834	struct xfs_inode	*ip = XFS_I(inode);
 835	ssize_t			ret;
 836	size_t			ocount = iov_iter_count(from);
 837
 838	XFS_STATS_INC(ip->i_mount, xs_write_calls);
 839
 840	if (ocount == 0)
 841		return 0;
 842
 843	if (xfs_is_shutdown(ip->i_mount))
 844		return -EIO;
 845
 846	if (IS_DAX(inode))
 847		return xfs_file_dax_write(iocb, from);
 848
 849	if (iocb->ki_flags & IOCB_DIRECT) {
 850		/*
 851		 * Allow a directio write to fall back to a buffered
 852		 * write *only* in the case that we're doing a reflink
 853		 * CoW.  In all other directio scenarios we do not
 854		 * allow an operation to fall back to buffered mode.
 855		 */
 856		ret = xfs_file_dio_write(iocb, from);
 857		if (ret != -ENOTBLK)
 858			return ret;
 859	}
 860
 861	return xfs_file_buffered_write(iocb, from);
 862}
 863
 864static void
 865xfs_wait_dax_page(
 866	struct inode		*inode)
 867{
 868	struct xfs_inode        *ip = XFS_I(inode);
 869
 870	xfs_iunlock(ip, XFS_MMAPLOCK_EXCL);
 871	schedule();
 872	xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
 873}
 874
 875int
 876xfs_break_dax_layouts(
 877	struct inode		*inode,
 878	bool			*retry)
 879{
 880	struct page		*page;
 881
 882	ASSERT(xfs_isilocked(XFS_I(inode), XFS_MMAPLOCK_EXCL));
 883
 884	page = dax_layout_busy_page(inode->i_mapping);
 885	if (!page)
 886		return 0;
 887
 888	*retry = true;
 889	return ___wait_var_event(&page->_refcount,
 890			atomic_read(&page->_refcount) == 1, TASK_INTERRUPTIBLE,
 891			0, 0, xfs_wait_dax_page(inode));
 892}
 893
 894int
 895xfs_break_layouts(
 896	struct inode		*inode,
 897	uint			*iolock,
 898	enum layout_break_reason reason)
 899{
 900	bool			retry;
 901	int			error;
 902
 903	ASSERT(xfs_isilocked(XFS_I(inode), XFS_IOLOCK_SHARED|XFS_IOLOCK_EXCL));
 904
 905	do {
 906		retry = false;
 907		switch (reason) {
 908		case BREAK_UNMAP:
 909			error = xfs_break_dax_layouts(inode, &retry);
 910			if (error || retry)
 911				break;
 912			fallthrough;
 913		case BREAK_WRITE:
 914			error = xfs_break_leased_layouts(inode, iolock, &retry);
 915			break;
 916		default:
 917			WARN_ON_ONCE(1);
 918			error = -EINVAL;
 919		}
 920	} while (error == 0 && retry);
 921
 922	return error;
 923}
 924
 925/* Does this file, inode, or mount want synchronous writes? */
 926static inline bool xfs_file_sync_writes(struct file *filp)
 927{
 928	struct xfs_inode	*ip = XFS_I(file_inode(filp));
 929
 930	if (xfs_has_wsync(ip->i_mount))
 931		return true;
 932	if (filp->f_flags & (__O_SYNC | O_DSYNC))
 933		return true;
 934	if (IS_SYNC(file_inode(filp)))
 935		return true;
 936
 937	return false;
 938}
 939
 940#define	XFS_FALLOC_FL_SUPPORTED						\
 941		(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |		\
 942		 FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE |	\
 943		 FALLOC_FL_INSERT_RANGE | FALLOC_FL_UNSHARE_RANGE)
 944
 945STATIC long
 946xfs_file_fallocate(
 947	struct file		*file,
 948	int			mode,
 949	loff_t			offset,
 950	loff_t			len)
 951{
 952	struct inode		*inode = file_inode(file);
 953	struct xfs_inode	*ip = XFS_I(inode);
 954	long			error;
 
 955	uint			iolock = XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL;
 956	loff_t			new_size = 0;
 957	bool			do_file_insert = false;
 958
 959	if (!S_ISREG(inode->i_mode))
 960		return -EINVAL;
 961	if (mode & ~XFS_FALLOC_FL_SUPPORTED)
 962		return -EOPNOTSUPP;
 963
 964	xfs_ilock(ip, iolock);
 965	error = xfs_break_layouts(inode, &iolock, BREAK_UNMAP);
 966	if (error)
 967		goto out_unlock;
 968
 969	/*
 970	 * Must wait for all AIO to complete before we continue as AIO can
 971	 * change the file size on completion without holding any locks we
 972	 * currently hold. We must do this first because AIO can update both
 973	 * the on disk and in memory inode sizes, and the operations that follow
 974	 * require the in-memory size to be fully up-to-date.
 975	 */
 976	inode_dio_wait(inode);
 977
 978	/*
 979	 * Now AIO and DIO has drained we flush and (if necessary) invalidate
 980	 * the cached range over the first operation we are about to run.
 981	 *
 982	 * We care about zero and collapse here because they both run a hole
 983	 * punch over the range first. Because that can zero data, and the range
 984	 * of invalidation for the shift operations is much larger, we still do
 985	 * the required flush for collapse in xfs_prepare_shift().
 986	 *
 987	 * Insert has the same range requirements as collapse, and we extend the
 988	 * file first which can zero data. Hence insert has the same
 989	 * flush/invalidate requirements as collapse and so they are both
 990	 * handled at the right time by xfs_prepare_shift().
 991	 */
 992	if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE |
 993		    FALLOC_FL_COLLAPSE_RANGE)) {
 994		error = xfs_flush_unmap_range(ip, offset, len);
 995		if (error)
 996			goto out_unlock;
 997	}
 998
 999	error = file_modified(file);
1000	if (error)
1001		goto out_unlock;
1002
1003	if (mode & FALLOC_FL_PUNCH_HOLE) {
1004		error = xfs_free_file_space(ip, offset, len);
1005		if (error)
1006			goto out_unlock;
1007	} else if (mode & FALLOC_FL_COLLAPSE_RANGE) {
1008		if (!xfs_is_falloc_aligned(ip, offset, len)) {
 
 
1009			error = -EINVAL;
1010			goto out_unlock;
1011		}
1012
1013		/*
1014		 * There is no need to overlap collapse range with EOF,
1015		 * in which case it is effectively a truncate operation
1016		 */
1017		if (offset + len >= i_size_read(inode)) {
1018			error = -EINVAL;
1019			goto out_unlock;
1020		}
1021
1022		new_size = i_size_read(inode) - len;
1023
1024		error = xfs_collapse_file_space(ip, offset, len);
1025		if (error)
1026			goto out_unlock;
1027	} else if (mode & FALLOC_FL_INSERT_RANGE) {
 
1028		loff_t		isize = i_size_read(inode);
1029
1030		if (!xfs_is_falloc_aligned(ip, offset, len)) {
1031			error = -EINVAL;
1032			goto out_unlock;
1033		}
1034
1035		/*
1036		 * New inode size must not exceed ->s_maxbytes, accounting for
1037		 * possible signed overflow.
1038		 */
1039		if (inode->i_sb->s_maxbytes - isize < len) {
1040			error = -EFBIG;
1041			goto out_unlock;
1042		}
1043		new_size = isize + len;
1044
1045		/* Offset should be less than i_size */
1046		if (offset >= isize) {
1047			error = -EINVAL;
1048			goto out_unlock;
1049		}
1050		do_file_insert = true;
1051	} else {
 
 
1052		if (!(mode & FALLOC_FL_KEEP_SIZE) &&
1053		    offset + len > i_size_read(inode)) {
1054			new_size = offset + len;
1055			error = inode_newsize_ok(inode, new_size);
1056			if (error)
1057				goto out_unlock;
1058		}
1059
1060		if (mode & FALLOC_FL_ZERO_RANGE) {
1061			/*
1062			 * Punch a hole and prealloc the range.  We use a hole
1063			 * punch rather than unwritten extent conversion for two
1064			 * reasons:
1065			 *
1066			 *   1.) Hole punch handles partial block zeroing for us.
1067			 *   2.) If prealloc returns ENOSPC, the file range is
1068			 *       still zero-valued by virtue of the hole punch.
1069			 */
1070			unsigned int blksize = i_blocksize(inode);
1071
1072			trace_xfs_zero_file_space(ip);
1073
1074			error = xfs_free_file_space(ip, offset, len);
1075			if (error)
1076				goto out_unlock;
1077
1078			len = round_up(offset + len, blksize) -
1079			      round_down(offset, blksize);
1080			offset = round_down(offset, blksize);
1081		} else if (mode & FALLOC_FL_UNSHARE_RANGE) {
1082			error = xfs_reflink_unshare(ip, offset, len);
1083			if (error)
1084				goto out_unlock;
1085		} else {
1086			/*
1087			 * If always_cow mode we can't use preallocations and
1088			 * thus should not create them.
1089			 */
1090			if (xfs_is_always_cow_inode(ip)) {
1091				error = -EOPNOTSUPP;
1092				goto out_unlock;
1093			}
1094		}
1095
1096		if (!xfs_is_always_cow_inode(ip)) {
1097			error = xfs_alloc_file_space(ip, offset, len);
 
1098			if (error)
1099				goto out_unlock;
1100		}
1101	}
1102
 
 
 
 
 
 
 
1103	/* Change file size if needed */
1104	if (new_size) {
1105		struct iattr iattr;
1106
1107		iattr.ia_valid = ATTR_SIZE;
1108		iattr.ia_size = new_size;
1109		error = xfs_vn_setattr_size(file_mnt_idmap(file),
1110					    file_dentry(file), &iattr);
1111		if (error)
1112			goto out_unlock;
1113	}
1114
1115	/*
1116	 * Perform hole insertion now that the file size has been
1117	 * updated so that if we crash during the operation we don't
1118	 * leave shifted extents past EOF and hence losing access to
1119	 * the data that is contained within them.
1120	 */
1121	if (do_file_insert) {
1122		error = xfs_insert_file_space(ip, offset, len);
1123		if (error)
1124			goto out_unlock;
1125	}
1126
1127	if (xfs_file_sync_writes(file))
1128		error = xfs_log_force_inode(ip);
1129
1130out_unlock:
1131	xfs_iunlock(ip, iolock);
1132	return error;
1133}
1134
1135STATIC int
1136xfs_file_fadvise(
1137	struct file	*file,
1138	loff_t		start,
1139	loff_t		end,
1140	int		advice)
1141{
1142	struct xfs_inode *ip = XFS_I(file_inode(file));
1143	int ret;
1144	int lockflags = 0;
1145
1146	/*
1147	 * Operations creating pages in page cache need protection from hole
1148	 * punching and similar ops
1149	 */
1150	if (advice == POSIX_FADV_WILLNEED) {
1151		lockflags = XFS_IOLOCK_SHARED;
1152		xfs_ilock(ip, lockflags);
1153	}
1154	ret = generic_fadvise(file, start, end, advice);
1155	if (lockflags)
1156		xfs_iunlock(ip, lockflags);
1157	return ret;
1158}
1159
1160STATIC loff_t
1161xfs_file_remap_range(
1162	struct file		*file_in,
1163	loff_t			pos_in,
1164	struct file		*file_out,
1165	loff_t			pos_out,
1166	loff_t			len,
1167	unsigned int		remap_flags)
1168{
1169	struct inode		*inode_in = file_inode(file_in);
1170	struct xfs_inode	*src = XFS_I(inode_in);
1171	struct inode		*inode_out = file_inode(file_out);
1172	struct xfs_inode	*dest = XFS_I(inode_out);
1173	struct xfs_mount	*mp = src->i_mount;
1174	loff_t			remapped = 0;
1175	xfs_extlen_t		cowextsize;
1176	int			ret;
1177
1178	if (remap_flags & ~(REMAP_FILE_DEDUP | REMAP_FILE_ADVISORY))
1179		return -EINVAL;
1180
1181	if (!xfs_has_reflink(mp))
1182		return -EOPNOTSUPP;
1183
1184	if (xfs_is_shutdown(mp))
1185		return -EIO;
1186
1187	/* Prepare and then clone file data. */
1188	ret = xfs_reflink_remap_prep(file_in, pos_in, file_out, pos_out,
1189			&len, remap_flags);
1190	if (ret || len == 0)
1191		return ret;
1192
1193	trace_xfs_reflink_remap_range(src, pos_in, len, dest, pos_out);
1194
1195	ret = xfs_reflink_remap_blocks(src, pos_in, dest, pos_out, len,
1196			&remapped);
1197	if (ret)
1198		goto out_unlock;
1199
1200	/*
1201	 * Carry the cowextsize hint from src to dest if we're sharing the
1202	 * entire source file to the entire destination file, the source file
1203	 * has a cowextsize hint, and the destination file does not.
1204	 */
1205	cowextsize = 0;
1206	if (pos_in == 0 && len == i_size_read(inode_in) &&
1207	    (src->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) &&
1208	    pos_out == 0 && len >= i_size_read(inode_out) &&
1209	    !(dest->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE))
1210		cowextsize = src->i_cowextsize;
1211
1212	ret = xfs_reflink_update_dest(dest, pos_out + len, cowextsize,
1213			remap_flags);
1214	if (ret)
1215		goto out_unlock;
1216
1217	if (xfs_file_sync_writes(file_in) || xfs_file_sync_writes(file_out))
1218		xfs_log_force_inode(dest);
1219out_unlock:
1220	xfs_iunlock2_remapping(src, dest);
1221	if (ret)
1222		trace_xfs_reflink_remap_range_error(dest, ret, _RET_IP_);
1223	return remapped > 0 ? remapped : ret;
1224}
1225
1226STATIC int
1227xfs_file_open(
1228	struct inode	*inode,
1229	struct file	*file)
1230{
1231	if (xfs_is_shutdown(XFS_M(inode->i_sb)))
 
 
1232		return -EIO;
1233	file->f_mode |= FMODE_NOWAIT | FMODE_BUF_RASYNC | FMODE_BUF_WASYNC |
1234			FMODE_DIO_PARALLEL_WRITE | FMODE_CAN_ODIRECT;
1235	return generic_file_open(inode, file);
1236}
1237
1238STATIC int
1239xfs_dir_open(
1240	struct inode	*inode,
1241	struct file	*file)
1242{
1243	struct xfs_inode *ip = XFS_I(inode);
1244	unsigned int	mode;
1245	int		error;
1246
1247	error = xfs_file_open(inode, file);
1248	if (error)
1249		return error;
1250
1251	/*
1252	 * If there are any blocks, read-ahead block 0 as we're almost
1253	 * certain to have the next operation be a read there.
1254	 */
1255	mode = xfs_ilock_data_map_shared(ip);
1256	if (ip->i_df.if_nextents > 0)
1257		error = xfs_dir3_data_readahead(ip, 0, 0);
1258	xfs_iunlock(ip, mode);
1259	return error;
1260}
1261
1262STATIC int
1263xfs_file_release(
1264	struct inode	*inode,
1265	struct file	*filp)
1266{
1267	return xfs_release(XFS_I(inode));
1268}
1269
1270STATIC int
1271xfs_file_readdir(
1272	struct file	*file,
1273	struct dir_context *ctx)
1274{
1275	struct inode	*inode = file_inode(file);
1276	xfs_inode_t	*ip = XFS_I(inode);
1277	size_t		bufsize;
1278
1279	/*
1280	 * The Linux API doesn't pass down the total size of the buffer
1281	 * we read into down to the filesystem.  With the filldir concept
1282	 * it's not needed for correct information, but the XFS dir2 leaf
1283	 * code wants an estimate of the buffer size to calculate it's
1284	 * readahead window and size the buffers used for mapping to
1285	 * physical blocks.
1286	 *
1287	 * Try to give it an estimate that's good enough, maybe at some
1288	 * point we can change the ->readdir prototype to include the
1289	 * buffer size.  For now we use the current glibc buffer size.
1290	 */
1291	bufsize = (size_t)min_t(loff_t, XFS_READDIR_BUFSIZE, ip->i_disk_size);
1292
1293	return xfs_readdir(NULL, ip, ctx, bufsize);
1294}
1295
1296STATIC loff_t
1297xfs_file_llseek(
1298	struct file	*file,
1299	loff_t		offset,
1300	int		whence)
1301{
1302	struct inode		*inode = file->f_mapping->host;
1303
1304	if (xfs_is_shutdown(XFS_I(inode)->i_mount))
1305		return -EIO;
1306
1307	switch (whence) {
1308	default:
1309		return generic_file_llseek(file, offset, whence);
1310	case SEEK_HOLE:
1311		offset = iomap_seek_hole(inode, offset, &xfs_seek_iomap_ops);
1312		break;
1313	case SEEK_DATA:
1314		offset = iomap_seek_data(inode, offset, &xfs_seek_iomap_ops);
1315		break;
1316	}
1317
1318	if (offset < 0)
1319		return offset;
1320	return vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
1321}
1322
1323#ifdef CONFIG_FS_DAX
1324static inline vm_fault_t
1325xfs_dax_fault(
1326	struct vm_fault		*vmf,
1327	unsigned int		order,
1328	bool			write_fault,
1329	pfn_t			*pfn)
1330{
1331	return dax_iomap_fault(vmf, order, pfn, NULL,
1332			(write_fault && !vmf->cow_page) ?
1333				&xfs_dax_write_iomap_ops :
1334				&xfs_read_iomap_ops);
1335}
1336#else
1337static inline vm_fault_t
1338xfs_dax_fault(
1339	struct vm_fault		*vmf,
1340	unsigned int		order,
1341	bool			write_fault,
1342	pfn_t			*pfn)
1343{
1344	ASSERT(0);
1345	return VM_FAULT_SIGBUS;
1346}
1347#endif
1348
1349/*
1350 * Locking for serialisation of IO during page faults. This results in a lock
1351 * ordering of:
1352 *
1353 * mmap_lock (MM)
1354 *   sb_start_pagefault(vfs, freeze)
1355 *     invalidate_lock (vfs/XFS_MMAPLOCK - truncate serialisation)
1356 *       page_lock (MM)
1357 *         i_lock (XFS - extent map serialisation)
1358 */
1359static vm_fault_t
1360__xfs_filemap_fault(
1361	struct vm_fault		*vmf,
1362	unsigned int		order,
1363	bool			write_fault)
1364{
1365	struct inode		*inode = file_inode(vmf->vma->vm_file);
1366	struct xfs_inode	*ip = XFS_I(inode);
1367	vm_fault_t		ret;
1368	unsigned int		lock_mode = 0;
1369
1370	trace_xfs_filemap_fault(ip, order, write_fault);
1371
1372	if (write_fault) {
1373		sb_start_pagefault(inode->i_sb);
1374		file_update_time(vmf->vma->vm_file);
1375	}
1376
1377	if (IS_DAX(inode) || write_fault)
1378		lock_mode = xfs_ilock_for_write_fault(XFS_I(inode));
1379
1380	if (IS_DAX(inode)) {
1381		pfn_t pfn;
1382
1383		ret = xfs_dax_fault(vmf, order, write_fault, &pfn);
 
 
 
1384		if (ret & VM_FAULT_NEEDDSYNC)
1385			ret = dax_finish_sync_fault(vmf, order, pfn);
1386	} else if (write_fault) {
1387		ret = iomap_page_mkwrite(vmf, &xfs_page_mkwrite_iomap_ops);
1388	} else {
1389		ret = filemap_fault(vmf);
 
 
 
 
1390	}
1391
1392	if (lock_mode)
1393		xfs_iunlock(XFS_I(inode), lock_mode);
1394
1395	if (write_fault)
1396		sb_end_pagefault(inode->i_sb);
1397	return ret;
1398}
1399
1400static inline bool
1401xfs_is_write_fault(
1402	struct vm_fault		*vmf)
1403{
1404	return (vmf->flags & FAULT_FLAG_WRITE) &&
1405	       (vmf->vma->vm_flags & VM_SHARED);
1406}
1407
1408static vm_fault_t
1409xfs_filemap_fault(
1410	struct vm_fault		*vmf)
1411{
1412	/* DAX can shortcut the normal fault path on write faults! */
1413	return __xfs_filemap_fault(vmf, 0,
1414			IS_DAX(file_inode(vmf->vma->vm_file)) &&
1415			xfs_is_write_fault(vmf));
1416}
1417
1418static vm_fault_t
1419xfs_filemap_huge_fault(
1420	struct vm_fault		*vmf,
1421	unsigned int		order)
1422{
1423	if (!IS_DAX(file_inode(vmf->vma->vm_file)))
1424		return VM_FAULT_FALLBACK;
1425
1426	/* DAX can shortcut the normal fault path on write faults! */
1427	return __xfs_filemap_fault(vmf, order,
1428			xfs_is_write_fault(vmf));
1429}
1430
1431static vm_fault_t
1432xfs_filemap_page_mkwrite(
1433	struct vm_fault		*vmf)
1434{
1435	return __xfs_filemap_fault(vmf, 0, true);
1436}
1437
1438/*
1439 * pfn_mkwrite was originally intended to ensure we capture time stamp updates
1440 * on write faults. In reality, it needs to serialise against truncate and
1441 * prepare memory for writing so handle is as standard write fault.
1442 */
1443static vm_fault_t
1444xfs_filemap_pfn_mkwrite(
1445	struct vm_fault		*vmf)
1446{
1447
1448	return __xfs_filemap_fault(vmf, 0, true);
 
 
 
 
 
 
 
 
 
 
 
 
 
1449}
1450
1451static const struct vm_operations_struct xfs_file_vm_ops = {
1452	.fault		= xfs_filemap_fault,
1453	.huge_fault	= xfs_filemap_huge_fault,
1454	.map_pages	= filemap_map_pages,
1455	.page_mkwrite	= xfs_filemap_page_mkwrite,
1456	.pfn_mkwrite	= xfs_filemap_pfn_mkwrite,
1457};
1458
1459STATIC int
1460xfs_file_mmap(
1461	struct file		*file,
1462	struct vm_area_struct	*vma)
1463{
1464	struct inode		*inode = file_inode(file);
1465	struct xfs_buftarg	*target = xfs_inode_buftarg(XFS_I(inode));
1466
1467	/*
1468	 * We don't support synchronous mappings for non-DAX files and
1469	 * for DAX files if underneath dax_device is not synchronous.
1470	 */
1471	if (!daxdev_mapping_supported(vma, target->bt_daxdev))
1472		return -EOPNOTSUPP;
1473
1474	file_accessed(file);
1475	vma->vm_ops = &xfs_file_vm_ops;
1476	if (IS_DAX(inode))
1477		vm_flags_set(vma, VM_HUGEPAGE);
1478	return 0;
1479}
1480
1481const struct file_operations xfs_file_operations = {
1482	.llseek		= xfs_file_llseek,
1483	.read_iter	= xfs_file_read_iter,
1484	.write_iter	= xfs_file_write_iter,
1485	.splice_read	= xfs_file_splice_read,
1486	.splice_write	= iter_file_splice_write,
1487	.iopoll		= iocb_bio_iopoll,
1488	.unlocked_ioctl	= xfs_file_ioctl,
1489#ifdef CONFIG_COMPAT
1490	.compat_ioctl	= xfs_file_compat_ioctl,
1491#endif
1492	.mmap		= xfs_file_mmap,
1493	.mmap_supported_flags = MAP_SYNC,
1494	.open		= xfs_file_open,
1495	.release	= xfs_file_release,
1496	.fsync		= xfs_file_fsync,
1497	.get_unmapped_area = thp_get_unmapped_area,
1498	.fallocate	= xfs_file_fallocate,
1499	.fadvise	= xfs_file_fadvise,
1500	.remap_file_range = xfs_file_remap_range,
1501};
1502
1503const struct file_operations xfs_dir_file_operations = {
1504	.open		= xfs_dir_open,
1505	.read		= generic_read_dir,
1506	.iterate_shared	= xfs_file_readdir,
1507	.llseek		= generic_file_llseek,
1508	.unlocked_ioctl	= xfs_file_ioctl,
1509#ifdef CONFIG_COMPAT
1510	.compat_ioctl	= xfs_file_compat_ioctl,
1511#endif
1512	.fsync		= xfs_dir_fsync,
1513};