1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (c) 2000-2006 Silicon Graphics, Inc.
   4 * Copyright (c) 2016-2018 Christoph Hellwig.
   5 * All Rights Reserved.
   6 */
   7#include "xfs.h"
   8#include "xfs_fs.h"
   9#include "xfs_shared.h"
  10#include "xfs_format.h"
  11#include "xfs_log_format.h"
  12#include "xfs_trans_resv.h"
  13#include "xfs_mount.h"
  14#include "xfs_inode.h"
  15#include "xfs_btree.h"
  16#include "xfs_bmap_btree.h"
  17#include "xfs_bmap.h"
  18#include "xfs_bmap_util.h"
  19#include "xfs_errortag.h"
  20#include "xfs_error.h"
  21#include "xfs_trans.h"
  22#include "xfs_trans_space.h"
  23#include "xfs_inode_item.h"
  24#include "xfs_iomap.h"
  25#include "xfs_trace.h"
  26#include "xfs_quota.h"
  27#include "xfs_dquot_item.h"
  28#include "xfs_dquot.h"
  29#include "xfs_reflink.h"
  30#include "xfs_health.h"
  31
  32#define XFS_ALLOC_ALIGN(mp, off) \
  33	(((off) >> mp->m_allocsize_log) << mp->m_allocsize_log)
  34
  35static int
  36xfs_alert_fsblock_zero(
  37	xfs_inode_t	*ip,
  38	xfs_bmbt_irec_t	*imap)
  39{
  40	xfs_alert_tag(ip->i_mount, XFS_PTAG_FSBLOCK_ZERO,
  41			"Access to block zero in inode %llu "
  42			"start_block: %llx start_off: %llx "
  43			"blkcnt: %llx extent-state: %x",
  44		(unsigned long long)ip->i_ino,
  45		(unsigned long long)imap->br_startblock,
  46		(unsigned long long)imap->br_startoff,
  47		(unsigned long long)imap->br_blockcount,
  48		imap->br_state);
  49	xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
  50	return -EFSCORRUPTED;
  51}
  52
  53u64
  54xfs_iomap_inode_sequence(
  55	struct xfs_inode	*ip,
  56	u16			iomap_flags)
  57{
  58	u64			cookie = 0;
  59
  60	if (iomap_flags & IOMAP_F_XATTR)
  61		return READ_ONCE(ip->i_af.if_seq);
  62	if ((iomap_flags & IOMAP_F_SHARED) && ip->i_cowfp)
  63		cookie = (u64)READ_ONCE(ip->i_cowfp->if_seq) << 32;
  64	return cookie | READ_ONCE(ip->i_df.if_seq);
  65}
  66
  67/*
  68 * Check that the iomap passed to us is still valid for the given offset and
  69 * length.
  70 */
  71static bool
  72xfs_iomap_valid(
  73	struct inode		*inode,
  74	const struct iomap	*iomap)
  75{
  76	struct xfs_inode	*ip = XFS_I(inode);
  77
  78	if (iomap->validity_cookie !=
  79			xfs_iomap_inode_sequence(ip, iomap->flags)) {
  80		trace_xfs_iomap_invalid(ip, iomap);
  81		return false;
  82	}
  83
  84	XFS_ERRORTAG_DELAY(ip->i_mount, XFS_ERRTAG_WRITE_DELAY_MS);
  85	return true;
  86}
  87
  88static const struct iomap_folio_ops xfs_iomap_folio_ops = {
  89	.iomap_valid		= xfs_iomap_valid,
  90};
  91
  92int
  93xfs_bmbt_to_iomap(
  94	struct xfs_inode	*ip,
  95	struct iomap		*iomap,
  96	struct xfs_bmbt_irec	*imap,
  97	unsigned int		mapping_flags,
  98	u16			iomap_flags,
  99	u64			sequence_cookie)
 100{
 101	struct xfs_mount	*mp = ip->i_mount;
 102	struct xfs_buftarg	*target = xfs_inode_buftarg(ip);
 103
 104	if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock))) {
 105		xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
 106		return xfs_alert_fsblock_zero(ip, imap);
 107	}
 108
 109	if (imap->br_startblock == HOLESTARTBLOCK) {
 110		iomap->addr = IOMAP_NULL_ADDR;
 111		iomap->type = IOMAP_HOLE;
 112	} else if (imap->br_startblock == DELAYSTARTBLOCK ||
 113		   isnullstartblock(imap->br_startblock)) {
 114		iomap->addr = IOMAP_NULL_ADDR;
 115		iomap->type = IOMAP_DELALLOC;
 116	} else {
 117		iomap->addr = BBTOB(xfs_fsb_to_db(ip, imap->br_startblock));
 118		if (mapping_flags & IOMAP_DAX)
 119			iomap->addr += target->bt_dax_part_off;
 120
 121		if (imap->br_state == XFS_EXT_UNWRITTEN)
 122			iomap->type = IOMAP_UNWRITTEN;
 123		else
 124			iomap->type = IOMAP_MAPPED;
 125
 126	}
 127	iomap->offset = XFS_FSB_TO_B(mp, imap->br_startoff);
 128	iomap->length = XFS_FSB_TO_B(mp, imap->br_blockcount);
 129	if (mapping_flags & IOMAP_DAX)
 130		iomap->dax_dev = target->bt_daxdev;
 131	else
 132		iomap->bdev = target->bt_bdev;
 133	iomap->flags = iomap_flags;
 134
 135	if (xfs_ipincount(ip) &&
 136	    (ip->i_itemp->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP))
 137		iomap->flags |= IOMAP_F_DIRTY;
 138
 139	iomap->validity_cookie = sequence_cookie;
 140	iomap->folio_ops = &xfs_iomap_folio_ops;
 141	return 0;
 142}
 143
 144static void
 145xfs_hole_to_iomap(
 146	struct xfs_inode	*ip,
 147	struct iomap		*iomap,
 148	xfs_fileoff_t		offset_fsb,
 149	xfs_fileoff_t		end_fsb)
 150{
 151	struct xfs_buftarg	*target = xfs_inode_buftarg(ip);
 152
 153	iomap->addr = IOMAP_NULL_ADDR;
 154	iomap->type = IOMAP_HOLE;
 155	iomap->offset = XFS_FSB_TO_B(ip->i_mount, offset_fsb);
 156	iomap->length = XFS_FSB_TO_B(ip->i_mount, end_fsb - offset_fsb);
 157	iomap->bdev = target->bt_bdev;
 158	iomap->dax_dev = target->bt_daxdev;
 159}
 160
 161static inline xfs_fileoff_t
 162xfs_iomap_end_fsb(
 163	struct xfs_mount	*mp,
 164	loff_t			offset,
 165	loff_t			count)
 166{
 167	ASSERT(offset <= mp->m_super->s_maxbytes);
 168	return min(XFS_B_TO_FSB(mp, offset + count),
 169		   XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes));
 170}
 171
 172static xfs_extlen_t
 173xfs_eof_alignment(
 174	struct xfs_inode	*ip)
 175{
 176	struct xfs_mount	*mp = ip->i_mount;
 177	xfs_extlen_t		align = 0;
 178
 179	if (!XFS_IS_REALTIME_INODE(ip)) {
 180		/*
 181		 * Round up the allocation request to a stripe unit
 182		 * (m_dalign) boundary if the file size is >= stripe unit
 183		 * size, and we are allocating past the allocation eof.
 184		 *
 185		 * If mounted with the "-o swalloc" option the alignment is
 186		 * increased from the strip unit size to the stripe width.
 187		 */
 188		if (mp->m_swidth && xfs_has_swalloc(mp))
 189			align = mp->m_swidth;
 190		else if (mp->m_dalign)
 191			align = mp->m_dalign;
 192
 193		if (align && XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, align))
 194			align = 0;
 195	}
 196
 197	return align;
 198}
 199
 200/*
 201 * Check if last_fsb is outside the last extent, and if so grow it to the next
 202 * stripe unit boundary.
 203 */
 204xfs_fileoff_t
 205xfs_iomap_eof_align_last_fsb(
 206	struct xfs_inode	*ip,
 207	xfs_fileoff_t		end_fsb)
 208{
 209	struct xfs_ifork	*ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK);
 210	xfs_extlen_t		extsz = xfs_get_extsz_hint(ip);
 211	xfs_extlen_t		align = xfs_eof_alignment(ip);
 212	struct xfs_bmbt_irec	irec;
 213	struct xfs_iext_cursor	icur;
 214
 215	ASSERT(!xfs_need_iread_extents(ifp));
 216
 217	/*
 218	 * Always round up the allocation request to the extent hint boundary.
 219	 */
 220	if (extsz) {
 221		if (align)
 222			align = roundup_64(align, extsz);
 223		else
 224			align = extsz;
 225	}
 226
 227	if (align) {
 228		xfs_fileoff_t	aligned_end_fsb = roundup_64(end_fsb, align);
 229
 230		xfs_iext_last(ifp, &icur);
 231		if (!xfs_iext_get_extent(ifp, &icur, &irec) ||
 232		    aligned_end_fsb >= irec.br_startoff + irec.br_blockcount)
 233			return aligned_end_fsb;
 234	}
 235
 236	return end_fsb;
 237}
 238
 239int
 240xfs_iomap_write_direct(
 241	struct xfs_inode	*ip,
 242	xfs_fileoff_t		offset_fsb,
 243	xfs_fileoff_t		count_fsb,
 244	unsigned int		flags,
 245	struct xfs_bmbt_irec	*imap,
 246	u64			*seq)
 247{
 248	struct xfs_mount	*mp = ip->i_mount;
 249	struct xfs_trans	*tp;
 250	xfs_filblks_t		resaligned;
 251	int			nimaps;
 252	unsigned int		dblocks, rblocks;
 253	bool			force = false;
 254	int			error;
 255	int			bmapi_flags = XFS_BMAPI_PREALLOC;
 256	int			nr_exts = XFS_IEXT_ADD_NOSPLIT_CNT;
 257
 258	ASSERT(count_fsb > 0);
 259
 260	resaligned = xfs_aligned_fsb_count(offset_fsb, count_fsb,
 261					   xfs_get_extsz_hint(ip));
 262	if (unlikely(XFS_IS_REALTIME_INODE(ip))) {
 263		dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0);
 264		rblocks = resaligned;
 265	} else {
 266		dblocks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned);
 267		rblocks = 0;
 268	}
 269
 270	error = xfs_qm_dqattach(ip);
 271	if (error)
 272		return error;
 273
 274	/*
 275	 * For DAX, we do not allocate unwritten extents, but instead we zero
 276	 * the block before we commit the transaction.  Ideally we'd like to do
 277	 * this outside the transaction context, but if we commit and then crash
 278	 * we may not have zeroed the blocks and this will be exposed on
 279	 * recovery of the allocation. Hence we must zero before commit.
 280	 *
 281	 * Further, if we are mapping unwritten extents here, we need to zero
 282	 * and convert them to written so that we don't need an unwritten extent
 283	 * callback for DAX. This also means that we need to be able to dip into
 284	 * the reserve block pool for bmbt block allocation if there is no space
 285	 * left but we need to do unwritten extent conversion.
 286	 */
 287	if (flags & IOMAP_DAX) {
 288		bmapi_flags = XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO;
 289		if (imap->br_state == XFS_EXT_UNWRITTEN) {
 290			force = true;
 291			nr_exts = XFS_IEXT_WRITE_UNWRITTEN_CNT;
 292			dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1;
 293		}
 294	}
 295
 296	error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, dblocks,
 297			rblocks, force, &tp);
 298	if (error)
 299		return error;
 300
 301	error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, nr_exts);
 302	if (error == -EFBIG)
 303		error = xfs_iext_count_upgrade(tp, ip, nr_exts);
 304	if (error)
 305		goto out_trans_cancel;
 306
 307	/*
 308	 * From this point onwards we overwrite the imap pointer that the
 309	 * caller gave to us.
 310	 */
 311	nimaps = 1;
 312	error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb, bmapi_flags, 0,
 313				imap, &nimaps);
 314	if (error)
 315		goto out_trans_cancel;
 316
 317	/*
 318	 * Complete the transaction
 319	 */
 320	error = xfs_trans_commit(tp);
 321	if (error)
 322		goto out_unlock;
 323
 324	/*
 325	 * Copy any maps to caller's array and return any error.
 326	 */
 327	if (nimaps == 0) {
 328		error = -ENOSPC;
 329		goto out_unlock;
 330	}
 331
 332	if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock))) {
 333		xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
 334		error = xfs_alert_fsblock_zero(ip, imap);
 335	}
 336
 337out_unlock:
 338	*seq = xfs_iomap_inode_sequence(ip, 0);
 339	xfs_iunlock(ip, XFS_ILOCK_EXCL);
 340	return error;
 341
 342out_trans_cancel:
 343	xfs_trans_cancel(tp);
 344	goto out_unlock;
 345}
 346
 347STATIC bool
 348xfs_quota_need_throttle(
 349	struct xfs_inode	*ip,
 350	xfs_dqtype_t		type,
 351	xfs_fsblock_t		alloc_blocks)
 352{
 353	struct xfs_dquot	*dq = xfs_inode_dquot(ip, type);
 354
 355	if (!dq || !xfs_this_quota_on(ip->i_mount, type))
 356		return false;
 357
 358	/* no hi watermark, no throttle */
 359	if (!dq->q_prealloc_hi_wmark)
 360		return false;
 361
 362	/* under the lo watermark, no throttle */
 363	if (dq->q_blk.reserved + alloc_blocks < dq->q_prealloc_lo_wmark)
 364		return false;
 365
 366	return true;
 367}
 368
 369STATIC void
 370xfs_quota_calc_throttle(
 371	struct xfs_inode	*ip,
 372	xfs_dqtype_t		type,
 373	xfs_fsblock_t		*qblocks,
 374	int			*qshift,
 375	int64_t			*qfreesp)
 376{
 377	struct xfs_dquot	*dq = xfs_inode_dquot(ip, type);
 378	int64_t			freesp;
 379	int			shift = 0;
 380
 381	/* no dq, or over hi wmark, squash the prealloc completely */
 382	if (!dq || dq->q_blk.reserved >= dq->q_prealloc_hi_wmark) {
 383		*qblocks = 0;
 384		*qfreesp = 0;
 385		return;
 386	}
 387
 388	freesp = dq->q_prealloc_hi_wmark - dq->q_blk.reserved;
 389	if (freesp < dq->q_low_space[XFS_QLOWSP_5_PCNT]) {
 390		shift = 2;
 391		if (freesp < dq->q_low_space[XFS_QLOWSP_3_PCNT])
 392			shift += 2;
 393		if (freesp < dq->q_low_space[XFS_QLOWSP_1_PCNT])
 394			shift += 2;
 395	}
 396
 397	if (freesp < *qfreesp)
 398		*qfreesp = freesp;
 399
 400	/* only overwrite the throttle values if we are more aggressive */
 401	if ((freesp >> shift) < (*qblocks >> *qshift)) {
 402		*qblocks = freesp;
 403		*qshift = shift;
 404	}
 405}
 406
 407/*
 408 * If we don't have a user specified preallocation size, dynamically increase
 409 * the preallocation size as the size of the file grows.  Cap the maximum size
 410 * at a single extent or less if the filesystem is near full. The closer the
 411 * filesystem is to being full, the smaller the maximum preallocation.
 412 */
 413STATIC xfs_fsblock_t
 414xfs_iomap_prealloc_size(
 415	struct xfs_inode	*ip,
 416	int			whichfork,
 417	loff_t			offset,
 418	loff_t			count,
 419	struct xfs_iext_cursor	*icur)
 420{
 421	struct xfs_iext_cursor	ncur = *icur;
 422	struct xfs_bmbt_irec	prev, got;
 423	struct xfs_mount	*mp = ip->i_mount;
 424	struct xfs_ifork	*ifp = xfs_ifork_ptr(ip, whichfork);
 425	xfs_fileoff_t		offset_fsb = XFS_B_TO_FSBT(mp, offset);
 426	int64_t			freesp;
 427	xfs_fsblock_t		qblocks;
 428	xfs_fsblock_t		alloc_blocks = 0;
 429	xfs_extlen_t		plen;
 430	int			shift = 0;
 431	int			qshift = 0;
 432
 433	/*
 434	 * As an exception we don't do any preallocation at all if the file is
 435	 * smaller than the minimum preallocation and we are using the default
 436	 * dynamic preallocation scheme, as it is likely this is the only write
 437	 * to the file that is going to be done.
 438	 */
 439	if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_allocsize_blocks))
 440		return 0;
 441
 442	/*
 443	 * Use the minimum preallocation size for small files or if we are
 444	 * writing right after a hole.
 445	 */
 446	if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_dalign) ||
 447	    !xfs_iext_prev_extent(ifp, &ncur, &prev) ||
 448	    prev.br_startoff + prev.br_blockcount < offset_fsb)
 449		return mp->m_allocsize_blocks;
 450
 451	/*
 452	 * Take the size of the preceding data extents as the basis for the
 453	 * preallocation size. Note that we don't care if the previous extents
 454	 * are written or not.
 455	 */
 456	plen = prev.br_blockcount;
 457	while (xfs_iext_prev_extent(ifp, &ncur, &got)) {
 458		if (plen > XFS_MAX_BMBT_EXTLEN / 2 ||
 459		    isnullstartblock(got.br_startblock) ||
 460		    got.br_startoff + got.br_blockcount != prev.br_startoff ||
 461		    got.br_startblock + got.br_blockcount != prev.br_startblock)
 462			break;
 463		plen += got.br_blockcount;
 464		prev = got;
 465	}
 466
 467	/*
 468	 * If the size of the extents is greater than half the maximum extent
 469	 * length, then use the current offset as the basis.  This ensures that
 470	 * for large files the preallocation size always extends to
 471	 * XFS_BMBT_MAX_EXTLEN rather than falling short due to things like stripe
 472	 * unit/width alignment of real extents.
 473	 */
 474	alloc_blocks = plen * 2;
 475	if (alloc_blocks > XFS_MAX_BMBT_EXTLEN)
 476		alloc_blocks = XFS_B_TO_FSB(mp, offset);
 477	qblocks = alloc_blocks;
 478
 479	/*
 480	 * XFS_BMBT_MAX_EXTLEN is not a power of two value but we round the prealloc
 481	 * down to the nearest power of two value after throttling. To prevent
 482	 * the round down from unconditionally reducing the maximum supported
 483	 * prealloc size, we round up first, apply appropriate throttling, round
 484	 * down and cap the value to XFS_BMBT_MAX_EXTLEN.
 485	 */
 486	alloc_blocks = XFS_FILEOFF_MIN(roundup_pow_of_two(XFS_MAX_BMBT_EXTLEN),
 487				       alloc_blocks);
 488
 489	freesp = percpu_counter_read_positive(&mp->m_fdblocks);
 490	if (freesp < mp->m_low_space[XFS_LOWSP_5_PCNT]) {
 491		shift = 2;
 492		if (freesp < mp->m_low_space[XFS_LOWSP_4_PCNT])
 493			shift++;
 494		if (freesp < mp->m_low_space[XFS_LOWSP_3_PCNT])
 495			shift++;
 496		if (freesp < mp->m_low_space[XFS_LOWSP_2_PCNT])
 497			shift++;
 498		if (freesp < mp->m_low_space[XFS_LOWSP_1_PCNT])
 499			shift++;
 500	}
 501
 502	/*
 503	 * Check each quota to cap the prealloc size, provide a shift value to
 504	 * throttle with and adjust amount of available space.
 505	 */
 506	if (xfs_quota_need_throttle(ip, XFS_DQTYPE_USER, alloc_blocks))
 507		xfs_quota_calc_throttle(ip, XFS_DQTYPE_USER, &qblocks, &qshift,
 508					&freesp);
 509	if (xfs_quota_need_throttle(ip, XFS_DQTYPE_GROUP, alloc_blocks))
 510		xfs_quota_calc_throttle(ip, XFS_DQTYPE_GROUP, &qblocks, &qshift,
 511					&freesp);
 512	if (xfs_quota_need_throttle(ip, XFS_DQTYPE_PROJ, alloc_blocks))
 513		xfs_quota_calc_throttle(ip, XFS_DQTYPE_PROJ, &qblocks, &qshift,
 514					&freesp);
 515
 516	/*
 517	 * The final prealloc size is set to the minimum of free space available
 518	 * in each of the quotas and the overall filesystem.
 519	 *
 520	 * The shift throttle value is set to the maximum value as determined by
 521	 * the global low free space values and per-quota low free space values.
 522	 */
 523	alloc_blocks = min(alloc_blocks, qblocks);
 524	shift = max(shift, qshift);
 525
 526	if (shift)
 527		alloc_blocks >>= shift;
 528	/*
 529	 * rounddown_pow_of_two() returns an undefined result if we pass in
 530	 * alloc_blocks = 0.
 531	 */
 532	if (alloc_blocks)
 533		alloc_blocks = rounddown_pow_of_two(alloc_blocks);
 534	if (alloc_blocks > XFS_MAX_BMBT_EXTLEN)
 535		alloc_blocks = XFS_MAX_BMBT_EXTLEN;
 536
 537	/*
 538	 * If we are still trying to allocate more space than is
 539	 * available, squash the prealloc hard. This can happen if we
 540	 * have a large file on a small filesystem and the above
 541	 * lowspace thresholds are smaller than XFS_BMBT_MAX_EXTLEN.
 542	 */
 543	while (alloc_blocks && alloc_blocks >= freesp)
 544		alloc_blocks >>= 4;
 545	if (alloc_blocks < mp->m_allocsize_blocks)
 546		alloc_blocks = mp->m_allocsize_blocks;
 547	trace_xfs_iomap_prealloc_size(ip, alloc_blocks, shift,
 548				      mp->m_allocsize_blocks);
 549	return alloc_blocks;
 550}
 551
 552int
 553xfs_iomap_write_unwritten(
 554	xfs_inode_t	*ip,
 555	xfs_off_t	offset,
 556	xfs_off_t	count,
 557	bool		update_isize)
 558{
 559	xfs_mount_t	*mp = ip->i_mount;
 560	xfs_fileoff_t	offset_fsb;
 561	xfs_filblks_t	count_fsb;
 562	xfs_filblks_t	numblks_fsb;
 563	int		nimaps;
 564	xfs_trans_t	*tp;
 565	xfs_bmbt_irec_t imap;
 566	struct inode	*inode = VFS_I(ip);
 567	xfs_fsize_t	i_size;
 568	uint		resblks;
 569	int		error;
 570
 571	trace_xfs_unwritten_convert(ip, offset, count);
 572
 573	offset_fsb = XFS_B_TO_FSBT(mp, offset);
 574	count_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count);
 575	count_fsb = (xfs_filblks_t)(count_fsb - offset_fsb);
 576
 577	/*
 578	 * Reserve enough blocks in this transaction for two complete extent
 579	 * btree splits.  We may be converting the middle part of an unwritten
 580	 * extent and in this case we will insert two new extents in the btree
 581	 * each of which could cause a full split.
 582	 *
 583	 * This reservation amount will be used in the first call to
 584	 * xfs_bmbt_split() to select an AG with enough space to satisfy the
 585	 * rest of the operation.
 586	 */
 587	resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1;
 588
 589	/* Attach dquots so that bmbt splits are accounted correctly. */
 590	error = xfs_qm_dqattach(ip);
 591	if (error)
 592		return error;
 593
 594	do {
 595		/*
 596		 * Set up a transaction to convert the range of extents
 597		 * from unwritten to real. Do allocations in a loop until
 598		 * we have covered the range passed in.
 599		 *
 600		 * Note that we can't risk to recursing back into the filesystem
 601		 * here as we might be asked to write out the same inode that we
 602		 * complete here and might deadlock on the iolock.
 603		 */
 604		error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks,
 605				0, true, &tp);
 606		if (error)
 607			return error;
 608
 609		error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK,
 610				XFS_IEXT_WRITE_UNWRITTEN_CNT);
 611		if (error == -EFBIG)
 612			error = xfs_iext_count_upgrade(tp, ip,
 613					XFS_IEXT_WRITE_UNWRITTEN_CNT);
 614		if (error)
 615			goto error_on_bmapi_transaction;
 616
 617		/*
 618		 * Modify the unwritten extent state of the buffer.
 619		 */
 620		nimaps = 1;
 621		error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb,
 622					XFS_BMAPI_CONVERT, resblks, &imap,
 623					&nimaps);
 624		if (error)
 625			goto error_on_bmapi_transaction;
 626
 627		/*
 628		 * Log the updated inode size as we go.  We have to be careful
 629		 * to only log it up to the actual write offset if it is
 630		 * halfway into a block.
 631		 */
 632		i_size = XFS_FSB_TO_B(mp, offset_fsb + count_fsb);
 633		if (i_size > offset + count)
 634			i_size = offset + count;
 635		if (update_isize && i_size > i_size_read(inode))
 636			i_size_write(inode, i_size);
 637		i_size = xfs_new_eof(ip, i_size);
 638		if (i_size) {
 639			ip->i_disk_size = i_size;
 640			xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
 641		}
 642
 643		error = xfs_trans_commit(tp);
 644		xfs_iunlock(ip, XFS_ILOCK_EXCL);
 645		if (error)
 646			return error;
 647
 648		if (unlikely(!xfs_valid_startblock(ip, imap.br_startblock))) {
 649			xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
 650			return xfs_alert_fsblock_zero(ip, &imap);
 651		}
 652
 653		if ((numblks_fsb = imap.br_blockcount) == 0) {
 654			/*
 655			 * The numblks_fsb value should always get
 656			 * smaller, otherwise the loop is stuck.
 657			 */
 658			ASSERT(imap.br_blockcount);
 659			break;
 660		}
 661		offset_fsb += numblks_fsb;
 662		count_fsb -= numblks_fsb;
 663	} while (count_fsb > 0);
 664
 665	return 0;
 666
 667error_on_bmapi_transaction:
 668	xfs_trans_cancel(tp);
 669	xfs_iunlock(ip, XFS_ILOCK_EXCL);
 670	return error;
 671}
 672
 673static inline bool
 674imap_needs_alloc(
 675	struct inode		*inode,
 676	unsigned		flags,
 677	struct xfs_bmbt_irec	*imap,
 678	int			nimaps)
 679{
 680	/* don't allocate blocks when just zeroing */
 681	if (flags & IOMAP_ZERO)
 682		return false;
 683	if (!nimaps ||
 684	    imap->br_startblock == HOLESTARTBLOCK ||
 685	    imap->br_startblock == DELAYSTARTBLOCK)
 686		return true;
 687	/* we convert unwritten extents before copying the data for DAX */
 688	if ((flags & IOMAP_DAX) && imap->br_state == XFS_EXT_UNWRITTEN)
 689		return true;
 690	return false;
 691}
 692
 693static inline bool
 694imap_needs_cow(
 695	struct xfs_inode	*ip,
 696	unsigned int		flags,
 697	struct xfs_bmbt_irec	*imap,
 698	int			nimaps)
 699{
 700	if (!xfs_is_cow_inode(ip))
 701		return false;
 702
 703	/* when zeroing we don't have to COW holes or unwritten extents */
 704	if (flags & IOMAP_ZERO) {
 705		if (!nimaps ||
 706		    imap->br_startblock == HOLESTARTBLOCK ||
 707		    imap->br_state == XFS_EXT_UNWRITTEN)
 708			return false;
 709	}
 710
 711	return true;
 712}
 713
 714static int
 715xfs_ilock_for_iomap(
 716	struct xfs_inode	*ip,
 717	unsigned		flags,
 718	unsigned		*lockmode)
 719{
 720	unsigned int		mode = *lockmode;
 721	bool			is_write = flags & (IOMAP_WRITE | IOMAP_ZERO);
 722
 723	/*
 724	 * COW writes may allocate delalloc space or convert unwritten COW
 725	 * extents, so we need to make sure to take the lock exclusively here.
 726	 */
 727	if (xfs_is_cow_inode(ip) && is_write)
 728		mode = XFS_ILOCK_EXCL;
 729
 730	/*
 731	 * Extents not yet cached requires exclusive access, don't block.  This
 732	 * is an opencoded xfs_ilock_data_map_shared() call but with
 733	 * non-blocking behaviour.
 734	 */
 735	if (xfs_need_iread_extents(&ip->i_df)) {
 736		if (flags & IOMAP_NOWAIT)
 737			return -EAGAIN;
 738		mode = XFS_ILOCK_EXCL;
 739	}
 740
 741relock:
 742	if (flags & IOMAP_NOWAIT) {
 743		if (!xfs_ilock_nowait(ip, mode))
 744			return -EAGAIN;
 745	} else {
 746		xfs_ilock(ip, mode);
 747	}
 748
 749	/*
 750	 * The reflink iflag could have changed since the earlier unlocked
 751	 * check, so if we got ILOCK_SHARED for a write and but we're now a
 752	 * reflink inode we have to switch to ILOCK_EXCL and relock.
 753	 */
 754	if (mode == XFS_ILOCK_SHARED && is_write && xfs_is_cow_inode(ip)) {
 755		xfs_iunlock(ip, mode);
 756		mode = XFS_ILOCK_EXCL;
 757		goto relock;
 758	}
 759
 760	*lockmode = mode;
 761	return 0;
 762}
 763
 764/*
 765 * Check that the imap we are going to return to the caller spans the entire
 766 * range that the caller requested for the IO.
 767 */
 768static bool
 769imap_spans_range(
 770	struct xfs_bmbt_irec	*imap,
 771	xfs_fileoff_t		offset_fsb,
 772	xfs_fileoff_t		end_fsb)
 773{
 774	if (imap->br_startoff > offset_fsb)
 775		return false;
 776	if (imap->br_startoff + imap->br_blockcount < end_fsb)
 777		return false;
 778	return true;
 779}
 780
 781static int
 782xfs_direct_write_iomap_begin(
 783	struct inode		*inode,
 784	loff_t			offset,
 785	loff_t			length,
 786	unsigned		flags,
 787	struct iomap		*iomap,
 788	struct iomap		*srcmap)
 789{
 790	struct xfs_inode	*ip = XFS_I(inode);
 791	struct xfs_mount	*mp = ip->i_mount;
 792	struct xfs_bmbt_irec	imap, cmap;
 793	xfs_fileoff_t		offset_fsb = XFS_B_TO_FSBT(mp, offset);
 794	xfs_fileoff_t		end_fsb = xfs_iomap_end_fsb(mp, offset, length);
 795	int			nimaps = 1, error = 0;
 796	bool			shared = false;
 797	u16			iomap_flags = 0;
 798	unsigned int		lockmode = XFS_ILOCK_SHARED;
 799	u64			seq;
 800
 801	ASSERT(flags & (IOMAP_WRITE | IOMAP_ZERO));
 802
 803	if (xfs_is_shutdown(mp))
 804		return -EIO;
 805
 806	/*
 807	 * Writes that span EOF might trigger an IO size update on completion,
 808	 * so consider them to be dirty for the purposes of O_DSYNC even if
 809	 * there is no other metadata changes pending or have been made here.
 810	 */
 811	if (offset + length > i_size_read(inode))
 812		iomap_flags |= IOMAP_F_DIRTY;
 813
 814	error = xfs_ilock_for_iomap(ip, flags, &lockmode);
 815	if (error)
 816		return error;
 817
 818	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
 819			       &nimaps, 0);
 820	if (error)
 821		goto out_unlock;
 822
 823	if (imap_needs_cow(ip, flags, &imap, nimaps)) {
 824		error = -EAGAIN;
 825		if (flags & IOMAP_NOWAIT)
 826			goto out_unlock;
 827
 828		/* may drop and re-acquire the ilock */
 829		error = xfs_reflink_allocate_cow(ip, &imap, &cmap, &shared,
 830				&lockmode,
 831				(flags & IOMAP_DIRECT) || IS_DAX(inode));
 832		if (error)
 833			goto out_unlock;
 834		if (shared)
 835			goto out_found_cow;
 836		end_fsb = imap.br_startoff + imap.br_blockcount;
 837		length = XFS_FSB_TO_B(mp, end_fsb) - offset;
 838	}
 839
 840	if (imap_needs_alloc(inode, flags, &imap, nimaps))
 841		goto allocate_blocks;
 842
 843	/*
 844	 * NOWAIT and OVERWRITE I/O needs to span the entire requested I/O with
 845	 * a single map so that we avoid partial IO failures due to the rest of
 846	 * the I/O range not covered by this map triggering an EAGAIN condition
 847	 * when it is subsequently mapped and aborting the I/O.
 848	 */
 849	if (flags & (IOMAP_NOWAIT | IOMAP_OVERWRITE_ONLY)) {
 850		error = -EAGAIN;
 851		if (!imap_spans_range(&imap, offset_fsb, end_fsb))
 852			goto out_unlock;
 853	}
 854
 855	/*
 856	 * For overwrite only I/O, we cannot convert unwritten extents without
 857	 * requiring sub-block zeroing.  This can only be done under an
 858	 * exclusive IOLOCK, hence return -EAGAIN if this is not a written
 859	 * extent to tell the caller to try again.
 860	 */
 861	if (flags & IOMAP_OVERWRITE_ONLY) {
 862		error = -EAGAIN;
 863		if (imap.br_state != XFS_EXT_NORM &&
 864	            ((offset | length) & mp->m_blockmask))
 865			goto out_unlock;
 866	}
 867
 868	seq = xfs_iomap_inode_sequence(ip, iomap_flags);
 869	xfs_iunlock(ip, lockmode);
 870	trace_xfs_iomap_found(ip, offset, length, XFS_DATA_FORK, &imap);
 871	return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, iomap_flags, seq);
 872
 873allocate_blocks:
 874	error = -EAGAIN;
 875	if (flags & (IOMAP_NOWAIT | IOMAP_OVERWRITE_ONLY))
 876		goto out_unlock;
 877
 878	/*
 879	 * We cap the maximum length we map to a sane size  to keep the chunks
 880	 * of work done where somewhat symmetric with the work writeback does.
 881	 * This is a completely arbitrary number pulled out of thin air as a
 882	 * best guess for initial testing.
 883	 *
 884	 * Note that the values needs to be less than 32-bits wide until the
 885	 * lower level functions are updated.
 886	 */
 887	length = min_t(loff_t, length, 1024 * PAGE_SIZE);
 888	end_fsb = xfs_iomap_end_fsb(mp, offset, length);
 889
 890	if (offset + length > XFS_ISIZE(ip))
 891		end_fsb = xfs_iomap_eof_align_last_fsb(ip, end_fsb);
 892	else if (nimaps && imap.br_startblock == HOLESTARTBLOCK)
 893		end_fsb = min(end_fsb, imap.br_startoff + imap.br_blockcount);
 894	xfs_iunlock(ip, lockmode);
 895
 896	error = xfs_iomap_write_direct(ip, offset_fsb, end_fsb - offset_fsb,
 897			flags, &imap, &seq);
 898	if (error)
 899		return error;
 900
 901	trace_xfs_iomap_alloc(ip, offset, length, XFS_DATA_FORK, &imap);
 902	return xfs_bmbt_to_iomap(ip, iomap, &imap, flags,
 903				 iomap_flags | IOMAP_F_NEW, seq);
 904
 905out_found_cow:
 
 906	length = XFS_FSB_TO_B(mp, cmap.br_startoff + cmap.br_blockcount);
 907	trace_xfs_iomap_found(ip, offset, length - offset, XFS_COW_FORK, &cmap);
 908	if (imap.br_startblock != HOLESTARTBLOCK) {
 909		seq = xfs_iomap_inode_sequence(ip, 0);
 910		error = xfs_bmbt_to_iomap(ip, srcmap, &imap, flags, 0, seq);
 911		if (error)
 912			goto out_unlock;
 913	}
 914	seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED);
 915	xfs_iunlock(ip, lockmode);
 916	return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, IOMAP_F_SHARED, seq);
 917
 918out_unlock:
 919	if (lockmode)
 920		xfs_iunlock(ip, lockmode);
 921	return error;
 922}
 923
 924const struct iomap_ops xfs_direct_write_iomap_ops = {
 925	.iomap_begin		= xfs_direct_write_iomap_begin,
 926};
 927
 928static int
 929xfs_dax_write_iomap_end(
 930	struct inode		*inode,
 931	loff_t			pos,
 932	loff_t			length,
 933	ssize_t			written,
 934	unsigned		flags,
 935	struct iomap		*iomap)
 936{
 937	struct xfs_inode	*ip = XFS_I(inode);
 938
 939	if (!xfs_is_cow_inode(ip))
 940		return 0;
 941
 942	if (!written) {
 943		xfs_reflink_cancel_cow_range(ip, pos, length, true);
 944		return 0;
 945	}
 946
 947	return xfs_reflink_end_cow(ip, pos, written);
 948}
 949
 950const struct iomap_ops xfs_dax_write_iomap_ops = {
 951	.iomap_begin	= xfs_direct_write_iomap_begin,
 952	.iomap_end	= xfs_dax_write_iomap_end,
 953};
 954
 955static int
 956xfs_buffered_write_iomap_begin(
 957	struct inode		*inode,
 958	loff_t			offset,
 959	loff_t			count,
 960	unsigned		flags,
 961	struct iomap		*iomap,
 962	struct iomap		*srcmap)
 963{
 964	struct xfs_inode	*ip = XFS_I(inode);
 965	struct xfs_mount	*mp = ip->i_mount;
 966	xfs_fileoff_t		offset_fsb = XFS_B_TO_FSBT(mp, offset);
 967	xfs_fileoff_t		end_fsb = xfs_iomap_end_fsb(mp, offset, count);
 968	struct xfs_bmbt_irec	imap, cmap;
 969	struct xfs_iext_cursor	icur, ccur;
 970	xfs_fsblock_t		prealloc_blocks = 0;
 971	bool			eof = false, cow_eof = false, shared = false;
 972	int			allocfork = XFS_DATA_FORK;
 973	int			error = 0;
 974	unsigned int		lockmode = XFS_ILOCK_EXCL;
 975	u64			seq;
 976
 977	if (xfs_is_shutdown(mp))
 978		return -EIO;
 979
 980	/* we can't use delayed allocations when using extent size hints */
 981	if (xfs_get_extsz_hint(ip))
 982		return xfs_direct_write_iomap_begin(inode, offset, count,
 983				flags, iomap, srcmap);
 984
 985	ASSERT(!XFS_IS_REALTIME_INODE(ip));
 986
 987	error = xfs_qm_dqattach(ip);
 988	if (error)
 989		return error;
 990
 991	error = xfs_ilock_for_iomap(ip, flags, &lockmode);
 992	if (error)
 993		return error;
 994
 995	if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(&ip->i_df)) ||
 996	    XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) {
 997		xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
 998		error = -EFSCORRUPTED;
 999		goto out_unlock;
1000	}
1001
1002	XFS_STATS_INC(mp, xs_blk_mapw);
1003
1004	error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK);
1005	if (error)
1006		goto out_unlock;
1007
1008	/*
1009	 * Search the data fork first to look up our source mapping.  We
1010	 * always need the data fork map, as we have to return it to the
1011	 * iomap code so that the higher level write code can read data in to
1012	 * perform read-modify-write cycles for unaligned writes.
1013	 */
1014	eof = !xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap);
1015	if (eof)
1016		imap.br_startoff = end_fsb; /* fake hole until the end */
1017
1018	/* We never need to allocate blocks for zeroing or unsharing a hole. */
1019	if ((flags & (IOMAP_UNSHARE | IOMAP_ZERO)) &&
1020	    imap.br_startoff > offset_fsb) {
1021		xfs_hole_to_iomap(ip, iomap, offset_fsb, imap.br_startoff);
1022		goto out_unlock;
1023	}
1024
1025	/*
1026	 * Search the COW fork extent list even if we did not find a data fork
1027	 * extent.  This serves two purposes: first this implements the
1028	 * speculative preallocation using cowextsize, so that we also unshare
1029	 * block adjacent to shared blocks instead of just the shared blocks
1030	 * themselves.  Second the lookup in the extent list is generally faster
1031	 * than going out to the shared extent tree.
1032	 */
1033	if (xfs_is_cow_inode(ip)) {
1034		if (!ip->i_cowfp) {
1035			ASSERT(!xfs_is_reflink_inode(ip));
1036			xfs_ifork_init_cow(ip);
1037		}
1038		cow_eof = !xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb,
1039				&ccur, &cmap);
1040		if (!cow_eof && cmap.br_startoff <= offset_fsb) {
1041			trace_xfs_reflink_cow_found(ip, &cmap);
1042			goto found_cow;
1043		}
1044	}
1045
1046	if (imap.br_startoff <= offset_fsb) {
1047		/*
1048		 * For reflink files we may need a delalloc reservation when
1049		 * overwriting shared extents.   This includes zeroing of
1050		 * existing extents that contain data.
1051		 */
1052		if (!xfs_is_cow_inode(ip) ||
1053		    ((flags & IOMAP_ZERO) && imap.br_state != XFS_EXT_NORM)) {
1054			trace_xfs_iomap_found(ip, offset, count, XFS_DATA_FORK,
1055					&imap);
1056			goto found_imap;
1057		}
1058
1059		xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb);
1060
1061		/* Trim the mapping to the nearest shared extent boundary. */
1062		error = xfs_bmap_trim_cow(ip, &imap, &shared);
1063		if (error)
1064			goto out_unlock;
1065
1066		/* Not shared?  Just report the (potentially capped) extent. */
1067		if (!shared) {
1068			trace_xfs_iomap_found(ip, offset, count, XFS_DATA_FORK,
1069					&imap);
1070			goto found_imap;
1071		}
1072
1073		/*
1074		 * Fork all the shared blocks from our write offset until the
1075		 * end of the extent.
1076		 */
1077		allocfork = XFS_COW_FORK;
1078		end_fsb = imap.br_startoff + imap.br_blockcount;
1079	} else {
1080		/*
1081		 * We cap the maximum length we map here to MAX_WRITEBACK_PAGES
1082		 * pages to keep the chunks of work done where somewhat
1083		 * symmetric with the work writeback does.  This is a completely
1084		 * arbitrary number pulled out of thin air.
1085		 *
1086		 * Note that the values needs to be less than 32-bits wide until
1087		 * the lower level functions are updated.
1088		 */
1089		count = min_t(loff_t, count, 1024 * PAGE_SIZE);
1090		end_fsb = xfs_iomap_end_fsb(mp, offset, count);
1091
1092		if (xfs_is_always_cow_inode(ip))
1093			allocfork = XFS_COW_FORK;
1094	}
1095
 
 
 
 
1096	if (eof && offset + count > XFS_ISIZE(ip)) {
1097		/*
1098		 * Determine the initial size of the preallocation.
1099		 * We clean up any extra preallocation when the file is closed.
1100		 */
1101		if (xfs_has_allocsize(mp))
1102			prealloc_blocks = mp->m_allocsize_blocks;
1103		else if (allocfork == XFS_DATA_FORK)
1104			prealloc_blocks = xfs_iomap_prealloc_size(ip, allocfork,
1105						offset, count, &icur);
1106		else
1107			prealloc_blocks = xfs_iomap_prealloc_size(ip, allocfork,
1108						offset, count, &ccur);
1109		if (prealloc_blocks) {
1110			xfs_extlen_t	align;
1111			xfs_off_t	end_offset;
1112			xfs_fileoff_t	p_end_fsb;
1113
1114			end_offset = XFS_ALLOC_ALIGN(mp, offset + count - 1);
1115			p_end_fsb = XFS_B_TO_FSBT(mp, end_offset) +
1116					prealloc_blocks;
1117
1118			align = xfs_eof_alignment(ip);
1119			if (align)
1120				p_end_fsb = roundup_64(p_end_fsb, align);
1121
1122			p_end_fsb = min(p_end_fsb,
1123				XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes));
1124			ASSERT(p_end_fsb > offset_fsb);
1125			prealloc_blocks = p_end_fsb - end_fsb;
1126		}
1127	}
1128
1129retry:
1130	error = xfs_bmapi_reserve_delalloc(ip, allocfork, offset_fsb,
1131			end_fsb - offset_fsb, prealloc_blocks,
1132			allocfork == XFS_DATA_FORK ? &imap : &cmap,
1133			allocfork == XFS_DATA_FORK ? &icur : &ccur,
1134			allocfork == XFS_DATA_FORK ? eof : cow_eof);
1135	switch (error) {
1136	case 0:
1137		break;
1138	case -ENOSPC:
1139	case -EDQUOT:
1140		/* retry without any preallocation */
1141		trace_xfs_delalloc_enospc(ip, offset, count);
1142		if (prealloc_blocks) {
1143			prealloc_blocks = 0;
1144			goto retry;
1145		}
1146		fallthrough;
1147	default:
1148		goto out_unlock;
1149	}
1150
1151	if (allocfork == XFS_COW_FORK) {
1152		trace_xfs_iomap_alloc(ip, offset, count, allocfork, &cmap);
1153		goto found_cow;
1154	}
1155
1156	/*
1157	 * Flag newly allocated delalloc blocks with IOMAP_F_NEW so we punch
1158	 * them out if the write happens to fail.
1159	 */
1160	seq = xfs_iomap_inode_sequence(ip, IOMAP_F_NEW);
1161	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1162	trace_xfs_iomap_alloc(ip, offset, count, allocfork, &imap);
1163	return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, IOMAP_F_NEW, seq);
1164
1165found_imap:
1166	seq = xfs_iomap_inode_sequence(ip, 0);
1167	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1168	return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, 0, seq);
1169
1170found_cow:
1171	seq = xfs_iomap_inode_sequence(ip, 0);
1172	if (imap.br_startoff <= offset_fsb) {
1173		error = xfs_bmbt_to_iomap(ip, srcmap, &imap, flags, 0, seq);
1174		if (error)
1175			goto out_unlock;
1176		seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED);
1177		xfs_iunlock(ip, XFS_ILOCK_EXCL);
1178		return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags,
1179					 IOMAP_F_SHARED, seq);
1180	}
1181
1182	xfs_trim_extent(&cmap, offset_fsb, imap.br_startoff - offset_fsb);
1183	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1184	return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, 0, seq);
1185
1186out_unlock:
1187	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1188	return error;
1189}
1190
1191static int
1192xfs_buffered_write_delalloc_punch(
1193	struct inode		*inode,
1194	loff_t			offset,
1195	loff_t			length)
1196{
1197	return xfs_bmap_punch_delalloc_range(XFS_I(inode), offset,
1198			offset + length);
1199}
1200
1201static int
1202xfs_buffered_write_iomap_end(
1203	struct inode		*inode,
1204	loff_t			offset,
1205	loff_t			length,
1206	ssize_t			written,
1207	unsigned		flags,
1208	struct iomap		*iomap)
1209{
 
 
 
 
 
1210
1211	struct xfs_mount	*mp = XFS_M(inode->i_sb);
1212	int			error;
1213
1214	error = iomap_file_buffered_write_punch_delalloc(inode, iomap, offset,
1215			length, written, &xfs_buffered_write_delalloc_punch);
1216	if (error && !xfs_is_shutdown(mp)) {
1217		xfs_alert(mp, "%s: unable to clean up ino 0x%llx",
1218			__func__, XFS_I(inode)->i_ino);
1219		return error;
 
1220	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1221	return 0;
1222}
1223
1224const struct iomap_ops xfs_buffered_write_iomap_ops = {
1225	.iomap_begin		= xfs_buffered_write_iomap_begin,
1226	.iomap_end		= xfs_buffered_write_iomap_end,
1227};
1228
1229/*
1230 * iomap_page_mkwrite() will never fail in a way that requires delalloc extents
1231 * that it allocated to be revoked. Hence we do not need an .iomap_end method
1232 * for this operation.
1233 */
1234const struct iomap_ops xfs_page_mkwrite_iomap_ops = {
1235	.iomap_begin		= xfs_buffered_write_iomap_begin,
1236};
1237
1238static int
1239xfs_read_iomap_begin(
1240	struct inode		*inode,
1241	loff_t			offset,
1242	loff_t			length,
1243	unsigned		flags,
1244	struct iomap		*iomap,
1245	struct iomap		*srcmap)
1246{
1247	struct xfs_inode	*ip = XFS_I(inode);
1248	struct xfs_mount	*mp = ip->i_mount;
1249	struct xfs_bmbt_irec	imap;
1250	xfs_fileoff_t		offset_fsb = XFS_B_TO_FSBT(mp, offset);
1251	xfs_fileoff_t		end_fsb = xfs_iomap_end_fsb(mp, offset, length);
1252	int			nimaps = 1, error = 0;
1253	bool			shared = false;
1254	unsigned int		lockmode = XFS_ILOCK_SHARED;
1255	u64			seq;
1256
1257	ASSERT(!(flags & (IOMAP_WRITE | IOMAP_ZERO)));
1258
1259	if (xfs_is_shutdown(mp))
1260		return -EIO;
1261
1262	error = xfs_ilock_for_iomap(ip, flags, &lockmode);
1263	if (error)
1264		return error;
1265	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
1266			       &nimaps, 0);
1267	if (!error && ((flags & IOMAP_REPORT) || IS_DAX(inode)))
1268		error = xfs_reflink_trim_around_shared(ip, &imap, &shared);
1269	seq = xfs_iomap_inode_sequence(ip, shared ? IOMAP_F_SHARED : 0);
1270	xfs_iunlock(ip, lockmode);
1271
1272	if (error)
1273		return error;
1274	trace_xfs_iomap_found(ip, offset, length, XFS_DATA_FORK, &imap);
1275	return xfs_bmbt_to_iomap(ip, iomap, &imap, flags,
1276				 shared ? IOMAP_F_SHARED : 0, seq);
1277}
1278
1279const struct iomap_ops xfs_read_iomap_ops = {
1280	.iomap_begin		= xfs_read_iomap_begin,
1281};
1282
1283static int
1284xfs_seek_iomap_begin(
1285	struct inode		*inode,
1286	loff_t			offset,
1287	loff_t			length,
1288	unsigned		flags,
1289	struct iomap		*iomap,
1290	struct iomap		*srcmap)
1291{
1292	struct xfs_inode	*ip = XFS_I(inode);
1293	struct xfs_mount	*mp = ip->i_mount;
1294	xfs_fileoff_t		offset_fsb = XFS_B_TO_FSBT(mp, offset);
1295	xfs_fileoff_t		end_fsb = XFS_B_TO_FSB(mp, offset + length);
1296	xfs_fileoff_t		cow_fsb = NULLFILEOFF, data_fsb = NULLFILEOFF;
1297	struct xfs_iext_cursor	icur;
1298	struct xfs_bmbt_irec	imap, cmap;
1299	int			error = 0;
1300	unsigned		lockmode;
1301	u64			seq;
1302
1303	if (xfs_is_shutdown(mp))
1304		return -EIO;
1305
1306	lockmode = xfs_ilock_data_map_shared(ip);
1307	error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK);
1308	if (error)
1309		goto out_unlock;
1310
1311	if (xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap)) {
1312		/*
1313		 * If we found a data extent we are done.
1314		 */
1315		if (imap.br_startoff <= offset_fsb)
1316			goto done;
1317		data_fsb = imap.br_startoff;
1318	} else {
1319		/*
1320		 * Fake a hole until the end of the file.
1321		 */
1322		data_fsb = xfs_iomap_end_fsb(mp, offset, length);
1323	}
1324
1325	/*
1326	 * If a COW fork extent covers the hole, report it - capped to the next
1327	 * data fork extent:
1328	 */
1329	if (xfs_inode_has_cow_data(ip) &&
1330	    xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &cmap))
1331		cow_fsb = cmap.br_startoff;
1332	if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) {
1333		if (data_fsb < cow_fsb + cmap.br_blockcount)
1334			end_fsb = min(end_fsb, data_fsb);
1335		xfs_trim_extent(&cmap, offset_fsb, end_fsb - offset_fsb);
1336		seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED);
1337		error = xfs_bmbt_to_iomap(ip, iomap, &cmap, flags,
1338				IOMAP_F_SHARED, seq);
1339		/*
1340		 * This is a COW extent, so we must probe the page cache
1341		 * because there could be dirty page cache being backed
1342		 * by this extent.
1343		 */
1344		iomap->type = IOMAP_UNWRITTEN;
1345		goto out_unlock;
1346	}
1347
1348	/*
1349	 * Else report a hole, capped to the next found data or COW extent.
1350	 */
1351	if (cow_fsb != NULLFILEOFF && cow_fsb < data_fsb)
1352		imap.br_blockcount = cow_fsb - offset_fsb;
1353	else
1354		imap.br_blockcount = data_fsb - offset_fsb;
1355	imap.br_startoff = offset_fsb;
1356	imap.br_startblock = HOLESTARTBLOCK;
1357	imap.br_state = XFS_EXT_NORM;
1358done:
1359	seq = xfs_iomap_inode_sequence(ip, 0);
1360	xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb);
1361	error = xfs_bmbt_to_iomap(ip, iomap, &imap, flags, 0, seq);
1362out_unlock:
1363	xfs_iunlock(ip, lockmode);
1364	return error;
1365}
1366
1367const struct iomap_ops xfs_seek_iomap_ops = {
1368	.iomap_begin		= xfs_seek_iomap_begin,
1369};
1370
1371static int
1372xfs_xattr_iomap_begin(
1373	struct inode		*inode,
1374	loff_t			offset,
1375	loff_t			length,
1376	unsigned		flags,
1377	struct iomap		*iomap,
1378	struct iomap		*srcmap)
1379{
1380	struct xfs_inode	*ip = XFS_I(inode);
1381	struct xfs_mount	*mp = ip->i_mount;
1382	xfs_fileoff_t		offset_fsb = XFS_B_TO_FSBT(mp, offset);
1383	xfs_fileoff_t		end_fsb = XFS_B_TO_FSB(mp, offset + length);
1384	struct xfs_bmbt_irec	imap;
1385	int			nimaps = 1, error = 0;
1386	unsigned		lockmode;
1387	int			seq;
1388
1389	if (xfs_is_shutdown(mp))
1390		return -EIO;
1391
1392	lockmode = xfs_ilock_attr_map_shared(ip);
1393
1394	/* if there are no attribute fork or extents, return ENOENT */
1395	if (!xfs_inode_has_attr_fork(ip) || !ip->i_af.if_nextents) {
1396		error = -ENOENT;
1397		goto out_unlock;
1398	}
1399
1400	ASSERT(ip->i_af.if_format != XFS_DINODE_FMT_LOCAL);
1401	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
1402			       &nimaps, XFS_BMAPI_ATTRFORK);
1403out_unlock:
1404
1405	seq = xfs_iomap_inode_sequence(ip, IOMAP_F_XATTR);
1406	xfs_iunlock(ip, lockmode);
1407
1408	if (error)
1409		return error;
1410	ASSERT(nimaps);
1411	return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, IOMAP_F_XATTR, seq);
1412}
1413
1414const struct iomap_ops xfs_xattr_iomap_ops = {
1415	.iomap_begin		= xfs_xattr_iomap_begin,
1416};
1417
1418int
1419xfs_zero_range(
1420	struct xfs_inode	*ip,
1421	loff_t			pos,
1422	loff_t			len,
1423	bool			*did_zero)
1424{
1425	struct inode		*inode = VFS_I(ip);
1426
1427	if (IS_DAX(inode))
1428		return dax_zero_range(inode, pos, len, did_zero,
1429				      &xfs_dax_write_iomap_ops);
1430	return iomap_zero_range(inode, pos, len, did_zero,
1431				&xfs_buffered_write_iomap_ops);
1432}
1433
1434int
1435xfs_truncate_page(
1436	struct xfs_inode	*ip,
1437	loff_t			pos,
1438	bool			*did_zero)
1439{
1440	struct inode		*inode = VFS_I(ip);
1441
1442	if (IS_DAX(inode))
1443		return dax_truncate_page(inode, pos, did_zero,
1444					&xfs_dax_write_iomap_ops);
1445	return iomap_truncate_page(inode, pos, did_zero,
1446				   &xfs_buffered_write_iomap_ops);
1447}