Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2000-2006 Silicon Graphics, Inc.
4 * Copyright (c) 2016-2018 Christoph Hellwig.
5 * All Rights Reserved.
6 */
7#include "xfs.h"
8#include "xfs_fs.h"
9#include "xfs_shared.h"
10#include "xfs_format.h"
11#include "xfs_log_format.h"
12#include "xfs_trans_resv.h"
13#include "xfs_mount.h"
14#include "xfs_inode.h"
15#include "xfs_btree.h"
16#include "xfs_bmap_btree.h"
17#include "xfs_bmap.h"
18#include "xfs_bmap_util.h"
19#include "xfs_errortag.h"
20#include "xfs_error.h"
21#include "xfs_trans.h"
22#include "xfs_trans_space.h"
23#include "xfs_inode_item.h"
24#include "xfs_iomap.h"
25#include "xfs_trace.h"
26#include "xfs_quota.h"
27#include "xfs_dquot_item.h"
28#include "xfs_dquot.h"
29#include "xfs_reflink.h"
30
31#define XFS_ALLOC_ALIGN(mp, off) \
32 (((off) >> mp->m_allocsize_log) << mp->m_allocsize_log)
33
34static int
35xfs_alert_fsblock_zero(
36 xfs_inode_t *ip,
37 xfs_bmbt_irec_t *imap)
38{
39 xfs_alert_tag(ip->i_mount, XFS_PTAG_FSBLOCK_ZERO,
40 "Access to block zero in inode %llu "
41 "start_block: %llx start_off: %llx "
42 "blkcnt: %llx extent-state: %x",
43 (unsigned long long)ip->i_ino,
44 (unsigned long long)imap->br_startblock,
45 (unsigned long long)imap->br_startoff,
46 (unsigned long long)imap->br_blockcount,
47 imap->br_state);
48 return -EFSCORRUPTED;
49}
50
51u64
52xfs_iomap_inode_sequence(
53 struct xfs_inode *ip,
54 u16 iomap_flags)
55{
56 u64 cookie = 0;
57
58 if (iomap_flags & IOMAP_F_XATTR)
59 return READ_ONCE(ip->i_af.if_seq);
60 if ((iomap_flags & IOMAP_F_SHARED) && ip->i_cowfp)
61 cookie = (u64)READ_ONCE(ip->i_cowfp->if_seq) << 32;
62 return cookie | READ_ONCE(ip->i_df.if_seq);
63}
64
65/*
66 * Check that the iomap passed to us is still valid for the given offset and
67 * length.
68 */
69static bool
70xfs_iomap_valid(
71 struct inode *inode,
72 const struct iomap *iomap)
73{
74 struct xfs_inode *ip = XFS_I(inode);
75
76 if (iomap->validity_cookie !=
77 xfs_iomap_inode_sequence(ip, iomap->flags)) {
78 trace_xfs_iomap_invalid(ip, iomap);
79 return false;
80 }
81
82 XFS_ERRORTAG_DELAY(ip->i_mount, XFS_ERRTAG_WRITE_DELAY_MS);
83 return true;
84}
85
86static const struct iomap_page_ops xfs_iomap_page_ops = {
87 .iomap_valid = xfs_iomap_valid,
88};
89
90int
91xfs_bmbt_to_iomap(
92 struct xfs_inode *ip,
93 struct iomap *iomap,
94 struct xfs_bmbt_irec *imap,
95 unsigned int mapping_flags,
96 u16 iomap_flags,
97 u64 sequence_cookie)
98{
99 struct xfs_mount *mp = ip->i_mount;
100 struct xfs_buftarg *target = xfs_inode_buftarg(ip);
101
102 if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock)))
103 return xfs_alert_fsblock_zero(ip, imap);
104
105 if (imap->br_startblock == HOLESTARTBLOCK) {
106 iomap->addr = IOMAP_NULL_ADDR;
107 iomap->type = IOMAP_HOLE;
108 } else if (imap->br_startblock == DELAYSTARTBLOCK ||
109 isnullstartblock(imap->br_startblock)) {
110 iomap->addr = IOMAP_NULL_ADDR;
111 iomap->type = IOMAP_DELALLOC;
112 } else {
113 iomap->addr = BBTOB(xfs_fsb_to_db(ip, imap->br_startblock));
114 if (mapping_flags & IOMAP_DAX)
115 iomap->addr += target->bt_dax_part_off;
116
117 if (imap->br_state == XFS_EXT_UNWRITTEN)
118 iomap->type = IOMAP_UNWRITTEN;
119 else
120 iomap->type = IOMAP_MAPPED;
121
122 }
123 iomap->offset = XFS_FSB_TO_B(mp, imap->br_startoff);
124 iomap->length = XFS_FSB_TO_B(mp, imap->br_blockcount);
125 if (mapping_flags & IOMAP_DAX)
126 iomap->dax_dev = target->bt_daxdev;
127 else
128 iomap->bdev = target->bt_bdev;
129 iomap->flags = iomap_flags;
130
131 if (xfs_ipincount(ip) &&
132 (ip->i_itemp->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP))
133 iomap->flags |= IOMAP_F_DIRTY;
134
135 iomap->validity_cookie = sequence_cookie;
136 iomap->page_ops = &xfs_iomap_page_ops;
137 return 0;
138}
139
140static void
141xfs_hole_to_iomap(
142 struct xfs_inode *ip,
143 struct iomap *iomap,
144 xfs_fileoff_t offset_fsb,
145 xfs_fileoff_t end_fsb)
146{
147 struct xfs_buftarg *target = xfs_inode_buftarg(ip);
148
149 iomap->addr = IOMAP_NULL_ADDR;
150 iomap->type = IOMAP_HOLE;
151 iomap->offset = XFS_FSB_TO_B(ip->i_mount, offset_fsb);
152 iomap->length = XFS_FSB_TO_B(ip->i_mount, end_fsb - offset_fsb);
153 iomap->bdev = target->bt_bdev;
154 iomap->dax_dev = target->bt_daxdev;
155}
156
157static inline xfs_fileoff_t
158xfs_iomap_end_fsb(
159 struct xfs_mount *mp,
160 loff_t offset,
161 loff_t count)
162{
163 ASSERT(offset <= mp->m_super->s_maxbytes);
164 return min(XFS_B_TO_FSB(mp, offset + count),
165 XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes));
166}
167
168static xfs_extlen_t
169xfs_eof_alignment(
170 struct xfs_inode *ip)
171{
172 struct xfs_mount *mp = ip->i_mount;
173 xfs_extlen_t align = 0;
174
175 if (!XFS_IS_REALTIME_INODE(ip)) {
176 /*
177 * Round up the allocation request to a stripe unit
178 * (m_dalign) boundary if the file size is >= stripe unit
179 * size, and we are allocating past the allocation eof.
180 *
181 * If mounted with the "-o swalloc" option the alignment is
182 * increased from the strip unit size to the stripe width.
183 */
184 if (mp->m_swidth && xfs_has_swalloc(mp))
185 align = mp->m_swidth;
186 else if (mp->m_dalign)
187 align = mp->m_dalign;
188
189 if (align && XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, align))
190 align = 0;
191 }
192
193 return align;
194}
195
196/*
197 * Check if last_fsb is outside the last extent, and if so grow it to the next
198 * stripe unit boundary.
199 */
200xfs_fileoff_t
201xfs_iomap_eof_align_last_fsb(
202 struct xfs_inode *ip,
203 xfs_fileoff_t end_fsb)
204{
205 struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK);
206 xfs_extlen_t extsz = xfs_get_extsz_hint(ip);
207 xfs_extlen_t align = xfs_eof_alignment(ip);
208 struct xfs_bmbt_irec irec;
209 struct xfs_iext_cursor icur;
210
211 ASSERT(!xfs_need_iread_extents(ifp));
212
213 /*
214 * Always round up the allocation request to the extent hint boundary.
215 */
216 if (extsz) {
217 if (align)
218 align = roundup_64(align, extsz);
219 else
220 align = extsz;
221 }
222
223 if (align) {
224 xfs_fileoff_t aligned_end_fsb = roundup_64(end_fsb, align);
225
226 xfs_iext_last(ifp, &icur);
227 if (!xfs_iext_get_extent(ifp, &icur, &irec) ||
228 aligned_end_fsb >= irec.br_startoff + irec.br_blockcount)
229 return aligned_end_fsb;
230 }
231
232 return end_fsb;
233}
234
235int
236xfs_iomap_write_direct(
237 struct xfs_inode *ip,
238 xfs_fileoff_t offset_fsb,
239 xfs_fileoff_t count_fsb,
240 unsigned int flags,
241 struct xfs_bmbt_irec *imap,
242 u64 *seq)
243{
244 struct xfs_mount *mp = ip->i_mount;
245 struct xfs_trans *tp;
246 xfs_filblks_t resaligned;
247 int nimaps;
248 unsigned int dblocks, rblocks;
249 bool force = false;
250 int error;
251 int bmapi_flags = XFS_BMAPI_PREALLOC;
252 int nr_exts = XFS_IEXT_ADD_NOSPLIT_CNT;
253
254 ASSERT(count_fsb > 0);
255
256 resaligned = xfs_aligned_fsb_count(offset_fsb, count_fsb,
257 xfs_get_extsz_hint(ip));
258 if (unlikely(XFS_IS_REALTIME_INODE(ip))) {
259 dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0);
260 rblocks = resaligned;
261 } else {
262 dblocks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned);
263 rblocks = 0;
264 }
265
266 error = xfs_qm_dqattach(ip);
267 if (error)
268 return error;
269
270 /*
271 * For DAX, we do not allocate unwritten extents, but instead we zero
272 * the block before we commit the transaction. Ideally we'd like to do
273 * this outside the transaction context, but if we commit and then crash
274 * we may not have zeroed the blocks and this will be exposed on
275 * recovery of the allocation. Hence we must zero before commit.
276 *
277 * Further, if we are mapping unwritten extents here, we need to zero
278 * and convert them to written so that we don't need an unwritten extent
279 * callback for DAX. This also means that we need to be able to dip into
280 * the reserve block pool for bmbt block allocation if there is no space
281 * left but we need to do unwritten extent conversion.
282 */
283 if (flags & IOMAP_DAX) {
284 bmapi_flags = XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO;
285 if (imap->br_state == XFS_EXT_UNWRITTEN) {
286 force = true;
287 nr_exts = XFS_IEXT_WRITE_UNWRITTEN_CNT;
288 dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1;
289 }
290 }
291
292 error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, dblocks,
293 rblocks, force, &tp);
294 if (error)
295 return error;
296
297 error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, nr_exts);
298 if (error == -EFBIG)
299 error = xfs_iext_count_upgrade(tp, ip, nr_exts);
300 if (error)
301 goto out_trans_cancel;
302
303 /*
304 * From this point onwards we overwrite the imap pointer that the
305 * caller gave to us.
306 */
307 nimaps = 1;
308 error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb, bmapi_flags, 0,
309 imap, &nimaps);
310 if (error)
311 goto out_trans_cancel;
312
313 /*
314 * Complete the transaction
315 */
316 error = xfs_trans_commit(tp);
317 if (error)
318 goto out_unlock;
319
320 /*
321 * Copy any maps to caller's array and return any error.
322 */
323 if (nimaps == 0) {
324 error = -ENOSPC;
325 goto out_unlock;
326 }
327
328 if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock)))
329 error = xfs_alert_fsblock_zero(ip, imap);
330
331out_unlock:
332 *seq = xfs_iomap_inode_sequence(ip, 0);
333 xfs_iunlock(ip, XFS_ILOCK_EXCL);
334 return error;
335
336out_trans_cancel:
337 xfs_trans_cancel(tp);
338 goto out_unlock;
339}
340
341STATIC bool
342xfs_quota_need_throttle(
343 struct xfs_inode *ip,
344 xfs_dqtype_t type,
345 xfs_fsblock_t alloc_blocks)
346{
347 struct xfs_dquot *dq = xfs_inode_dquot(ip, type);
348
349 if (!dq || !xfs_this_quota_on(ip->i_mount, type))
350 return false;
351
352 /* no hi watermark, no throttle */
353 if (!dq->q_prealloc_hi_wmark)
354 return false;
355
356 /* under the lo watermark, no throttle */
357 if (dq->q_blk.reserved + alloc_blocks < dq->q_prealloc_lo_wmark)
358 return false;
359
360 return true;
361}
362
363STATIC void
364xfs_quota_calc_throttle(
365 struct xfs_inode *ip,
366 xfs_dqtype_t type,
367 xfs_fsblock_t *qblocks,
368 int *qshift,
369 int64_t *qfreesp)
370{
371 struct xfs_dquot *dq = xfs_inode_dquot(ip, type);
372 int64_t freesp;
373 int shift = 0;
374
375 /* no dq, or over hi wmark, squash the prealloc completely */
376 if (!dq || dq->q_blk.reserved >= dq->q_prealloc_hi_wmark) {
377 *qblocks = 0;
378 *qfreesp = 0;
379 return;
380 }
381
382 freesp = dq->q_prealloc_hi_wmark - dq->q_blk.reserved;
383 if (freesp < dq->q_low_space[XFS_QLOWSP_5_PCNT]) {
384 shift = 2;
385 if (freesp < dq->q_low_space[XFS_QLOWSP_3_PCNT])
386 shift += 2;
387 if (freesp < dq->q_low_space[XFS_QLOWSP_1_PCNT])
388 shift += 2;
389 }
390
391 if (freesp < *qfreesp)
392 *qfreesp = freesp;
393
394 /* only overwrite the throttle values if we are more aggressive */
395 if ((freesp >> shift) < (*qblocks >> *qshift)) {
396 *qblocks = freesp;
397 *qshift = shift;
398 }
399}
400
401/*
402 * If we don't have a user specified preallocation size, dynamically increase
403 * the preallocation size as the size of the file grows. Cap the maximum size
404 * at a single extent or less if the filesystem is near full. The closer the
405 * filesystem is to being full, the smaller the maximum preallocation.
406 */
407STATIC xfs_fsblock_t
408xfs_iomap_prealloc_size(
409 struct xfs_inode *ip,
410 int whichfork,
411 loff_t offset,
412 loff_t count,
413 struct xfs_iext_cursor *icur)
414{
415 struct xfs_iext_cursor ncur = *icur;
416 struct xfs_bmbt_irec prev, got;
417 struct xfs_mount *mp = ip->i_mount;
418 struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork);
419 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
420 int64_t freesp;
421 xfs_fsblock_t qblocks;
422 xfs_fsblock_t alloc_blocks = 0;
423 xfs_extlen_t plen;
424 int shift = 0;
425 int qshift = 0;
426
427 /*
428 * As an exception we don't do any preallocation at all if the file is
429 * smaller than the minimum preallocation and we are using the default
430 * dynamic preallocation scheme, as it is likely this is the only write
431 * to the file that is going to be done.
432 */
433 if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_allocsize_blocks))
434 return 0;
435
436 /*
437 * Use the minimum preallocation size for small files or if we are
438 * writing right after a hole.
439 */
440 if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_dalign) ||
441 !xfs_iext_prev_extent(ifp, &ncur, &prev) ||
442 prev.br_startoff + prev.br_blockcount < offset_fsb)
443 return mp->m_allocsize_blocks;
444
445 /*
446 * Take the size of the preceding data extents as the basis for the
447 * preallocation size. Note that we don't care if the previous extents
448 * are written or not.
449 */
450 plen = prev.br_blockcount;
451 while (xfs_iext_prev_extent(ifp, &ncur, &got)) {
452 if (plen > XFS_MAX_BMBT_EXTLEN / 2 ||
453 isnullstartblock(got.br_startblock) ||
454 got.br_startoff + got.br_blockcount != prev.br_startoff ||
455 got.br_startblock + got.br_blockcount != prev.br_startblock)
456 break;
457 plen += got.br_blockcount;
458 prev = got;
459 }
460
461 /*
462 * If the size of the extents is greater than half the maximum extent
463 * length, then use the current offset as the basis. This ensures that
464 * for large files the preallocation size always extends to
465 * XFS_BMBT_MAX_EXTLEN rather than falling short due to things like stripe
466 * unit/width alignment of real extents.
467 */
468 alloc_blocks = plen * 2;
469 if (alloc_blocks > XFS_MAX_BMBT_EXTLEN)
470 alloc_blocks = XFS_B_TO_FSB(mp, offset);
471 qblocks = alloc_blocks;
472
473 /*
474 * XFS_BMBT_MAX_EXTLEN is not a power of two value but we round the prealloc
475 * down to the nearest power of two value after throttling. To prevent
476 * the round down from unconditionally reducing the maximum supported
477 * prealloc size, we round up first, apply appropriate throttling, round
478 * down and cap the value to XFS_BMBT_MAX_EXTLEN.
479 */
480 alloc_blocks = XFS_FILEOFF_MIN(roundup_pow_of_two(XFS_MAX_BMBT_EXTLEN),
481 alloc_blocks);
482
483 freesp = percpu_counter_read_positive(&mp->m_fdblocks);
484 if (freesp < mp->m_low_space[XFS_LOWSP_5_PCNT]) {
485 shift = 2;
486 if (freesp < mp->m_low_space[XFS_LOWSP_4_PCNT])
487 shift++;
488 if (freesp < mp->m_low_space[XFS_LOWSP_3_PCNT])
489 shift++;
490 if (freesp < mp->m_low_space[XFS_LOWSP_2_PCNT])
491 shift++;
492 if (freesp < mp->m_low_space[XFS_LOWSP_1_PCNT])
493 shift++;
494 }
495
496 /*
497 * Check each quota to cap the prealloc size, provide a shift value to
498 * throttle with and adjust amount of available space.
499 */
500 if (xfs_quota_need_throttle(ip, XFS_DQTYPE_USER, alloc_blocks))
501 xfs_quota_calc_throttle(ip, XFS_DQTYPE_USER, &qblocks, &qshift,
502 &freesp);
503 if (xfs_quota_need_throttle(ip, XFS_DQTYPE_GROUP, alloc_blocks))
504 xfs_quota_calc_throttle(ip, XFS_DQTYPE_GROUP, &qblocks, &qshift,
505 &freesp);
506 if (xfs_quota_need_throttle(ip, XFS_DQTYPE_PROJ, alloc_blocks))
507 xfs_quota_calc_throttle(ip, XFS_DQTYPE_PROJ, &qblocks, &qshift,
508 &freesp);
509
510 /*
511 * The final prealloc size is set to the minimum of free space available
512 * in each of the quotas and the overall filesystem.
513 *
514 * The shift throttle value is set to the maximum value as determined by
515 * the global low free space values and per-quota low free space values.
516 */
517 alloc_blocks = min(alloc_blocks, qblocks);
518 shift = max(shift, qshift);
519
520 if (shift)
521 alloc_blocks >>= shift;
522 /*
523 * rounddown_pow_of_two() returns an undefined result if we pass in
524 * alloc_blocks = 0.
525 */
526 if (alloc_blocks)
527 alloc_blocks = rounddown_pow_of_two(alloc_blocks);
528 if (alloc_blocks > XFS_MAX_BMBT_EXTLEN)
529 alloc_blocks = XFS_MAX_BMBT_EXTLEN;
530
531 /*
532 * If we are still trying to allocate more space than is
533 * available, squash the prealloc hard. This can happen if we
534 * have a large file on a small filesystem and the above
535 * lowspace thresholds are smaller than XFS_BMBT_MAX_EXTLEN.
536 */
537 while (alloc_blocks && alloc_blocks >= freesp)
538 alloc_blocks >>= 4;
539 if (alloc_blocks < mp->m_allocsize_blocks)
540 alloc_blocks = mp->m_allocsize_blocks;
541 trace_xfs_iomap_prealloc_size(ip, alloc_blocks, shift,
542 mp->m_allocsize_blocks);
543 return alloc_blocks;
544}
545
546int
547xfs_iomap_write_unwritten(
548 xfs_inode_t *ip,
549 xfs_off_t offset,
550 xfs_off_t count,
551 bool update_isize)
552{
553 xfs_mount_t *mp = ip->i_mount;
554 xfs_fileoff_t offset_fsb;
555 xfs_filblks_t count_fsb;
556 xfs_filblks_t numblks_fsb;
557 int nimaps;
558 xfs_trans_t *tp;
559 xfs_bmbt_irec_t imap;
560 struct inode *inode = VFS_I(ip);
561 xfs_fsize_t i_size;
562 uint resblks;
563 int error;
564
565 trace_xfs_unwritten_convert(ip, offset, count);
566
567 offset_fsb = XFS_B_TO_FSBT(mp, offset);
568 count_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count);
569 count_fsb = (xfs_filblks_t)(count_fsb - offset_fsb);
570
571 /*
572 * Reserve enough blocks in this transaction for two complete extent
573 * btree splits. We may be converting the middle part of an unwritten
574 * extent and in this case we will insert two new extents in the btree
575 * each of which could cause a full split.
576 *
577 * This reservation amount will be used in the first call to
578 * xfs_bmbt_split() to select an AG with enough space to satisfy the
579 * rest of the operation.
580 */
581 resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1;
582
583 /* Attach dquots so that bmbt splits are accounted correctly. */
584 error = xfs_qm_dqattach(ip);
585 if (error)
586 return error;
587
588 do {
589 /*
590 * Set up a transaction to convert the range of extents
591 * from unwritten to real. Do allocations in a loop until
592 * we have covered the range passed in.
593 *
594 * Note that we can't risk to recursing back into the filesystem
595 * here as we might be asked to write out the same inode that we
596 * complete here and might deadlock on the iolock.
597 */
598 error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks,
599 0, true, &tp);
600 if (error)
601 return error;
602
603 error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK,
604 XFS_IEXT_WRITE_UNWRITTEN_CNT);
605 if (error == -EFBIG)
606 error = xfs_iext_count_upgrade(tp, ip,
607 XFS_IEXT_WRITE_UNWRITTEN_CNT);
608 if (error)
609 goto error_on_bmapi_transaction;
610
611 /*
612 * Modify the unwritten extent state of the buffer.
613 */
614 nimaps = 1;
615 error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb,
616 XFS_BMAPI_CONVERT, resblks, &imap,
617 &nimaps);
618 if (error)
619 goto error_on_bmapi_transaction;
620
621 /*
622 * Log the updated inode size as we go. We have to be careful
623 * to only log it up to the actual write offset if it is
624 * halfway into a block.
625 */
626 i_size = XFS_FSB_TO_B(mp, offset_fsb + count_fsb);
627 if (i_size > offset + count)
628 i_size = offset + count;
629 if (update_isize && i_size > i_size_read(inode))
630 i_size_write(inode, i_size);
631 i_size = xfs_new_eof(ip, i_size);
632 if (i_size) {
633 ip->i_disk_size = i_size;
634 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
635 }
636
637 error = xfs_trans_commit(tp);
638 xfs_iunlock(ip, XFS_ILOCK_EXCL);
639 if (error)
640 return error;
641
642 if (unlikely(!xfs_valid_startblock(ip, imap.br_startblock)))
643 return xfs_alert_fsblock_zero(ip, &imap);
644
645 if ((numblks_fsb = imap.br_blockcount) == 0) {
646 /*
647 * The numblks_fsb value should always get
648 * smaller, otherwise the loop is stuck.
649 */
650 ASSERT(imap.br_blockcount);
651 break;
652 }
653 offset_fsb += numblks_fsb;
654 count_fsb -= numblks_fsb;
655 } while (count_fsb > 0);
656
657 return 0;
658
659error_on_bmapi_transaction:
660 xfs_trans_cancel(tp);
661 xfs_iunlock(ip, XFS_ILOCK_EXCL);
662 return error;
663}
664
665static inline bool
666imap_needs_alloc(
667 struct inode *inode,
668 unsigned flags,
669 struct xfs_bmbt_irec *imap,
670 int nimaps)
671{
672 /* don't allocate blocks when just zeroing */
673 if (flags & IOMAP_ZERO)
674 return false;
675 if (!nimaps ||
676 imap->br_startblock == HOLESTARTBLOCK ||
677 imap->br_startblock == DELAYSTARTBLOCK)
678 return true;
679 /* we convert unwritten extents before copying the data for DAX */
680 if ((flags & IOMAP_DAX) && imap->br_state == XFS_EXT_UNWRITTEN)
681 return true;
682 return false;
683}
684
685static inline bool
686imap_needs_cow(
687 struct xfs_inode *ip,
688 unsigned int flags,
689 struct xfs_bmbt_irec *imap,
690 int nimaps)
691{
692 if (!xfs_is_cow_inode(ip))
693 return false;
694
695 /* when zeroing we don't have to COW holes or unwritten extents */
696 if (flags & IOMAP_ZERO) {
697 if (!nimaps ||
698 imap->br_startblock == HOLESTARTBLOCK ||
699 imap->br_state == XFS_EXT_UNWRITTEN)
700 return false;
701 }
702
703 return true;
704}
705
706static int
707xfs_ilock_for_iomap(
708 struct xfs_inode *ip,
709 unsigned flags,
710 unsigned *lockmode)
711{
712 unsigned int mode = *lockmode;
713 bool is_write = flags & (IOMAP_WRITE | IOMAP_ZERO);
714
715 /*
716 * COW writes may allocate delalloc space or convert unwritten COW
717 * extents, so we need to make sure to take the lock exclusively here.
718 */
719 if (xfs_is_cow_inode(ip) && is_write)
720 mode = XFS_ILOCK_EXCL;
721
722 /*
723 * Extents not yet cached requires exclusive access, don't block. This
724 * is an opencoded xfs_ilock_data_map_shared() call but with
725 * non-blocking behaviour.
726 */
727 if (xfs_need_iread_extents(&ip->i_df)) {
728 if (flags & IOMAP_NOWAIT)
729 return -EAGAIN;
730 mode = XFS_ILOCK_EXCL;
731 }
732
733relock:
734 if (flags & IOMAP_NOWAIT) {
735 if (!xfs_ilock_nowait(ip, mode))
736 return -EAGAIN;
737 } else {
738 xfs_ilock(ip, mode);
739 }
740
741 /*
742 * The reflink iflag could have changed since the earlier unlocked
743 * check, so if we got ILOCK_SHARED for a write and but we're now a
744 * reflink inode we have to switch to ILOCK_EXCL and relock.
745 */
746 if (mode == XFS_ILOCK_SHARED && is_write && xfs_is_cow_inode(ip)) {
747 xfs_iunlock(ip, mode);
748 mode = XFS_ILOCK_EXCL;
749 goto relock;
750 }
751
752 *lockmode = mode;
753 return 0;
754}
755
756/*
757 * Check that the imap we are going to return to the caller spans the entire
758 * range that the caller requested for the IO.
759 */
760static bool
761imap_spans_range(
762 struct xfs_bmbt_irec *imap,
763 xfs_fileoff_t offset_fsb,
764 xfs_fileoff_t end_fsb)
765{
766 if (imap->br_startoff > offset_fsb)
767 return false;
768 if (imap->br_startoff + imap->br_blockcount < end_fsb)
769 return false;
770 return true;
771}
772
773static int
774xfs_direct_write_iomap_begin(
775 struct inode *inode,
776 loff_t offset,
777 loff_t length,
778 unsigned flags,
779 struct iomap *iomap,
780 struct iomap *srcmap)
781{
782 struct xfs_inode *ip = XFS_I(inode);
783 struct xfs_mount *mp = ip->i_mount;
784 struct xfs_bmbt_irec imap, cmap;
785 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
786 xfs_fileoff_t end_fsb = xfs_iomap_end_fsb(mp, offset, length);
787 int nimaps = 1, error = 0;
788 bool shared = false;
789 u16 iomap_flags = 0;
790 unsigned int lockmode = XFS_ILOCK_SHARED;
791 u64 seq;
792
793 ASSERT(flags & (IOMAP_WRITE | IOMAP_ZERO));
794
795 if (xfs_is_shutdown(mp))
796 return -EIO;
797
798 /*
799 * Writes that span EOF might trigger an IO size update on completion,
800 * so consider them to be dirty for the purposes of O_DSYNC even if
801 * there is no other metadata changes pending or have been made here.
802 */
803 if (offset + length > i_size_read(inode))
804 iomap_flags |= IOMAP_F_DIRTY;
805
806 error = xfs_ilock_for_iomap(ip, flags, &lockmode);
807 if (error)
808 return error;
809
810 error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
811 &nimaps, 0);
812 if (error)
813 goto out_unlock;
814
815 if (imap_needs_cow(ip, flags, &imap, nimaps)) {
816 error = -EAGAIN;
817 if (flags & IOMAP_NOWAIT)
818 goto out_unlock;
819
820 /* may drop and re-acquire the ilock */
821 error = xfs_reflink_allocate_cow(ip, &imap, &cmap, &shared,
822 &lockmode,
823 (flags & IOMAP_DIRECT) || IS_DAX(inode));
824 if (error)
825 goto out_unlock;
826 if (shared)
827 goto out_found_cow;
828 end_fsb = imap.br_startoff + imap.br_blockcount;
829 length = XFS_FSB_TO_B(mp, end_fsb) - offset;
830 }
831
832 if (imap_needs_alloc(inode, flags, &imap, nimaps))
833 goto allocate_blocks;
834
835 /*
836 * NOWAIT and OVERWRITE I/O needs to span the entire requested I/O with
837 * a single map so that we avoid partial IO failures due to the rest of
838 * the I/O range not covered by this map triggering an EAGAIN condition
839 * when it is subsequently mapped and aborting the I/O.
840 */
841 if (flags & (IOMAP_NOWAIT | IOMAP_OVERWRITE_ONLY)) {
842 error = -EAGAIN;
843 if (!imap_spans_range(&imap, offset_fsb, end_fsb))
844 goto out_unlock;
845 }
846
847 /*
848 * For overwrite only I/O, we cannot convert unwritten extents without
849 * requiring sub-block zeroing. This can only be done under an
850 * exclusive IOLOCK, hence return -EAGAIN if this is not a written
851 * extent to tell the caller to try again.
852 */
853 if (flags & IOMAP_OVERWRITE_ONLY) {
854 error = -EAGAIN;
855 if (imap.br_state != XFS_EXT_NORM &&
856 ((offset | length) & mp->m_blockmask))
857 goto out_unlock;
858 }
859
860 seq = xfs_iomap_inode_sequence(ip, iomap_flags);
861 xfs_iunlock(ip, lockmode);
862 trace_xfs_iomap_found(ip, offset, length, XFS_DATA_FORK, &imap);
863 return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, iomap_flags, seq);
864
865allocate_blocks:
866 error = -EAGAIN;
867 if (flags & (IOMAP_NOWAIT | IOMAP_OVERWRITE_ONLY))
868 goto out_unlock;
869
870 /*
871 * We cap the maximum length we map to a sane size to keep the chunks
872 * of work done where somewhat symmetric with the work writeback does.
873 * This is a completely arbitrary number pulled out of thin air as a
874 * best guess for initial testing.
875 *
876 * Note that the values needs to be less than 32-bits wide until the
877 * lower level functions are updated.
878 */
879 length = min_t(loff_t, length, 1024 * PAGE_SIZE);
880 end_fsb = xfs_iomap_end_fsb(mp, offset, length);
881
882 if (offset + length > XFS_ISIZE(ip))
883 end_fsb = xfs_iomap_eof_align_last_fsb(ip, end_fsb);
884 else if (nimaps && imap.br_startblock == HOLESTARTBLOCK)
885 end_fsb = min(end_fsb, imap.br_startoff + imap.br_blockcount);
886 xfs_iunlock(ip, lockmode);
887
888 error = xfs_iomap_write_direct(ip, offset_fsb, end_fsb - offset_fsb,
889 flags, &imap, &seq);
890 if (error)
891 return error;
892
893 trace_xfs_iomap_alloc(ip, offset, length, XFS_DATA_FORK, &imap);
894 return xfs_bmbt_to_iomap(ip, iomap, &imap, flags,
895 iomap_flags | IOMAP_F_NEW, seq);
896
897out_found_cow:
898 length = XFS_FSB_TO_B(mp, cmap.br_startoff + cmap.br_blockcount);
899 trace_xfs_iomap_found(ip, offset, length - offset, XFS_COW_FORK, &cmap);
900 if (imap.br_startblock != HOLESTARTBLOCK) {
901 seq = xfs_iomap_inode_sequence(ip, 0);
902 error = xfs_bmbt_to_iomap(ip, srcmap, &imap, flags, 0, seq);
903 if (error)
904 goto out_unlock;
905 }
906 seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED);
907 xfs_iunlock(ip, lockmode);
908 return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, IOMAP_F_SHARED, seq);
909
910out_unlock:
911 if (lockmode)
912 xfs_iunlock(ip, lockmode);
913 return error;
914}
915
916const struct iomap_ops xfs_direct_write_iomap_ops = {
917 .iomap_begin = xfs_direct_write_iomap_begin,
918};
919
920static int
921xfs_dax_write_iomap_end(
922 struct inode *inode,
923 loff_t pos,
924 loff_t length,
925 ssize_t written,
926 unsigned flags,
927 struct iomap *iomap)
928{
929 struct xfs_inode *ip = XFS_I(inode);
930
931 if (!xfs_is_cow_inode(ip))
932 return 0;
933
934 if (!written) {
935 xfs_reflink_cancel_cow_range(ip, pos, length, true);
936 return 0;
937 }
938
939 return xfs_reflink_end_cow(ip, pos, written);
940}
941
942const struct iomap_ops xfs_dax_write_iomap_ops = {
943 .iomap_begin = xfs_direct_write_iomap_begin,
944 .iomap_end = xfs_dax_write_iomap_end,
945};
946
947static int
948xfs_buffered_write_iomap_begin(
949 struct inode *inode,
950 loff_t offset,
951 loff_t count,
952 unsigned flags,
953 struct iomap *iomap,
954 struct iomap *srcmap)
955{
956 struct xfs_inode *ip = XFS_I(inode);
957 struct xfs_mount *mp = ip->i_mount;
958 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
959 xfs_fileoff_t end_fsb = xfs_iomap_end_fsb(mp, offset, count);
960 struct xfs_bmbt_irec imap, cmap;
961 struct xfs_iext_cursor icur, ccur;
962 xfs_fsblock_t prealloc_blocks = 0;
963 bool eof = false, cow_eof = false, shared = false;
964 int allocfork = XFS_DATA_FORK;
965 int error = 0;
966 unsigned int lockmode = XFS_ILOCK_EXCL;
967 u64 seq;
968
969 if (xfs_is_shutdown(mp))
970 return -EIO;
971
972 /* we can't use delayed allocations when using extent size hints */
973 if (xfs_get_extsz_hint(ip))
974 return xfs_direct_write_iomap_begin(inode, offset, count,
975 flags, iomap, srcmap);
976
977 ASSERT(!XFS_IS_REALTIME_INODE(ip));
978
979 error = xfs_qm_dqattach(ip);
980 if (error)
981 return error;
982
983 error = xfs_ilock_for_iomap(ip, flags, &lockmode);
984 if (error)
985 return error;
986
987 if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(&ip->i_df)) ||
988 XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) {
989 error = -EFSCORRUPTED;
990 goto out_unlock;
991 }
992
993 XFS_STATS_INC(mp, xs_blk_mapw);
994
995 error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK);
996 if (error)
997 goto out_unlock;
998
999 /*
1000 * Search the data fork first to look up our source mapping. We
1001 * always need the data fork map, as we have to return it to the
1002 * iomap code so that the higher level write code can read data in to
1003 * perform read-modify-write cycles for unaligned writes.
1004 */
1005 eof = !xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap);
1006 if (eof)
1007 imap.br_startoff = end_fsb; /* fake hole until the end */
1008
1009 /* We never need to allocate blocks for zeroing a hole. */
1010 if ((flags & IOMAP_ZERO) && imap.br_startoff > offset_fsb) {
1011 xfs_hole_to_iomap(ip, iomap, offset_fsb, imap.br_startoff);
1012 goto out_unlock;
1013 }
1014
1015 /*
1016 * Search the COW fork extent list even if we did not find a data fork
1017 * extent. This serves two purposes: first this implements the
1018 * speculative preallocation using cowextsize, so that we also unshare
1019 * block adjacent to shared blocks instead of just the shared blocks
1020 * themselves. Second the lookup in the extent list is generally faster
1021 * than going out to the shared extent tree.
1022 */
1023 if (xfs_is_cow_inode(ip)) {
1024 if (!ip->i_cowfp) {
1025 ASSERT(!xfs_is_reflink_inode(ip));
1026 xfs_ifork_init_cow(ip);
1027 }
1028 cow_eof = !xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb,
1029 &ccur, &cmap);
1030 if (!cow_eof && cmap.br_startoff <= offset_fsb) {
1031 trace_xfs_reflink_cow_found(ip, &cmap);
1032 goto found_cow;
1033 }
1034 }
1035
1036 if (imap.br_startoff <= offset_fsb) {
1037 /*
1038 * For reflink files we may need a delalloc reservation when
1039 * overwriting shared extents. This includes zeroing of
1040 * existing extents that contain data.
1041 */
1042 if (!xfs_is_cow_inode(ip) ||
1043 ((flags & IOMAP_ZERO) && imap.br_state != XFS_EXT_NORM)) {
1044 trace_xfs_iomap_found(ip, offset, count, XFS_DATA_FORK,
1045 &imap);
1046 goto found_imap;
1047 }
1048
1049 xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb);
1050
1051 /* Trim the mapping to the nearest shared extent boundary. */
1052 error = xfs_bmap_trim_cow(ip, &imap, &shared);
1053 if (error)
1054 goto out_unlock;
1055
1056 /* Not shared? Just report the (potentially capped) extent. */
1057 if (!shared) {
1058 trace_xfs_iomap_found(ip, offset, count, XFS_DATA_FORK,
1059 &imap);
1060 goto found_imap;
1061 }
1062
1063 /*
1064 * Fork all the shared blocks from our write offset until the
1065 * end of the extent.
1066 */
1067 allocfork = XFS_COW_FORK;
1068 end_fsb = imap.br_startoff + imap.br_blockcount;
1069 } else {
1070 /*
1071 * We cap the maximum length we map here to MAX_WRITEBACK_PAGES
1072 * pages to keep the chunks of work done where somewhat
1073 * symmetric with the work writeback does. This is a completely
1074 * arbitrary number pulled out of thin air.
1075 *
1076 * Note that the values needs to be less than 32-bits wide until
1077 * the lower level functions are updated.
1078 */
1079 count = min_t(loff_t, count, 1024 * PAGE_SIZE);
1080 end_fsb = xfs_iomap_end_fsb(mp, offset, count);
1081
1082 if (xfs_is_always_cow_inode(ip))
1083 allocfork = XFS_COW_FORK;
1084 }
1085
1086 if (eof && offset + count > XFS_ISIZE(ip)) {
1087 /*
1088 * Determine the initial size of the preallocation.
1089 * We clean up any extra preallocation when the file is closed.
1090 */
1091 if (xfs_has_allocsize(mp))
1092 prealloc_blocks = mp->m_allocsize_blocks;
1093 else
1094 prealloc_blocks = xfs_iomap_prealloc_size(ip, allocfork,
1095 offset, count, &icur);
1096 if (prealloc_blocks) {
1097 xfs_extlen_t align;
1098 xfs_off_t end_offset;
1099 xfs_fileoff_t p_end_fsb;
1100
1101 end_offset = XFS_ALLOC_ALIGN(mp, offset + count - 1);
1102 p_end_fsb = XFS_B_TO_FSBT(mp, end_offset) +
1103 prealloc_blocks;
1104
1105 align = xfs_eof_alignment(ip);
1106 if (align)
1107 p_end_fsb = roundup_64(p_end_fsb, align);
1108
1109 p_end_fsb = min(p_end_fsb,
1110 XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes));
1111 ASSERT(p_end_fsb > offset_fsb);
1112 prealloc_blocks = p_end_fsb - end_fsb;
1113 }
1114 }
1115
1116retry:
1117 error = xfs_bmapi_reserve_delalloc(ip, allocfork, offset_fsb,
1118 end_fsb - offset_fsb, prealloc_blocks,
1119 allocfork == XFS_DATA_FORK ? &imap : &cmap,
1120 allocfork == XFS_DATA_FORK ? &icur : &ccur,
1121 allocfork == XFS_DATA_FORK ? eof : cow_eof);
1122 switch (error) {
1123 case 0:
1124 break;
1125 case -ENOSPC:
1126 case -EDQUOT:
1127 /* retry without any preallocation */
1128 trace_xfs_delalloc_enospc(ip, offset, count);
1129 if (prealloc_blocks) {
1130 prealloc_blocks = 0;
1131 goto retry;
1132 }
1133 fallthrough;
1134 default:
1135 goto out_unlock;
1136 }
1137
1138 if (allocfork == XFS_COW_FORK) {
1139 trace_xfs_iomap_alloc(ip, offset, count, allocfork, &cmap);
1140 goto found_cow;
1141 }
1142
1143 /*
1144 * Flag newly allocated delalloc blocks with IOMAP_F_NEW so we punch
1145 * them out if the write happens to fail.
1146 */
1147 seq = xfs_iomap_inode_sequence(ip, IOMAP_F_NEW);
1148 xfs_iunlock(ip, XFS_ILOCK_EXCL);
1149 trace_xfs_iomap_alloc(ip, offset, count, allocfork, &imap);
1150 return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, IOMAP_F_NEW, seq);
1151
1152found_imap:
1153 seq = xfs_iomap_inode_sequence(ip, 0);
1154 xfs_iunlock(ip, XFS_ILOCK_EXCL);
1155 return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, 0, seq);
1156
1157found_cow:
1158 seq = xfs_iomap_inode_sequence(ip, 0);
1159 if (imap.br_startoff <= offset_fsb) {
1160 error = xfs_bmbt_to_iomap(ip, srcmap, &imap, flags, 0, seq);
1161 if (error)
1162 goto out_unlock;
1163 seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED);
1164 xfs_iunlock(ip, XFS_ILOCK_EXCL);
1165 return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags,
1166 IOMAP_F_SHARED, seq);
1167 }
1168
1169 xfs_trim_extent(&cmap, offset_fsb, imap.br_startoff - offset_fsb);
1170 xfs_iunlock(ip, XFS_ILOCK_EXCL);
1171 return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, 0, seq);
1172
1173out_unlock:
1174 xfs_iunlock(ip, XFS_ILOCK_EXCL);
1175 return error;
1176}
1177
1178static int
1179xfs_buffered_write_delalloc_punch(
1180 struct inode *inode,
1181 loff_t offset,
1182 loff_t length)
1183{
1184 return xfs_bmap_punch_delalloc_range(XFS_I(inode), offset,
1185 offset + length);
1186}
1187
1188static int
1189xfs_buffered_write_iomap_end(
1190 struct inode *inode,
1191 loff_t offset,
1192 loff_t length,
1193 ssize_t written,
1194 unsigned flags,
1195 struct iomap *iomap)
1196{
1197
1198 struct xfs_mount *mp = XFS_M(inode->i_sb);
1199 int error;
1200
1201 error = iomap_file_buffered_write_punch_delalloc(inode, iomap, offset,
1202 length, written, &xfs_buffered_write_delalloc_punch);
1203 if (error && !xfs_is_shutdown(mp)) {
1204 xfs_alert(mp, "%s: unable to clean up ino 0x%llx",
1205 __func__, XFS_I(inode)->i_ino);
1206 return error;
1207 }
1208 return 0;
1209}
1210
1211const struct iomap_ops xfs_buffered_write_iomap_ops = {
1212 .iomap_begin = xfs_buffered_write_iomap_begin,
1213 .iomap_end = xfs_buffered_write_iomap_end,
1214};
1215
1216/*
1217 * iomap_page_mkwrite() will never fail in a way that requires delalloc extents
1218 * that it allocated to be revoked. Hence we do not need an .iomap_end method
1219 * for this operation.
1220 */
1221const struct iomap_ops xfs_page_mkwrite_iomap_ops = {
1222 .iomap_begin = xfs_buffered_write_iomap_begin,
1223};
1224
1225static int
1226xfs_read_iomap_begin(
1227 struct inode *inode,
1228 loff_t offset,
1229 loff_t length,
1230 unsigned flags,
1231 struct iomap *iomap,
1232 struct iomap *srcmap)
1233{
1234 struct xfs_inode *ip = XFS_I(inode);
1235 struct xfs_mount *mp = ip->i_mount;
1236 struct xfs_bmbt_irec imap;
1237 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
1238 xfs_fileoff_t end_fsb = xfs_iomap_end_fsb(mp, offset, length);
1239 int nimaps = 1, error = 0;
1240 bool shared = false;
1241 unsigned int lockmode = XFS_ILOCK_SHARED;
1242 u64 seq;
1243
1244 ASSERT(!(flags & (IOMAP_WRITE | IOMAP_ZERO)));
1245
1246 if (xfs_is_shutdown(mp))
1247 return -EIO;
1248
1249 error = xfs_ilock_for_iomap(ip, flags, &lockmode);
1250 if (error)
1251 return error;
1252 error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
1253 &nimaps, 0);
1254 if (!error && ((flags & IOMAP_REPORT) || IS_DAX(inode)))
1255 error = xfs_reflink_trim_around_shared(ip, &imap, &shared);
1256 seq = xfs_iomap_inode_sequence(ip, shared ? IOMAP_F_SHARED : 0);
1257 xfs_iunlock(ip, lockmode);
1258
1259 if (error)
1260 return error;
1261 trace_xfs_iomap_found(ip, offset, length, XFS_DATA_FORK, &imap);
1262 return xfs_bmbt_to_iomap(ip, iomap, &imap, flags,
1263 shared ? IOMAP_F_SHARED : 0, seq);
1264}
1265
1266const struct iomap_ops xfs_read_iomap_ops = {
1267 .iomap_begin = xfs_read_iomap_begin,
1268};
1269
1270static int
1271xfs_seek_iomap_begin(
1272 struct inode *inode,
1273 loff_t offset,
1274 loff_t length,
1275 unsigned flags,
1276 struct iomap *iomap,
1277 struct iomap *srcmap)
1278{
1279 struct xfs_inode *ip = XFS_I(inode);
1280 struct xfs_mount *mp = ip->i_mount;
1281 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
1282 xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + length);
1283 xfs_fileoff_t cow_fsb = NULLFILEOFF, data_fsb = NULLFILEOFF;
1284 struct xfs_iext_cursor icur;
1285 struct xfs_bmbt_irec imap, cmap;
1286 int error = 0;
1287 unsigned lockmode;
1288 u64 seq;
1289
1290 if (xfs_is_shutdown(mp))
1291 return -EIO;
1292
1293 lockmode = xfs_ilock_data_map_shared(ip);
1294 error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK);
1295 if (error)
1296 goto out_unlock;
1297
1298 if (xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap)) {
1299 /*
1300 * If we found a data extent we are done.
1301 */
1302 if (imap.br_startoff <= offset_fsb)
1303 goto done;
1304 data_fsb = imap.br_startoff;
1305 } else {
1306 /*
1307 * Fake a hole until the end of the file.
1308 */
1309 data_fsb = xfs_iomap_end_fsb(mp, offset, length);
1310 }
1311
1312 /*
1313 * If a COW fork extent covers the hole, report it - capped to the next
1314 * data fork extent:
1315 */
1316 if (xfs_inode_has_cow_data(ip) &&
1317 xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &cmap))
1318 cow_fsb = cmap.br_startoff;
1319 if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) {
1320 if (data_fsb < cow_fsb + cmap.br_blockcount)
1321 end_fsb = min(end_fsb, data_fsb);
1322 xfs_trim_extent(&cmap, offset_fsb, end_fsb);
1323 seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED);
1324 error = xfs_bmbt_to_iomap(ip, iomap, &cmap, flags,
1325 IOMAP_F_SHARED, seq);
1326 /*
1327 * This is a COW extent, so we must probe the page cache
1328 * because there could be dirty page cache being backed
1329 * by this extent.
1330 */
1331 iomap->type = IOMAP_UNWRITTEN;
1332 goto out_unlock;
1333 }
1334
1335 /*
1336 * Else report a hole, capped to the next found data or COW extent.
1337 */
1338 if (cow_fsb != NULLFILEOFF && cow_fsb < data_fsb)
1339 imap.br_blockcount = cow_fsb - offset_fsb;
1340 else
1341 imap.br_blockcount = data_fsb - offset_fsb;
1342 imap.br_startoff = offset_fsb;
1343 imap.br_startblock = HOLESTARTBLOCK;
1344 imap.br_state = XFS_EXT_NORM;
1345done:
1346 seq = xfs_iomap_inode_sequence(ip, 0);
1347 xfs_trim_extent(&imap, offset_fsb, end_fsb);
1348 error = xfs_bmbt_to_iomap(ip, iomap, &imap, flags, 0, seq);
1349out_unlock:
1350 xfs_iunlock(ip, lockmode);
1351 return error;
1352}
1353
1354const struct iomap_ops xfs_seek_iomap_ops = {
1355 .iomap_begin = xfs_seek_iomap_begin,
1356};
1357
1358static int
1359xfs_xattr_iomap_begin(
1360 struct inode *inode,
1361 loff_t offset,
1362 loff_t length,
1363 unsigned flags,
1364 struct iomap *iomap,
1365 struct iomap *srcmap)
1366{
1367 struct xfs_inode *ip = XFS_I(inode);
1368 struct xfs_mount *mp = ip->i_mount;
1369 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
1370 xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + length);
1371 struct xfs_bmbt_irec imap;
1372 int nimaps = 1, error = 0;
1373 unsigned lockmode;
1374 int seq;
1375
1376 if (xfs_is_shutdown(mp))
1377 return -EIO;
1378
1379 lockmode = xfs_ilock_attr_map_shared(ip);
1380
1381 /* if there are no attribute fork or extents, return ENOENT */
1382 if (!xfs_inode_has_attr_fork(ip) || !ip->i_af.if_nextents) {
1383 error = -ENOENT;
1384 goto out_unlock;
1385 }
1386
1387 ASSERT(ip->i_af.if_format != XFS_DINODE_FMT_LOCAL);
1388 error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
1389 &nimaps, XFS_BMAPI_ATTRFORK);
1390out_unlock:
1391
1392 seq = xfs_iomap_inode_sequence(ip, IOMAP_F_XATTR);
1393 xfs_iunlock(ip, lockmode);
1394
1395 if (error)
1396 return error;
1397 ASSERT(nimaps);
1398 return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, IOMAP_F_XATTR, seq);
1399}
1400
1401const struct iomap_ops xfs_xattr_iomap_ops = {
1402 .iomap_begin = xfs_xattr_iomap_begin,
1403};
1404
1405int
1406xfs_zero_range(
1407 struct xfs_inode *ip,
1408 loff_t pos,
1409 loff_t len,
1410 bool *did_zero)
1411{
1412 struct inode *inode = VFS_I(ip);
1413
1414 if (IS_DAX(inode))
1415 return dax_zero_range(inode, pos, len, did_zero,
1416 &xfs_dax_write_iomap_ops);
1417 return iomap_zero_range(inode, pos, len, did_zero,
1418 &xfs_buffered_write_iomap_ops);
1419}
1420
1421int
1422xfs_truncate_page(
1423 struct xfs_inode *ip,
1424 loff_t pos,
1425 bool *did_zero)
1426{
1427 struct inode *inode = VFS_I(ip);
1428
1429 if (IS_DAX(inode))
1430 return dax_truncate_page(inode, pos, did_zero,
1431 &xfs_dax_write_iomap_ops);
1432 return iomap_truncate_page(inode, pos, did_zero,
1433 &xfs_buffered_write_iomap_ops);
1434}
1/*
2 * Copyright (c) 2000-2006 Silicon Graphics, Inc.
3 * All Rights Reserved.
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18#include "xfs.h"
19#include "xfs_fs.h"
20#include "xfs_shared.h"
21#include "xfs_format.h"
22#include "xfs_log_format.h"
23#include "xfs_trans_resv.h"
24#include "xfs_mount.h"
25#include "xfs_inode.h"
26#include "xfs_btree.h"
27#include "xfs_bmap_btree.h"
28#include "xfs_bmap.h"
29#include "xfs_bmap_util.h"
30#include "xfs_error.h"
31#include "xfs_trans.h"
32#include "xfs_trans_space.h"
33#include "xfs_iomap.h"
34#include "xfs_trace.h"
35#include "xfs_icache.h"
36#include "xfs_quota.h"
37#include "xfs_dquot_item.h"
38#include "xfs_dquot.h"
39
40
41#define XFS_WRITEIO_ALIGN(mp,off) (((off) >> mp->m_writeio_log) \
42 << mp->m_writeio_log)
43#define XFS_WRITE_IMAPS XFS_BMAP_MAX_NMAP
44
45STATIC int
46xfs_iomap_eof_align_last_fsb(
47 xfs_mount_t *mp,
48 xfs_inode_t *ip,
49 xfs_extlen_t extsize,
50 xfs_fileoff_t *last_fsb)
51{
52 xfs_extlen_t align = 0;
53 int eof, error;
54
55 if (!XFS_IS_REALTIME_INODE(ip)) {
56 /*
57 * Round up the allocation request to a stripe unit
58 * (m_dalign) boundary if the file size is >= stripe unit
59 * size, and we are allocating past the allocation eof.
60 *
61 * If mounted with the "-o swalloc" option the alignment is
62 * increased from the strip unit size to the stripe width.
63 */
64 if (mp->m_swidth && (mp->m_flags & XFS_MOUNT_SWALLOC))
65 align = mp->m_swidth;
66 else if (mp->m_dalign)
67 align = mp->m_dalign;
68
69 if (align && XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, align))
70 align = 0;
71 }
72
73 /*
74 * Always round up the allocation request to an extent boundary
75 * (when file on a real-time subvolume or has di_extsize hint).
76 */
77 if (extsize) {
78 if (align)
79 align = roundup_64(align, extsize);
80 else
81 align = extsize;
82 }
83
84 if (align) {
85 xfs_fileoff_t new_last_fsb = roundup_64(*last_fsb, align);
86 error = xfs_bmap_eof(ip, new_last_fsb, XFS_DATA_FORK, &eof);
87 if (error)
88 return error;
89 if (eof)
90 *last_fsb = new_last_fsb;
91 }
92 return 0;
93}
94
95STATIC int
96xfs_alert_fsblock_zero(
97 xfs_inode_t *ip,
98 xfs_bmbt_irec_t *imap)
99{
100 xfs_alert_tag(ip->i_mount, XFS_PTAG_FSBLOCK_ZERO,
101 "Access to block zero in inode %llu "
102 "start_block: %llx start_off: %llx "
103 "blkcnt: %llx extent-state: %x",
104 (unsigned long long)ip->i_ino,
105 (unsigned long long)imap->br_startblock,
106 (unsigned long long)imap->br_startoff,
107 (unsigned long long)imap->br_blockcount,
108 imap->br_state);
109 return -EFSCORRUPTED;
110}
111
112int
113xfs_iomap_write_direct(
114 xfs_inode_t *ip,
115 xfs_off_t offset,
116 size_t count,
117 xfs_bmbt_irec_t *imap,
118 int nmaps)
119{
120 xfs_mount_t *mp = ip->i_mount;
121 xfs_fileoff_t offset_fsb;
122 xfs_fileoff_t last_fsb;
123 xfs_filblks_t count_fsb, resaligned;
124 xfs_fsblock_t firstfsb;
125 xfs_extlen_t extsz, temp;
126 int nimaps;
127 int quota_flag;
128 int rt;
129 xfs_trans_t *tp;
130 xfs_bmap_free_t free_list;
131 uint qblocks, resblks, resrtextents;
132 int error;
133 int lockmode;
134 int bmapi_flags = XFS_BMAPI_PREALLOC;
135
136 rt = XFS_IS_REALTIME_INODE(ip);
137 extsz = xfs_get_extsz_hint(ip);
138 lockmode = XFS_ILOCK_SHARED; /* locked by caller */
139
140 ASSERT(xfs_isilocked(ip, lockmode));
141
142 offset_fsb = XFS_B_TO_FSBT(mp, offset);
143 last_fsb = XFS_B_TO_FSB(mp, ((xfs_ufsize_t)(offset + count)));
144 if ((offset + count) > XFS_ISIZE(ip)) {
145 /*
146 * Assert that the in-core extent list is present since this can
147 * call xfs_iread_extents() and we only have the ilock shared.
148 * This should be safe because the lock was held around a bmapi
149 * call in the caller and we only need it to access the in-core
150 * list.
151 */
152 ASSERT(XFS_IFORK_PTR(ip, XFS_DATA_FORK)->if_flags &
153 XFS_IFEXTENTS);
154 error = xfs_iomap_eof_align_last_fsb(mp, ip, extsz, &last_fsb);
155 if (error)
156 goto out_unlock;
157 } else {
158 if (nmaps && (imap->br_startblock == HOLESTARTBLOCK))
159 last_fsb = MIN(last_fsb, (xfs_fileoff_t)
160 imap->br_blockcount +
161 imap->br_startoff);
162 }
163 count_fsb = last_fsb - offset_fsb;
164 ASSERT(count_fsb > 0);
165
166 resaligned = count_fsb;
167 if (unlikely(extsz)) {
168 if ((temp = do_mod(offset_fsb, extsz)))
169 resaligned += temp;
170 if ((temp = do_mod(resaligned, extsz)))
171 resaligned += extsz - temp;
172 }
173
174 if (unlikely(rt)) {
175 resrtextents = qblocks = resaligned;
176 resrtextents /= mp->m_sb.sb_rextsize;
177 resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0);
178 quota_flag = XFS_QMOPT_RES_RTBLKS;
179 } else {
180 resrtextents = 0;
181 resblks = qblocks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned);
182 quota_flag = XFS_QMOPT_RES_REGBLKS;
183 }
184
185 /*
186 * Drop the shared lock acquired by the caller, attach the dquot if
187 * necessary and move on to transaction setup.
188 */
189 xfs_iunlock(ip, lockmode);
190 error = xfs_qm_dqattach(ip, 0);
191 if (error)
192 return error;
193
194 /*
195 * Allocate and setup the transaction
196 */
197 tp = xfs_trans_alloc(mp, XFS_TRANS_DIOSTRAT);
198
199 /*
200 * For DAX, we do not allocate unwritten extents, but instead we zero
201 * the block before we commit the transaction. Ideally we'd like to do
202 * this outside the transaction context, but if we commit and then crash
203 * we may not have zeroed the blocks and this will be exposed on
204 * recovery of the allocation. Hence we must zero before commit.
205 *
206 * Further, if we are mapping unwritten extents here, we need to zero
207 * and convert them to written so that we don't need an unwritten extent
208 * callback for DAX. This also means that we need to be able to dip into
209 * the reserve block pool for bmbt block allocation if there is no space
210 * left but we need to do unwritten extent conversion.
211 */
212
213 if (IS_DAX(VFS_I(ip))) {
214 bmapi_flags = XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO;
215 if (ISUNWRITTEN(imap)) {
216 tp->t_flags |= XFS_TRANS_RESERVE;
217 resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1;
218 }
219 }
220 error = xfs_trans_reserve(tp, &M_RES(mp)->tr_write,
221 resblks, resrtextents);
222 /*
223 * Check for running out of space, note: need lock to return
224 */
225 if (error) {
226 xfs_trans_cancel(tp);
227 return error;
228 }
229
230 lockmode = XFS_ILOCK_EXCL;
231 xfs_ilock(ip, lockmode);
232
233 error = xfs_trans_reserve_quota_nblks(tp, ip, qblocks, 0, quota_flag);
234 if (error)
235 goto out_trans_cancel;
236
237 xfs_trans_ijoin(tp, ip, 0);
238
239 /*
240 * From this point onwards we overwrite the imap pointer that the
241 * caller gave to us.
242 */
243 xfs_bmap_init(&free_list, &firstfsb);
244 nimaps = 1;
245 error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb,
246 bmapi_flags, &firstfsb, resblks, imap,
247 &nimaps, &free_list);
248 if (error)
249 goto out_bmap_cancel;
250
251 /*
252 * Complete the transaction
253 */
254 error = xfs_bmap_finish(&tp, &free_list, NULL);
255 if (error)
256 goto out_bmap_cancel;
257
258 error = xfs_trans_commit(tp);
259 if (error)
260 goto out_unlock;
261
262 /*
263 * Copy any maps to caller's array and return any error.
264 */
265 if (nimaps == 0) {
266 error = -ENOSPC;
267 goto out_unlock;
268 }
269
270 if (!(imap->br_startblock || XFS_IS_REALTIME_INODE(ip)))
271 error = xfs_alert_fsblock_zero(ip, imap);
272
273out_unlock:
274 xfs_iunlock(ip, lockmode);
275 return error;
276
277out_bmap_cancel:
278 xfs_bmap_cancel(&free_list);
279 xfs_trans_unreserve_quota_nblks(tp, ip, (long)qblocks, 0, quota_flag);
280out_trans_cancel:
281 xfs_trans_cancel(tp);
282 goto out_unlock;
283}
284
285/*
286 * If the caller is doing a write at the end of the file, then extend the
287 * allocation out to the file system's write iosize. We clean up any extra
288 * space left over when the file is closed in xfs_inactive().
289 *
290 * If we find we already have delalloc preallocation beyond EOF, don't do more
291 * preallocation as it it not needed.
292 */
293STATIC int
294xfs_iomap_eof_want_preallocate(
295 xfs_mount_t *mp,
296 xfs_inode_t *ip,
297 xfs_off_t offset,
298 size_t count,
299 xfs_bmbt_irec_t *imap,
300 int nimaps,
301 int *prealloc)
302{
303 xfs_fileoff_t start_fsb;
304 xfs_filblks_t count_fsb;
305 int n, error, imaps;
306 int found_delalloc = 0;
307
308 *prealloc = 0;
309 if (offset + count <= XFS_ISIZE(ip))
310 return 0;
311
312 /*
313 * If the file is smaller than the minimum prealloc and we are using
314 * dynamic preallocation, don't do any preallocation at all as it is
315 * likely this is the only write to the file that is going to be done.
316 */
317 if (!(mp->m_flags & XFS_MOUNT_DFLT_IOSIZE) &&
318 XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_writeio_blocks))
319 return 0;
320
321 /*
322 * If there are any real blocks past eof, then don't
323 * do any speculative allocation.
324 */
325 start_fsb = XFS_B_TO_FSBT(mp, ((xfs_ufsize_t)(offset + count - 1)));
326 count_fsb = XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes);
327 while (count_fsb > 0) {
328 imaps = nimaps;
329 error = xfs_bmapi_read(ip, start_fsb, count_fsb, imap, &imaps,
330 0);
331 if (error)
332 return error;
333 for (n = 0; n < imaps; n++) {
334 if ((imap[n].br_startblock != HOLESTARTBLOCK) &&
335 (imap[n].br_startblock != DELAYSTARTBLOCK))
336 return 0;
337 start_fsb += imap[n].br_blockcount;
338 count_fsb -= imap[n].br_blockcount;
339
340 if (imap[n].br_startblock == DELAYSTARTBLOCK)
341 found_delalloc = 1;
342 }
343 }
344 if (!found_delalloc)
345 *prealloc = 1;
346 return 0;
347}
348
349/*
350 * Determine the initial size of the preallocation. We are beyond the current
351 * EOF here, but we need to take into account whether this is a sparse write or
352 * an extending write when determining the preallocation size. Hence we need to
353 * look up the extent that ends at the current write offset and use the result
354 * to determine the preallocation size.
355 *
356 * If the extent is a hole, then preallocation is essentially disabled.
357 * Otherwise we take the size of the preceeding data extent as the basis for the
358 * preallocation size. If the size of the extent is greater than half the
359 * maximum extent length, then use the current offset as the basis. This ensures
360 * that for large files the preallocation size always extends to MAXEXTLEN
361 * rather than falling short due to things like stripe unit/width alignment of
362 * real extents.
363 */
364STATIC xfs_fsblock_t
365xfs_iomap_eof_prealloc_initial_size(
366 struct xfs_mount *mp,
367 struct xfs_inode *ip,
368 xfs_off_t offset,
369 xfs_bmbt_irec_t *imap,
370 int nimaps)
371{
372 xfs_fileoff_t start_fsb;
373 int imaps = 1;
374 int error;
375
376 ASSERT(nimaps >= imaps);
377
378 /* if we are using a specific prealloc size, return now */
379 if (mp->m_flags & XFS_MOUNT_DFLT_IOSIZE)
380 return 0;
381
382 /* If the file is small, then use the minimum prealloc */
383 if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_dalign))
384 return 0;
385
386 /*
387 * As we write multiple pages, the offset will always align to the
388 * start of a page and hence point to a hole at EOF. i.e. if the size is
389 * 4096 bytes, we only have one block at FSB 0, but XFS_B_TO_FSB(4096)
390 * will return FSB 1. Hence if there are blocks in the file, we want to
391 * point to the block prior to the EOF block and not the hole that maps
392 * directly at @offset.
393 */
394 start_fsb = XFS_B_TO_FSB(mp, offset);
395 if (start_fsb)
396 start_fsb--;
397 error = xfs_bmapi_read(ip, start_fsb, 1, imap, &imaps, XFS_BMAPI_ENTIRE);
398 if (error)
399 return 0;
400
401 ASSERT(imaps == 1);
402 if (imap[0].br_startblock == HOLESTARTBLOCK)
403 return 0;
404 if (imap[0].br_blockcount <= (MAXEXTLEN >> 1))
405 return imap[0].br_blockcount << 1;
406 return XFS_B_TO_FSB(mp, offset);
407}
408
409STATIC bool
410xfs_quota_need_throttle(
411 struct xfs_inode *ip,
412 int type,
413 xfs_fsblock_t alloc_blocks)
414{
415 struct xfs_dquot *dq = xfs_inode_dquot(ip, type);
416
417 if (!dq || !xfs_this_quota_on(ip->i_mount, type))
418 return false;
419
420 /* no hi watermark, no throttle */
421 if (!dq->q_prealloc_hi_wmark)
422 return false;
423
424 /* under the lo watermark, no throttle */
425 if (dq->q_res_bcount + alloc_blocks < dq->q_prealloc_lo_wmark)
426 return false;
427
428 return true;
429}
430
431STATIC void
432xfs_quota_calc_throttle(
433 struct xfs_inode *ip,
434 int type,
435 xfs_fsblock_t *qblocks,
436 int *qshift,
437 int64_t *qfreesp)
438{
439 int64_t freesp;
440 int shift = 0;
441 struct xfs_dquot *dq = xfs_inode_dquot(ip, type);
442
443 /* no dq, or over hi wmark, squash the prealloc completely */
444 if (!dq || dq->q_res_bcount >= dq->q_prealloc_hi_wmark) {
445 *qblocks = 0;
446 *qfreesp = 0;
447 return;
448 }
449
450 freesp = dq->q_prealloc_hi_wmark - dq->q_res_bcount;
451 if (freesp < dq->q_low_space[XFS_QLOWSP_5_PCNT]) {
452 shift = 2;
453 if (freesp < dq->q_low_space[XFS_QLOWSP_3_PCNT])
454 shift += 2;
455 if (freesp < dq->q_low_space[XFS_QLOWSP_1_PCNT])
456 shift += 2;
457 }
458
459 if (freesp < *qfreesp)
460 *qfreesp = freesp;
461
462 /* only overwrite the throttle values if we are more aggressive */
463 if ((freesp >> shift) < (*qblocks >> *qshift)) {
464 *qblocks = freesp;
465 *qshift = shift;
466 }
467}
468
469/*
470 * If we don't have a user specified preallocation size, dynamically increase
471 * the preallocation size as the size of the file grows. Cap the maximum size
472 * at a single extent or less if the filesystem is near full. The closer the
473 * filesystem is to full, the smaller the maximum prealocation.
474 */
475STATIC xfs_fsblock_t
476xfs_iomap_prealloc_size(
477 struct xfs_mount *mp,
478 struct xfs_inode *ip,
479 xfs_off_t offset,
480 struct xfs_bmbt_irec *imap,
481 int nimaps)
482{
483 xfs_fsblock_t alloc_blocks = 0;
484 int shift = 0;
485 int64_t freesp;
486 xfs_fsblock_t qblocks;
487 int qshift = 0;
488
489 alloc_blocks = xfs_iomap_eof_prealloc_initial_size(mp, ip, offset,
490 imap, nimaps);
491 if (!alloc_blocks)
492 goto check_writeio;
493 qblocks = alloc_blocks;
494
495 /*
496 * MAXEXTLEN is not a power of two value but we round the prealloc down
497 * to the nearest power of two value after throttling. To prevent the
498 * round down from unconditionally reducing the maximum supported prealloc
499 * size, we round up first, apply appropriate throttling, round down and
500 * cap the value to MAXEXTLEN.
501 */
502 alloc_blocks = XFS_FILEOFF_MIN(roundup_pow_of_two(MAXEXTLEN),
503 alloc_blocks);
504
505 freesp = percpu_counter_read_positive(&mp->m_fdblocks);
506 if (freesp < mp->m_low_space[XFS_LOWSP_5_PCNT]) {
507 shift = 2;
508 if (freesp < mp->m_low_space[XFS_LOWSP_4_PCNT])
509 shift++;
510 if (freesp < mp->m_low_space[XFS_LOWSP_3_PCNT])
511 shift++;
512 if (freesp < mp->m_low_space[XFS_LOWSP_2_PCNT])
513 shift++;
514 if (freesp < mp->m_low_space[XFS_LOWSP_1_PCNT])
515 shift++;
516 }
517
518 /*
519 * Check each quota to cap the prealloc size, provide a shift value to
520 * throttle with and adjust amount of available space.
521 */
522 if (xfs_quota_need_throttle(ip, XFS_DQ_USER, alloc_blocks))
523 xfs_quota_calc_throttle(ip, XFS_DQ_USER, &qblocks, &qshift,
524 &freesp);
525 if (xfs_quota_need_throttle(ip, XFS_DQ_GROUP, alloc_blocks))
526 xfs_quota_calc_throttle(ip, XFS_DQ_GROUP, &qblocks, &qshift,
527 &freesp);
528 if (xfs_quota_need_throttle(ip, XFS_DQ_PROJ, alloc_blocks))
529 xfs_quota_calc_throttle(ip, XFS_DQ_PROJ, &qblocks, &qshift,
530 &freesp);
531
532 /*
533 * The final prealloc size is set to the minimum of free space available
534 * in each of the quotas and the overall filesystem.
535 *
536 * The shift throttle value is set to the maximum value as determined by
537 * the global low free space values and per-quota low free space values.
538 */
539 alloc_blocks = MIN(alloc_blocks, qblocks);
540 shift = MAX(shift, qshift);
541
542 if (shift)
543 alloc_blocks >>= shift;
544 /*
545 * rounddown_pow_of_two() returns an undefined result if we pass in
546 * alloc_blocks = 0.
547 */
548 if (alloc_blocks)
549 alloc_blocks = rounddown_pow_of_two(alloc_blocks);
550 if (alloc_blocks > MAXEXTLEN)
551 alloc_blocks = MAXEXTLEN;
552
553 /*
554 * If we are still trying to allocate more space than is
555 * available, squash the prealloc hard. This can happen if we
556 * have a large file on a small filesystem and the above
557 * lowspace thresholds are smaller than MAXEXTLEN.
558 */
559 while (alloc_blocks && alloc_blocks >= freesp)
560 alloc_blocks >>= 4;
561
562check_writeio:
563 if (alloc_blocks < mp->m_writeio_blocks)
564 alloc_blocks = mp->m_writeio_blocks;
565
566 trace_xfs_iomap_prealloc_size(ip, alloc_blocks, shift,
567 mp->m_writeio_blocks);
568
569 return alloc_blocks;
570}
571
572int
573xfs_iomap_write_delay(
574 xfs_inode_t *ip,
575 xfs_off_t offset,
576 size_t count,
577 xfs_bmbt_irec_t *ret_imap)
578{
579 xfs_mount_t *mp = ip->i_mount;
580 xfs_fileoff_t offset_fsb;
581 xfs_fileoff_t last_fsb;
582 xfs_off_t aligned_offset;
583 xfs_fileoff_t ioalign;
584 xfs_extlen_t extsz;
585 int nimaps;
586 xfs_bmbt_irec_t imap[XFS_WRITE_IMAPS];
587 int prealloc;
588 int error;
589
590 ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL));
591
592 /*
593 * Make sure that the dquots are there. This doesn't hold
594 * the ilock across a disk read.
595 */
596 error = xfs_qm_dqattach_locked(ip, 0);
597 if (error)
598 return error;
599
600 extsz = xfs_get_extsz_hint(ip);
601 offset_fsb = XFS_B_TO_FSBT(mp, offset);
602
603 error = xfs_iomap_eof_want_preallocate(mp, ip, offset, count,
604 imap, XFS_WRITE_IMAPS, &prealloc);
605 if (error)
606 return error;
607
608retry:
609 if (prealloc) {
610 xfs_fsblock_t alloc_blocks;
611
612 alloc_blocks = xfs_iomap_prealloc_size(mp, ip, offset, imap,
613 XFS_WRITE_IMAPS);
614
615 aligned_offset = XFS_WRITEIO_ALIGN(mp, (offset + count - 1));
616 ioalign = XFS_B_TO_FSBT(mp, aligned_offset);
617 last_fsb = ioalign + alloc_blocks;
618 } else {
619 last_fsb = XFS_B_TO_FSB(mp, ((xfs_ufsize_t)(offset + count)));
620 }
621
622 if (prealloc || extsz) {
623 error = xfs_iomap_eof_align_last_fsb(mp, ip, extsz, &last_fsb);
624 if (error)
625 return error;
626 }
627
628 /*
629 * Make sure preallocation does not create extents beyond the range we
630 * actually support in this filesystem.
631 */
632 if (last_fsb > XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes))
633 last_fsb = XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes);
634
635 ASSERT(last_fsb > offset_fsb);
636
637 nimaps = XFS_WRITE_IMAPS;
638 error = xfs_bmapi_delay(ip, offset_fsb, last_fsb - offset_fsb,
639 imap, &nimaps, XFS_BMAPI_ENTIRE);
640 switch (error) {
641 case 0:
642 case -ENOSPC:
643 case -EDQUOT:
644 break;
645 default:
646 return error;
647 }
648
649 /*
650 * If bmapi returned us nothing, we got either ENOSPC or EDQUOT. Retry
651 * without EOF preallocation.
652 */
653 if (nimaps == 0) {
654 trace_xfs_delalloc_enospc(ip, offset, count);
655 if (prealloc) {
656 prealloc = 0;
657 error = 0;
658 goto retry;
659 }
660 return error ? error : -ENOSPC;
661 }
662
663 if (!(imap[0].br_startblock || XFS_IS_REALTIME_INODE(ip)))
664 return xfs_alert_fsblock_zero(ip, &imap[0]);
665
666 /*
667 * Tag the inode as speculatively preallocated so we can reclaim this
668 * space on demand, if necessary.
669 */
670 if (prealloc)
671 xfs_inode_set_eofblocks_tag(ip);
672
673 *ret_imap = imap[0];
674 return 0;
675}
676
677/*
678 * Pass in a delayed allocate extent, convert it to real extents;
679 * return to the caller the extent we create which maps on top of
680 * the originating callers request.
681 *
682 * Called without a lock on the inode.
683 *
684 * We no longer bother to look at the incoming map - all we have to
685 * guarantee is that whatever we allocate fills the required range.
686 */
687int
688xfs_iomap_write_allocate(
689 xfs_inode_t *ip,
690 xfs_off_t offset,
691 xfs_bmbt_irec_t *imap)
692{
693 xfs_mount_t *mp = ip->i_mount;
694 xfs_fileoff_t offset_fsb, last_block;
695 xfs_fileoff_t end_fsb, map_start_fsb;
696 xfs_fsblock_t first_block;
697 xfs_bmap_free_t free_list;
698 xfs_filblks_t count_fsb;
699 xfs_trans_t *tp;
700 int nimaps;
701 int error = 0;
702 int nres;
703
704 /*
705 * Make sure that the dquots are there.
706 */
707 error = xfs_qm_dqattach(ip, 0);
708 if (error)
709 return error;
710
711 offset_fsb = XFS_B_TO_FSBT(mp, offset);
712 count_fsb = imap->br_blockcount;
713 map_start_fsb = imap->br_startoff;
714
715 XFS_STATS_ADD(mp, xs_xstrat_bytes, XFS_FSB_TO_B(mp, count_fsb));
716
717 while (count_fsb != 0) {
718 /*
719 * Set up a transaction with which to allocate the
720 * backing store for the file. Do allocations in a
721 * loop until we get some space in the range we are
722 * interested in. The other space that might be allocated
723 * is in the delayed allocation extent on which we sit
724 * but before our buffer starts.
725 */
726
727 nimaps = 0;
728 while (nimaps == 0) {
729 tp = xfs_trans_alloc(mp, XFS_TRANS_STRAT_WRITE);
730 tp->t_flags |= XFS_TRANS_RESERVE;
731 nres = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK);
732 error = xfs_trans_reserve(tp, &M_RES(mp)->tr_write,
733 nres, 0);
734 if (error) {
735 xfs_trans_cancel(tp);
736 return error;
737 }
738 xfs_ilock(ip, XFS_ILOCK_EXCL);
739 xfs_trans_ijoin(tp, ip, 0);
740
741 xfs_bmap_init(&free_list, &first_block);
742
743 /*
744 * it is possible that the extents have changed since
745 * we did the read call as we dropped the ilock for a
746 * while. We have to be careful about truncates or hole
747 * punchs here - we are not allowed to allocate
748 * non-delalloc blocks here.
749 *
750 * The only protection against truncation is the pages
751 * for the range we are being asked to convert are
752 * locked and hence a truncate will block on them
753 * first.
754 *
755 * As a result, if we go beyond the range we really
756 * need and hit an delalloc extent boundary followed by
757 * a hole while we have excess blocks in the map, we
758 * will fill the hole incorrectly and overrun the
759 * transaction reservation.
760 *
761 * Using a single map prevents this as we are forced to
762 * check each map we look for overlap with the desired
763 * range and abort as soon as we find it. Also, given
764 * that we only return a single map, having one beyond
765 * what we can return is probably a bit silly.
766 *
767 * We also need to check that we don't go beyond EOF;
768 * this is a truncate optimisation as a truncate sets
769 * the new file size before block on the pages we
770 * currently have locked under writeback. Because they
771 * are about to be tossed, we don't need to write them
772 * back....
773 */
774 nimaps = 1;
775 end_fsb = XFS_B_TO_FSB(mp, XFS_ISIZE(ip));
776 error = xfs_bmap_last_offset(ip, &last_block,
777 XFS_DATA_FORK);
778 if (error)
779 goto trans_cancel;
780
781 last_block = XFS_FILEOFF_MAX(last_block, end_fsb);
782 if ((map_start_fsb + count_fsb) > last_block) {
783 count_fsb = last_block - map_start_fsb;
784 if (count_fsb == 0) {
785 error = -EAGAIN;
786 goto trans_cancel;
787 }
788 }
789
790 /*
791 * From this point onwards we overwrite the imap
792 * pointer that the caller gave to us.
793 */
794 error = xfs_bmapi_write(tp, ip, map_start_fsb,
795 count_fsb, 0, &first_block,
796 nres, imap, &nimaps,
797 &free_list);
798 if (error)
799 goto trans_cancel;
800
801 error = xfs_bmap_finish(&tp, &free_list, NULL);
802 if (error)
803 goto trans_cancel;
804
805 error = xfs_trans_commit(tp);
806 if (error)
807 goto error0;
808
809 xfs_iunlock(ip, XFS_ILOCK_EXCL);
810 }
811
812 /*
813 * See if we were able to allocate an extent that
814 * covers at least part of the callers request
815 */
816 if (!(imap->br_startblock || XFS_IS_REALTIME_INODE(ip)))
817 return xfs_alert_fsblock_zero(ip, imap);
818
819 if ((offset_fsb >= imap->br_startoff) &&
820 (offset_fsb < (imap->br_startoff +
821 imap->br_blockcount))) {
822 XFS_STATS_INC(mp, xs_xstrat_quick);
823 return 0;
824 }
825
826 /*
827 * So far we have not mapped the requested part of the
828 * file, just surrounding data, try again.
829 */
830 count_fsb -= imap->br_blockcount;
831 map_start_fsb = imap->br_startoff + imap->br_blockcount;
832 }
833
834trans_cancel:
835 xfs_bmap_cancel(&free_list);
836 xfs_trans_cancel(tp);
837error0:
838 xfs_iunlock(ip, XFS_ILOCK_EXCL);
839 return error;
840}
841
842int
843xfs_iomap_write_unwritten(
844 xfs_inode_t *ip,
845 xfs_off_t offset,
846 xfs_off_t count)
847{
848 xfs_mount_t *mp = ip->i_mount;
849 xfs_fileoff_t offset_fsb;
850 xfs_filblks_t count_fsb;
851 xfs_filblks_t numblks_fsb;
852 xfs_fsblock_t firstfsb;
853 int nimaps;
854 xfs_trans_t *tp;
855 xfs_bmbt_irec_t imap;
856 xfs_bmap_free_t free_list;
857 xfs_fsize_t i_size;
858 uint resblks;
859 int error;
860
861 trace_xfs_unwritten_convert(ip, offset, count);
862
863 offset_fsb = XFS_B_TO_FSBT(mp, offset);
864 count_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count);
865 count_fsb = (xfs_filblks_t)(count_fsb - offset_fsb);
866
867 /*
868 * Reserve enough blocks in this transaction for two complete extent
869 * btree splits. We may be converting the middle part of an unwritten
870 * extent and in this case we will insert two new extents in the btree
871 * each of which could cause a full split.
872 *
873 * This reservation amount will be used in the first call to
874 * xfs_bmbt_split() to select an AG with enough space to satisfy the
875 * rest of the operation.
876 */
877 resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1;
878
879 do {
880 /*
881 * set up a transaction to convert the range of extents
882 * from unwritten to real. Do allocations in a loop until
883 * we have covered the range passed in.
884 *
885 * Note that we open code the transaction allocation here
886 * to pass KM_NOFS--we can't risk to recursing back into
887 * the filesystem here as we might be asked to write out
888 * the same inode that we complete here and might deadlock
889 * on the iolock.
890 */
891 sb_start_intwrite(mp->m_super);
892 tp = _xfs_trans_alloc(mp, XFS_TRANS_STRAT_WRITE, KM_NOFS);
893 tp->t_flags |= XFS_TRANS_RESERVE | XFS_TRANS_FREEZE_PROT;
894 error = xfs_trans_reserve(tp, &M_RES(mp)->tr_write,
895 resblks, 0);
896 if (error) {
897 xfs_trans_cancel(tp);
898 return error;
899 }
900
901 xfs_ilock(ip, XFS_ILOCK_EXCL);
902 xfs_trans_ijoin(tp, ip, 0);
903
904 /*
905 * Modify the unwritten extent state of the buffer.
906 */
907 xfs_bmap_init(&free_list, &firstfsb);
908 nimaps = 1;
909 error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb,
910 XFS_BMAPI_CONVERT, &firstfsb, resblks,
911 &imap, &nimaps, &free_list);
912 if (error)
913 goto error_on_bmapi_transaction;
914
915 /*
916 * Log the updated inode size as we go. We have to be careful
917 * to only log it up to the actual write offset if it is
918 * halfway into a block.
919 */
920 i_size = XFS_FSB_TO_B(mp, offset_fsb + count_fsb);
921 if (i_size > offset + count)
922 i_size = offset + count;
923
924 i_size = xfs_new_eof(ip, i_size);
925 if (i_size) {
926 ip->i_d.di_size = i_size;
927 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
928 }
929
930 error = xfs_bmap_finish(&tp, &free_list, NULL);
931 if (error)
932 goto error_on_bmapi_transaction;
933
934 error = xfs_trans_commit(tp);
935 xfs_iunlock(ip, XFS_ILOCK_EXCL);
936 if (error)
937 return error;
938
939 if (!(imap.br_startblock || XFS_IS_REALTIME_INODE(ip)))
940 return xfs_alert_fsblock_zero(ip, &imap);
941
942 if ((numblks_fsb = imap.br_blockcount) == 0) {
943 /*
944 * The numblks_fsb value should always get
945 * smaller, otherwise the loop is stuck.
946 */
947 ASSERT(imap.br_blockcount);
948 break;
949 }
950 offset_fsb += numblks_fsb;
951 count_fsb -= numblks_fsb;
952 } while (count_fsb > 0);
953
954 return 0;
955
956error_on_bmapi_transaction:
957 xfs_bmap_cancel(&free_list);
958 xfs_trans_cancel(tp);
959 xfs_iunlock(ip, XFS_ILOCK_EXCL);
960 return error;
961}