Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: GPL-2.0-only
2/* Network filesystem high-level write support.
3 *
4 * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8#include <linux/export.h>
9#include <linux/fs.h>
10#include <linux/mm.h>
11#include <linux/pagemap.h>
12#include <linux/slab.h>
13#include <linux/pagevec.h>
14#include "internal.h"
15
16/*
17 * Determined write method. Adjust netfs_folio_traces if this is changed.
18 */
19enum netfs_how_to_modify {
20 NETFS_FOLIO_IS_UPTODATE, /* Folio is uptodate already */
21 NETFS_JUST_PREFETCH, /* We have to read the folio anyway */
22 NETFS_WHOLE_FOLIO_MODIFY, /* We're going to overwrite the whole folio */
23 NETFS_MODIFY_AND_CLEAR, /* We can assume there is no data to be downloaded. */
24 NETFS_STREAMING_WRITE, /* Store incomplete data in non-uptodate page. */
25 NETFS_STREAMING_WRITE_CONT, /* Continue streaming write. */
26 NETFS_FLUSH_CONTENT, /* Flush incompatible content. */
27};
28
29static void netfs_cleanup_buffered_write(struct netfs_io_request *wreq);
30
31static void netfs_set_group(struct folio *folio, struct netfs_group *netfs_group)
32{
33 if (netfs_group && !folio_get_private(folio))
34 folio_attach_private(folio, netfs_get_group(netfs_group));
35}
36
37#if IS_ENABLED(CONFIG_FSCACHE)
38static void netfs_folio_start_fscache(bool caching, struct folio *folio)
39{
40 if (caching)
41 folio_start_fscache(folio);
42}
43#else
44static void netfs_folio_start_fscache(bool caching, struct folio *folio)
45{
46}
47#endif
48
49/*
50 * Decide how we should modify a folio. We might be attempting to do
51 * write-streaming, in which case we don't want to a local RMW cycle if we can
52 * avoid it. If we're doing local caching or content crypto, we award that
53 * priority over avoiding RMW. If the file is open readably, then we also
54 * assume that we may want to read what we wrote.
55 */
56static enum netfs_how_to_modify netfs_how_to_modify(struct netfs_inode *ctx,
57 struct file *file,
58 struct folio *folio,
59 void *netfs_group,
60 size_t flen,
61 size_t offset,
62 size_t len,
63 bool maybe_trouble)
64{
65 struct netfs_folio *finfo = netfs_folio_info(folio);
66 loff_t pos = folio_file_pos(folio);
67
68 _enter("");
69
70 if (netfs_folio_group(folio) != netfs_group)
71 return NETFS_FLUSH_CONTENT;
72
73 if (folio_test_uptodate(folio))
74 return NETFS_FOLIO_IS_UPTODATE;
75
76 if (pos >= ctx->zero_point)
77 return NETFS_MODIFY_AND_CLEAR;
78
79 if (!maybe_trouble && offset == 0 && len >= flen)
80 return NETFS_WHOLE_FOLIO_MODIFY;
81
82 if (file->f_mode & FMODE_READ)
83 goto no_write_streaming;
84 if (test_bit(NETFS_ICTX_NO_WRITE_STREAMING, &ctx->flags))
85 goto no_write_streaming;
86
87 if (netfs_is_cache_enabled(ctx)) {
88 /* We don't want to get a streaming write on a file that loses
89 * caching service temporarily because the backing store got
90 * culled.
91 */
92 if (!test_bit(NETFS_ICTX_NO_WRITE_STREAMING, &ctx->flags))
93 set_bit(NETFS_ICTX_NO_WRITE_STREAMING, &ctx->flags);
94 goto no_write_streaming;
95 }
96
97 if (!finfo)
98 return NETFS_STREAMING_WRITE;
99
100 /* We can continue a streaming write only if it continues on from the
101 * previous. If it overlaps, we must flush lest we suffer a partial
102 * copy and disjoint dirty regions.
103 */
104 if (offset == finfo->dirty_offset + finfo->dirty_len)
105 return NETFS_STREAMING_WRITE_CONT;
106 return NETFS_FLUSH_CONTENT;
107
108no_write_streaming:
109 if (finfo) {
110 netfs_stat(&netfs_n_wh_wstream_conflict);
111 return NETFS_FLUSH_CONTENT;
112 }
113 return NETFS_JUST_PREFETCH;
114}
115
116/*
117 * Grab a folio for writing and lock it. Attempt to allocate as large a folio
118 * as possible to hold as much of the remaining length as possible in one go.
119 */
120static struct folio *netfs_grab_folio_for_write(struct address_space *mapping,
121 loff_t pos, size_t part)
122{
123 pgoff_t index = pos / PAGE_SIZE;
124 fgf_t fgp_flags = FGP_WRITEBEGIN;
125
126 if (mapping_large_folio_support(mapping))
127 fgp_flags |= fgf_set_order(pos % PAGE_SIZE + part);
128
129 return __filemap_get_folio(mapping, index, fgp_flags,
130 mapping_gfp_mask(mapping));
131}
132
133/**
134 * netfs_perform_write - Copy data into the pagecache.
135 * @iocb: The operation parameters
136 * @iter: The source buffer
137 * @netfs_group: Grouping for dirty pages (eg. ceph snaps).
138 *
139 * Copy data into pagecache pages attached to the inode specified by @iocb.
140 * The caller must hold appropriate inode locks.
141 *
142 * Dirty pages are tagged with a netfs_folio struct if they're not up to date
143 * to indicate the range modified. Dirty pages may also be tagged with a
144 * netfs-specific grouping such that data from an old group gets flushed before
145 * a new one is started.
146 */
147ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
148 struct netfs_group *netfs_group)
149{
150 struct file *file = iocb->ki_filp;
151 struct inode *inode = file_inode(file);
152 struct address_space *mapping = inode->i_mapping;
153 struct netfs_inode *ctx = netfs_inode(inode);
154 struct writeback_control wbc = {
155 .sync_mode = WB_SYNC_NONE,
156 .for_sync = true,
157 .nr_to_write = LONG_MAX,
158 .range_start = iocb->ki_pos,
159 .range_end = iocb->ki_pos + iter->count,
160 };
161 struct netfs_io_request *wreq = NULL;
162 struct netfs_folio *finfo;
163 struct folio *folio;
164 enum netfs_how_to_modify howto;
165 enum netfs_folio_trace trace;
166 unsigned int bdp_flags = (iocb->ki_flags & IOCB_NOWAIT) ? BDP_ASYNC : 0;
167 ssize_t written = 0, ret, ret2;
168 loff_t i_size, pos = iocb->ki_pos, from, to;
169 size_t max_chunk = PAGE_SIZE << MAX_PAGECACHE_ORDER;
170 bool maybe_trouble = false;
171
172 if (unlikely(test_bit(NETFS_ICTX_WRITETHROUGH, &ctx->flags) ||
173 iocb->ki_flags & (IOCB_DSYNC | IOCB_SYNC))
174 ) {
175 wbc_attach_fdatawrite_inode(&wbc, mapping->host);
176
177 ret = filemap_write_and_wait_range(mapping, pos, pos + iter->count);
178 if (ret < 0) {
179 wbc_detach_inode(&wbc);
180 goto out;
181 }
182
183 wreq = netfs_begin_writethrough(iocb, iter->count);
184 if (IS_ERR(wreq)) {
185 wbc_detach_inode(&wbc);
186 ret = PTR_ERR(wreq);
187 wreq = NULL;
188 goto out;
189 }
190 if (!is_sync_kiocb(iocb))
191 wreq->iocb = iocb;
192 wreq->cleanup = netfs_cleanup_buffered_write;
193 }
194
195 do {
196 size_t flen;
197 size_t offset; /* Offset into pagecache folio */
198 size_t part; /* Bytes to write to folio */
199 size_t copied; /* Bytes copied from user */
200
201 ret = balance_dirty_pages_ratelimited_flags(mapping, bdp_flags);
202 if (unlikely(ret < 0))
203 break;
204
205 offset = pos & (max_chunk - 1);
206 part = min(max_chunk - offset, iov_iter_count(iter));
207
208 /* Bring in the user pages that we will copy from _first_ lest
209 * we hit a nasty deadlock on copying from the same page as
210 * we're writing to, without it being marked uptodate.
211 *
212 * Not only is this an optimisation, but it is also required to
213 * check that the address is actually valid, when atomic
214 * usercopies are used below.
215 *
216 * We rely on the page being held onto long enough by the LRU
217 * that we can grab it below if this causes it to be read.
218 */
219 ret = -EFAULT;
220 if (unlikely(fault_in_iov_iter_readable(iter, part) == part))
221 break;
222
223 folio = netfs_grab_folio_for_write(mapping, pos, part);
224 if (IS_ERR(folio)) {
225 ret = PTR_ERR(folio);
226 break;
227 }
228
229 flen = folio_size(folio);
230 offset = pos & (flen - 1);
231 part = min_t(size_t, flen - offset, part);
232
233 if (signal_pending(current)) {
234 ret = written ? -EINTR : -ERESTARTSYS;
235 goto error_folio_unlock;
236 }
237
238 /* See if we need to prefetch the area we're going to modify.
239 * We need to do this before we get a lock on the folio in case
240 * there's more than one writer competing for the same cache
241 * block.
242 */
243 howto = netfs_how_to_modify(ctx, file, folio, netfs_group,
244 flen, offset, part, maybe_trouble);
245 _debug("howto %u", howto);
246 switch (howto) {
247 case NETFS_JUST_PREFETCH:
248 ret = netfs_prefetch_for_write(file, folio, offset, part);
249 if (ret < 0) {
250 _debug("prefetch = %zd", ret);
251 goto error_folio_unlock;
252 }
253 break;
254 case NETFS_FOLIO_IS_UPTODATE:
255 case NETFS_WHOLE_FOLIO_MODIFY:
256 case NETFS_STREAMING_WRITE_CONT:
257 break;
258 case NETFS_MODIFY_AND_CLEAR:
259 zero_user_segment(&folio->page, 0, offset);
260 break;
261 case NETFS_STREAMING_WRITE:
262 ret = -EIO;
263 if (WARN_ON(folio_get_private(folio)))
264 goto error_folio_unlock;
265 break;
266 case NETFS_FLUSH_CONTENT:
267 trace_netfs_folio(folio, netfs_flush_content);
268 from = folio_pos(folio);
269 to = from + folio_size(folio) - 1;
270 folio_unlock(folio);
271 folio_put(folio);
272 ret = filemap_write_and_wait_range(mapping, from, to);
273 if (ret < 0)
274 goto error_folio_unlock;
275 continue;
276 }
277
278 if (mapping_writably_mapped(mapping))
279 flush_dcache_folio(folio);
280
281 copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
282
283 flush_dcache_folio(folio);
284
285 /* Deal with a (partially) failed copy */
286 if (copied == 0) {
287 ret = -EFAULT;
288 goto error_folio_unlock;
289 }
290
291 trace = (enum netfs_folio_trace)howto;
292 switch (howto) {
293 case NETFS_FOLIO_IS_UPTODATE:
294 case NETFS_JUST_PREFETCH:
295 netfs_set_group(folio, netfs_group);
296 break;
297 case NETFS_MODIFY_AND_CLEAR:
298 zero_user_segment(&folio->page, offset + copied, flen);
299 netfs_set_group(folio, netfs_group);
300 folio_mark_uptodate(folio);
301 break;
302 case NETFS_WHOLE_FOLIO_MODIFY:
303 if (unlikely(copied < part)) {
304 maybe_trouble = true;
305 iov_iter_revert(iter, copied);
306 copied = 0;
307 goto retry;
308 }
309 netfs_set_group(folio, netfs_group);
310 folio_mark_uptodate(folio);
311 break;
312 case NETFS_STREAMING_WRITE:
313 if (offset == 0 && copied == flen) {
314 netfs_set_group(folio, netfs_group);
315 folio_mark_uptodate(folio);
316 trace = netfs_streaming_filled_page;
317 break;
318 }
319 finfo = kzalloc(sizeof(*finfo), GFP_KERNEL);
320 if (!finfo) {
321 iov_iter_revert(iter, copied);
322 ret = -ENOMEM;
323 goto error_folio_unlock;
324 }
325 finfo->netfs_group = netfs_get_group(netfs_group);
326 finfo->dirty_offset = offset;
327 finfo->dirty_len = copied;
328 folio_attach_private(folio, (void *)((unsigned long)finfo |
329 NETFS_FOLIO_INFO));
330 break;
331 case NETFS_STREAMING_WRITE_CONT:
332 finfo = netfs_folio_info(folio);
333 finfo->dirty_len += copied;
334 if (finfo->dirty_offset == 0 && finfo->dirty_len == flen) {
335 if (finfo->netfs_group)
336 folio_change_private(folio, finfo->netfs_group);
337 else
338 folio_detach_private(folio);
339 folio_mark_uptodate(folio);
340 kfree(finfo);
341 trace = netfs_streaming_cont_filled_page;
342 }
343 break;
344 default:
345 WARN(true, "Unexpected modify type %u ix=%lx\n",
346 howto, folio->index);
347 ret = -EIO;
348 goto error_folio_unlock;
349 }
350
351 trace_netfs_folio(folio, trace);
352
353 /* Update the inode size if we moved the EOF marker */
354 i_size = i_size_read(inode);
355 pos += copied;
356 if (pos > i_size) {
357 if (ctx->ops->update_i_size) {
358 ctx->ops->update_i_size(inode, pos);
359 } else {
360 i_size_write(inode, pos);
361#if IS_ENABLED(CONFIG_FSCACHE)
362 fscache_update_cookie(ctx->cache, NULL, &pos);
363#endif
364 }
365 }
366 written += copied;
367
368 if (likely(!wreq)) {
369 folio_mark_dirty(folio);
370 } else {
371 if (folio_test_dirty(folio))
372 /* Sigh. mmap. */
373 folio_clear_dirty_for_io(folio);
374 /* We make multiple writes to the folio... */
375 if (!folio_test_writeback(folio)) {
376 folio_wait_fscache(folio);
377 folio_start_writeback(folio);
378 folio_start_fscache(folio);
379 if (wreq->iter.count == 0)
380 trace_netfs_folio(folio, netfs_folio_trace_wthru);
381 else
382 trace_netfs_folio(folio, netfs_folio_trace_wthru_plus);
383 }
384 netfs_advance_writethrough(wreq, copied,
385 offset + copied == flen);
386 }
387 retry:
388 folio_unlock(folio);
389 folio_put(folio);
390 folio = NULL;
391
392 cond_resched();
393 } while (iov_iter_count(iter));
394
395out:
396 if (unlikely(wreq)) {
397 ret2 = netfs_end_writethrough(wreq, iocb);
398 wbc_detach_inode(&wbc);
399 if (ret2 == -EIOCBQUEUED)
400 return ret2;
401 if (ret == 0)
402 ret = ret2;
403 }
404
405 iocb->ki_pos += written;
406 _leave(" = %zd [%zd]", written, ret);
407 return written ? written : ret;
408
409error_folio_unlock:
410 folio_unlock(folio);
411 folio_put(folio);
412 goto out;
413}
414EXPORT_SYMBOL(netfs_perform_write);
415
416/**
417 * netfs_buffered_write_iter_locked - write data to a file
418 * @iocb: IO state structure (file, offset, etc.)
419 * @from: iov_iter with data to write
420 * @netfs_group: Grouping for dirty pages (eg. ceph snaps).
421 *
422 * This function does all the work needed for actually writing data to a
423 * file. It does all basic checks, removes SUID from the file, updates
424 * modification times and calls proper subroutines depending on whether we
425 * do direct IO or a standard buffered write.
426 *
427 * The caller must hold appropriate locks around this function and have called
428 * generic_write_checks() already. The caller is also responsible for doing
429 * any necessary syncing afterwards.
430 *
431 * This function does *not* take care of syncing data in case of O_SYNC write.
432 * A caller has to handle it. This is mainly due to the fact that we want to
433 * avoid syncing under i_rwsem.
434 *
435 * Return:
436 * * number of bytes written, even for truncated writes
437 * * negative error code if no data has been written at all
438 */
439ssize_t netfs_buffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *from,
440 struct netfs_group *netfs_group)
441{
442 struct file *file = iocb->ki_filp;
443 ssize_t ret;
444
445 trace_netfs_write_iter(iocb, from);
446
447 ret = file_remove_privs(file);
448 if (ret)
449 return ret;
450
451 ret = file_update_time(file);
452 if (ret)
453 return ret;
454
455 return netfs_perform_write(iocb, from, netfs_group);
456}
457EXPORT_SYMBOL(netfs_buffered_write_iter_locked);
458
459/**
460 * netfs_file_write_iter - write data to a file
461 * @iocb: IO state structure
462 * @from: iov_iter with data to write
463 *
464 * Perform a write to a file, writing into the pagecache if possible and doing
465 * an unbuffered write instead if not.
466 *
467 * Return:
468 * * Negative error code if no data has been written at all of
469 * vfs_fsync_range() failed for a synchronous write
470 * * Number of bytes written, even for truncated writes
471 */
472ssize_t netfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
473{
474 struct file *file = iocb->ki_filp;
475 struct inode *inode = file->f_mapping->host;
476 struct netfs_inode *ictx = netfs_inode(inode);
477 ssize_t ret;
478
479 _enter("%llx,%zx,%llx", iocb->ki_pos, iov_iter_count(from), i_size_read(inode));
480
481 if (!iov_iter_count(from))
482 return 0;
483
484 if ((iocb->ki_flags & IOCB_DIRECT) ||
485 test_bit(NETFS_ICTX_UNBUFFERED, &ictx->flags))
486 return netfs_unbuffered_write_iter(iocb, from);
487
488 ret = netfs_start_io_write(inode);
489 if (ret < 0)
490 return ret;
491
492 ret = generic_write_checks(iocb, from);
493 if (ret > 0)
494 ret = netfs_buffered_write_iter_locked(iocb, from, NULL);
495 netfs_end_io_write(inode);
496 if (ret > 0)
497 ret = generic_write_sync(iocb, ret);
498 return ret;
499}
500EXPORT_SYMBOL(netfs_file_write_iter);
501
502/*
503 * Notification that a previously read-only page is about to become writable.
504 * Note that the caller indicates a single page of a multipage folio.
505 */
506vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_group)
507{
508 struct folio *folio = page_folio(vmf->page);
509 struct file *file = vmf->vma->vm_file;
510 struct inode *inode = file_inode(file);
511 vm_fault_t ret = VM_FAULT_RETRY;
512 int err;
513
514 _enter("%lx", folio->index);
515
516 sb_start_pagefault(inode->i_sb);
517
518 if (folio_wait_writeback_killable(folio))
519 goto out;
520
521 if (folio_lock_killable(folio) < 0)
522 goto out;
523
524 /* Can we see a streaming write here? */
525 if (WARN_ON(!folio_test_uptodate(folio))) {
526 ret = VM_FAULT_SIGBUS | VM_FAULT_LOCKED;
527 goto out;
528 }
529
530 if (netfs_folio_group(folio) != netfs_group) {
531 folio_unlock(folio);
532 err = filemap_fdatawait_range(inode->i_mapping,
533 folio_pos(folio),
534 folio_pos(folio) + folio_size(folio));
535 switch (err) {
536 case 0:
537 ret = VM_FAULT_RETRY;
538 goto out;
539 case -ENOMEM:
540 ret = VM_FAULT_OOM;
541 goto out;
542 default:
543 ret = VM_FAULT_SIGBUS;
544 goto out;
545 }
546 }
547
548 if (folio_test_dirty(folio))
549 trace_netfs_folio(folio, netfs_folio_trace_mkwrite_plus);
550 else
551 trace_netfs_folio(folio, netfs_folio_trace_mkwrite);
552 netfs_set_group(folio, netfs_group);
553 file_update_time(file);
554 ret = VM_FAULT_LOCKED;
555out:
556 sb_end_pagefault(inode->i_sb);
557 return ret;
558}
559EXPORT_SYMBOL(netfs_page_mkwrite);
560
561/*
562 * Kill all the pages in the given range
563 */
564static void netfs_kill_pages(struct address_space *mapping,
565 loff_t start, loff_t len)
566{
567 struct folio *folio;
568 pgoff_t index = start / PAGE_SIZE;
569 pgoff_t last = (start + len - 1) / PAGE_SIZE, next;
570
571 _enter("%llx-%llx", start, start + len - 1);
572
573 do {
574 _debug("kill %lx (to %lx)", index, last);
575
576 folio = filemap_get_folio(mapping, index);
577 if (IS_ERR(folio)) {
578 next = index + 1;
579 continue;
580 }
581
582 next = folio_next_index(folio);
583
584 trace_netfs_folio(folio, netfs_folio_trace_kill);
585 folio_clear_uptodate(folio);
586 if (folio_test_fscache(folio))
587 folio_end_fscache(folio);
588 folio_end_writeback(folio);
589 folio_lock(folio);
590 generic_error_remove_folio(mapping, folio);
591 folio_unlock(folio);
592 folio_put(folio);
593
594 } while (index = next, index <= last);
595
596 _leave("");
597}
598
599/*
600 * Redirty all the pages in a given range.
601 */
602static void netfs_redirty_pages(struct address_space *mapping,
603 loff_t start, loff_t len)
604{
605 struct folio *folio;
606 pgoff_t index = start / PAGE_SIZE;
607 pgoff_t last = (start + len - 1) / PAGE_SIZE, next;
608
609 _enter("%llx-%llx", start, start + len - 1);
610
611 do {
612 _debug("redirty %llx @%llx", len, start);
613
614 folio = filemap_get_folio(mapping, index);
615 if (IS_ERR(folio)) {
616 next = index + 1;
617 continue;
618 }
619
620 next = folio_next_index(folio);
621 trace_netfs_folio(folio, netfs_folio_trace_redirty);
622 filemap_dirty_folio(mapping, folio);
623 if (folio_test_fscache(folio))
624 folio_end_fscache(folio);
625 folio_end_writeback(folio);
626 folio_put(folio);
627 } while (index = next, index <= last);
628
629 balance_dirty_pages_ratelimited(mapping);
630
631 _leave("");
632}
633
634/*
635 * Completion of write to server
636 */
637static void netfs_pages_written_back(struct netfs_io_request *wreq)
638{
639 struct address_space *mapping = wreq->mapping;
640 struct netfs_folio *finfo;
641 struct netfs_group *group = NULL;
642 struct folio *folio;
643 pgoff_t last;
644 int gcount = 0;
645
646 XA_STATE(xas, &mapping->i_pages, wreq->start / PAGE_SIZE);
647
648 _enter("%llx-%llx", wreq->start, wreq->start + wreq->len);
649
650 rcu_read_lock();
651
652 last = (wreq->start + wreq->len - 1) / PAGE_SIZE;
653 xas_for_each(&xas, folio, last) {
654 WARN(!folio_test_writeback(folio),
655 "bad %zx @%llx page %lx %lx\n",
656 wreq->len, wreq->start, folio->index, last);
657
658 if ((finfo = netfs_folio_info(folio))) {
659 /* Streaming writes cannot be redirtied whilst under
660 * writeback, so discard the streaming record.
661 */
662 folio_detach_private(folio);
663 group = finfo->netfs_group;
664 gcount++;
665 trace_netfs_folio(folio, netfs_folio_trace_clear_s);
666 kfree(finfo);
667 } else if ((group = netfs_folio_group(folio))) {
668 /* Need to detach the group pointer if the page didn't
669 * get redirtied. If it has been redirtied, then it
670 * must be within the same group.
671 */
672 if (folio_test_dirty(folio)) {
673 trace_netfs_folio(folio, netfs_folio_trace_redirtied);
674 goto end_wb;
675 }
676 if (folio_trylock(folio)) {
677 if (!folio_test_dirty(folio)) {
678 folio_detach_private(folio);
679 gcount++;
680 trace_netfs_folio(folio, netfs_folio_trace_clear_g);
681 } else {
682 trace_netfs_folio(folio, netfs_folio_trace_redirtied);
683 }
684 folio_unlock(folio);
685 goto end_wb;
686 }
687
688 xas_pause(&xas);
689 rcu_read_unlock();
690 folio_lock(folio);
691 if (!folio_test_dirty(folio)) {
692 folio_detach_private(folio);
693 gcount++;
694 trace_netfs_folio(folio, netfs_folio_trace_clear_g);
695 } else {
696 trace_netfs_folio(folio, netfs_folio_trace_redirtied);
697 }
698 folio_unlock(folio);
699 rcu_read_lock();
700 } else {
701 trace_netfs_folio(folio, netfs_folio_trace_clear);
702 }
703 end_wb:
704 if (folio_test_fscache(folio))
705 folio_end_fscache(folio);
706 xas_advance(&xas, folio_next_index(folio) - 1);
707 folio_end_writeback(folio);
708 }
709
710 rcu_read_unlock();
711 netfs_put_group_many(group, gcount);
712 _leave("");
713}
714
715/*
716 * Deal with the disposition of the folios that are under writeback to close
717 * out the operation.
718 */
719static void netfs_cleanup_buffered_write(struct netfs_io_request *wreq)
720{
721 struct address_space *mapping = wreq->mapping;
722
723 _enter("");
724
725 switch (wreq->error) {
726 case 0:
727 netfs_pages_written_back(wreq);
728 break;
729
730 default:
731 pr_notice("R=%08x Unexpected error %d\n", wreq->debug_id, wreq->error);
732 fallthrough;
733 case -EACCES:
734 case -EPERM:
735 case -ENOKEY:
736 case -EKEYEXPIRED:
737 case -EKEYREJECTED:
738 case -EKEYREVOKED:
739 case -ENETRESET:
740 case -EDQUOT:
741 case -ENOSPC:
742 netfs_redirty_pages(mapping, wreq->start, wreq->len);
743 break;
744
745 case -EROFS:
746 case -EIO:
747 case -EREMOTEIO:
748 case -EFBIG:
749 case -ENOENT:
750 case -ENOMEDIUM:
751 case -ENXIO:
752 netfs_kill_pages(mapping, wreq->start, wreq->len);
753 break;
754 }
755
756 if (wreq->error)
757 mapping_set_error(mapping, wreq->error);
758 if (wreq->netfs_ops->done)
759 wreq->netfs_ops->done(wreq);
760}
761
762/*
763 * Extend the region to be written back to include subsequent contiguously
764 * dirty pages if possible, but don't sleep while doing so.
765 *
766 * If this page holds new content, then we can include filler zeros in the
767 * writeback.
768 */
769static void netfs_extend_writeback(struct address_space *mapping,
770 struct netfs_group *group,
771 struct xa_state *xas,
772 long *_count,
773 loff_t start,
774 loff_t max_len,
775 bool caching,
776 size_t *_len,
777 size_t *_top)
778{
779 struct netfs_folio *finfo;
780 struct folio_batch fbatch;
781 struct folio *folio;
782 unsigned int i;
783 pgoff_t index = (start + *_len) / PAGE_SIZE;
784 size_t len;
785 void *priv;
786 bool stop = true;
787
788 folio_batch_init(&fbatch);
789
790 do {
791 /* Firstly, we gather up a batch of contiguous dirty pages
792 * under the RCU read lock - but we can't clear the dirty flags
793 * there if any of those pages are mapped.
794 */
795 rcu_read_lock();
796
797 xas_for_each(xas, folio, ULONG_MAX) {
798 stop = true;
799 if (xas_retry(xas, folio))
800 continue;
801 if (xa_is_value(folio))
802 break;
803 if (folio->index != index) {
804 xas_reset(xas);
805 break;
806 }
807
808 if (!folio_try_get_rcu(folio)) {
809 xas_reset(xas);
810 continue;
811 }
812
813 /* Has the folio moved or been split? */
814 if (unlikely(folio != xas_reload(xas))) {
815 folio_put(folio);
816 xas_reset(xas);
817 break;
818 }
819
820 if (!folio_trylock(folio)) {
821 folio_put(folio);
822 xas_reset(xas);
823 break;
824 }
825 if (!folio_test_dirty(folio) ||
826 folio_test_writeback(folio) ||
827 folio_test_fscache(folio)) {
828 folio_unlock(folio);
829 folio_put(folio);
830 xas_reset(xas);
831 break;
832 }
833
834 stop = false;
835 len = folio_size(folio);
836 priv = folio_get_private(folio);
837 if ((const struct netfs_group *)priv != group) {
838 stop = true;
839 finfo = netfs_folio_info(folio);
840 if (finfo->netfs_group != group ||
841 finfo->dirty_offset > 0) {
842 folio_unlock(folio);
843 folio_put(folio);
844 xas_reset(xas);
845 break;
846 }
847 len = finfo->dirty_len;
848 }
849
850 *_top += folio_size(folio);
851 index += folio_nr_pages(folio);
852 *_count -= folio_nr_pages(folio);
853 *_len += len;
854 if (*_len >= max_len || *_count <= 0)
855 stop = true;
856
857 if (!folio_batch_add(&fbatch, folio))
858 break;
859 if (stop)
860 break;
861 }
862
863 xas_pause(xas);
864 rcu_read_unlock();
865
866 /* Now, if we obtained any folios, we can shift them to being
867 * writable and mark them for caching.
868 */
869 if (!folio_batch_count(&fbatch))
870 break;
871
872 for (i = 0; i < folio_batch_count(&fbatch); i++) {
873 folio = fbatch.folios[i];
874 trace_netfs_folio(folio, netfs_folio_trace_store_plus);
875
876 if (!folio_clear_dirty_for_io(folio))
877 BUG();
878 folio_start_writeback(folio);
879 netfs_folio_start_fscache(caching, folio);
880 folio_unlock(folio);
881 }
882
883 folio_batch_release(&fbatch);
884 cond_resched();
885 } while (!stop);
886}
887
888/*
889 * Synchronously write back the locked page and any subsequent non-locked dirty
890 * pages.
891 */
892static ssize_t netfs_write_back_from_locked_folio(struct address_space *mapping,
893 struct writeback_control *wbc,
894 struct netfs_group *group,
895 struct xa_state *xas,
896 struct folio *folio,
897 unsigned long long start,
898 unsigned long long end)
899{
900 struct netfs_io_request *wreq;
901 struct netfs_folio *finfo;
902 struct netfs_inode *ctx = netfs_inode(mapping->host);
903 unsigned long long i_size = i_size_read(&ctx->inode);
904 size_t len, max_len;
905 bool caching = netfs_is_cache_enabled(ctx);
906 long count = wbc->nr_to_write;
907 int ret;
908
909 _enter(",%lx,%llx-%llx,%u", folio->index, start, end, caching);
910
911 wreq = netfs_alloc_request(mapping, NULL, start, folio_size(folio),
912 NETFS_WRITEBACK);
913 if (IS_ERR(wreq)) {
914 folio_unlock(folio);
915 return PTR_ERR(wreq);
916 }
917
918 if (!folio_clear_dirty_for_io(folio))
919 BUG();
920 folio_start_writeback(folio);
921 netfs_folio_start_fscache(caching, folio);
922
923 count -= folio_nr_pages(folio);
924
925 /* Find all consecutive lockable dirty pages that have contiguous
926 * written regions, stopping when we find a page that is not
927 * immediately lockable, is not dirty or is missing, or we reach the
928 * end of the range.
929 */
930 trace_netfs_folio(folio, netfs_folio_trace_store);
931
932 len = wreq->len;
933 finfo = netfs_folio_info(folio);
934 if (finfo) {
935 start += finfo->dirty_offset;
936 if (finfo->dirty_offset + finfo->dirty_len != len) {
937 len = finfo->dirty_len;
938 goto cant_expand;
939 }
940 len = finfo->dirty_len;
941 }
942
943 if (start < i_size) {
944 /* Trim the write to the EOF; the extra data is ignored. Also
945 * put an upper limit on the size of a single storedata op.
946 */
947 max_len = 65536 * 4096;
948 max_len = min_t(unsigned long long, max_len, end - start + 1);
949 max_len = min_t(unsigned long long, max_len, i_size - start);
950
951 if (len < max_len)
952 netfs_extend_writeback(mapping, group, xas, &count, start,
953 max_len, caching, &len, &wreq->upper_len);
954 }
955
956cant_expand:
957 len = min_t(unsigned long long, len, i_size - start);
958
959 /* We now have a contiguous set of dirty pages, each with writeback
960 * set; the first page is still locked at this point, but all the rest
961 * have been unlocked.
962 */
963 folio_unlock(folio);
964 wreq->start = start;
965 wreq->len = len;
966
967 if (start < i_size) {
968 _debug("write back %zx @%llx [%llx]", len, start, i_size);
969
970 /* Speculatively write to the cache. We have to fix this up
971 * later if the store fails.
972 */
973 wreq->cleanup = netfs_cleanup_buffered_write;
974
975 iov_iter_xarray(&wreq->iter, ITER_SOURCE, &mapping->i_pages, start,
976 wreq->upper_len);
977 __set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags);
978 ret = netfs_begin_write(wreq, true, netfs_write_trace_writeback);
979 if (ret == 0 || ret == -EIOCBQUEUED)
980 wbc->nr_to_write -= len / PAGE_SIZE;
981 } else {
982 _debug("write discard %zx @%llx [%llx]", len, start, i_size);
983
984 /* The dirty region was entirely beyond the EOF. */
985 fscache_clear_page_bits(mapping, start, len, caching);
986 netfs_pages_written_back(wreq);
987 ret = 0;
988 }
989
990 netfs_put_request(wreq, false, netfs_rreq_trace_put_return);
991 _leave(" = 1");
992 return 1;
993}
994
995/*
996 * Write a region of pages back to the server
997 */
998static ssize_t netfs_writepages_begin(struct address_space *mapping,
999 struct writeback_control *wbc,
1000 struct netfs_group *group,
1001 struct xa_state *xas,
1002 unsigned long long *_start,
1003 unsigned long long end)
1004{
1005 const struct netfs_folio *finfo;
1006 struct folio *folio;
1007 unsigned long long start = *_start;
1008 ssize_t ret;
1009 void *priv;
1010 int skips = 0;
1011
1012 _enter("%llx,%llx,", start, end);
1013
1014search_again:
1015 /* Find the first dirty page in the group. */
1016 rcu_read_lock();
1017
1018 for (;;) {
1019 folio = xas_find_marked(xas, end / PAGE_SIZE, PAGECACHE_TAG_DIRTY);
1020 if (xas_retry(xas, folio) || xa_is_value(folio))
1021 continue;
1022 if (!folio)
1023 break;
1024
1025 if (!folio_try_get_rcu(folio)) {
1026 xas_reset(xas);
1027 continue;
1028 }
1029
1030 if (unlikely(folio != xas_reload(xas))) {
1031 folio_put(folio);
1032 xas_reset(xas);
1033 continue;
1034 }
1035
1036 /* Skip any dirty folio that's not in the group of interest. */
1037 priv = folio_get_private(folio);
1038 if ((const struct netfs_group *)priv != group) {
1039 finfo = netfs_folio_info(folio);
1040 if (finfo->netfs_group != group) {
1041 folio_put(folio);
1042 continue;
1043 }
1044 }
1045
1046 xas_pause(xas);
1047 break;
1048 }
1049 rcu_read_unlock();
1050 if (!folio)
1051 return 0;
1052
1053 start = folio_pos(folio); /* May regress with THPs */
1054
1055 _debug("wback %lx", folio->index);
1056
1057 /* At this point we hold neither the i_pages lock nor the page lock:
1058 * the page may be truncated or invalidated (changing page->mapping to
1059 * NULL), or even swizzled back from swapper_space to tmpfs file
1060 * mapping
1061 */
1062lock_again:
1063 if (wbc->sync_mode != WB_SYNC_NONE) {
1064 ret = folio_lock_killable(folio);
1065 if (ret < 0)
1066 return ret;
1067 } else {
1068 if (!folio_trylock(folio))
1069 goto search_again;
1070 }
1071
1072 if (folio->mapping != mapping ||
1073 !folio_test_dirty(folio)) {
1074 start += folio_size(folio);
1075 folio_unlock(folio);
1076 goto search_again;
1077 }
1078
1079 if (folio_test_writeback(folio) ||
1080 folio_test_fscache(folio)) {
1081 folio_unlock(folio);
1082 if (wbc->sync_mode != WB_SYNC_NONE) {
1083 folio_wait_writeback(folio);
1084#ifdef CONFIG_FSCACHE
1085 folio_wait_fscache(folio);
1086#endif
1087 goto lock_again;
1088 }
1089
1090 start += folio_size(folio);
1091 if (wbc->sync_mode == WB_SYNC_NONE) {
1092 if (skips >= 5 || need_resched()) {
1093 ret = 0;
1094 goto out;
1095 }
1096 skips++;
1097 }
1098 goto search_again;
1099 }
1100
1101 ret = netfs_write_back_from_locked_folio(mapping, wbc, group, xas,
1102 folio, start, end);
1103out:
1104 if (ret > 0)
1105 *_start = start + ret;
1106 _leave(" = %zd [%llx]", ret, *_start);
1107 return ret;
1108}
1109
1110/*
1111 * Write a region of pages back to the server
1112 */
1113static int netfs_writepages_region(struct address_space *mapping,
1114 struct writeback_control *wbc,
1115 struct netfs_group *group,
1116 unsigned long long *_start,
1117 unsigned long long end)
1118{
1119 ssize_t ret;
1120
1121 XA_STATE(xas, &mapping->i_pages, *_start / PAGE_SIZE);
1122
1123 do {
1124 ret = netfs_writepages_begin(mapping, wbc, group, &xas,
1125 _start, end);
1126 if (ret > 0 && wbc->nr_to_write > 0)
1127 cond_resched();
1128 } while (ret > 0 && wbc->nr_to_write > 0);
1129
1130 return ret > 0 ? 0 : ret;
1131}
1132
1133/*
1134 * write some of the pending data back to the server
1135 */
1136int netfs_writepages(struct address_space *mapping,
1137 struct writeback_control *wbc)
1138{
1139 struct netfs_group *group = NULL;
1140 loff_t start, end;
1141 int ret;
1142
1143 _enter("");
1144
1145 /* We have to be careful as we can end up racing with setattr()
1146 * truncating the pagecache since the caller doesn't take a lock here
1147 * to prevent it.
1148 */
1149
1150 if (wbc->range_cyclic && mapping->writeback_index) {
1151 start = mapping->writeback_index * PAGE_SIZE;
1152 ret = netfs_writepages_region(mapping, wbc, group,
1153 &start, LLONG_MAX);
1154 if (ret < 0)
1155 goto out;
1156
1157 if (wbc->nr_to_write <= 0) {
1158 mapping->writeback_index = start / PAGE_SIZE;
1159 goto out;
1160 }
1161
1162 start = 0;
1163 end = mapping->writeback_index * PAGE_SIZE;
1164 mapping->writeback_index = 0;
1165 ret = netfs_writepages_region(mapping, wbc, group, &start, end);
1166 if (ret == 0)
1167 mapping->writeback_index = start / PAGE_SIZE;
1168 } else if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) {
1169 start = 0;
1170 ret = netfs_writepages_region(mapping, wbc, group,
1171 &start, LLONG_MAX);
1172 if (wbc->nr_to_write > 0 && ret == 0)
1173 mapping->writeback_index = start / PAGE_SIZE;
1174 } else {
1175 start = wbc->range_start;
1176 ret = netfs_writepages_region(mapping, wbc, group,
1177 &start, wbc->range_end);
1178 }
1179
1180out:
1181 _leave(" = %d", ret);
1182 return ret;
1183}
1184EXPORT_SYMBOL(netfs_writepages);
1185
1186/*
1187 * Deal with the disposition of a laundered folio.
1188 */
1189static void netfs_cleanup_launder_folio(struct netfs_io_request *wreq)
1190{
1191 if (wreq->error) {
1192 pr_notice("R=%08x Laundering error %d\n", wreq->debug_id, wreq->error);
1193 mapping_set_error(wreq->mapping, wreq->error);
1194 }
1195}
1196
1197/**
1198 * netfs_launder_folio - Clean up a dirty folio that's being invalidated
1199 * @folio: The folio to clean
1200 *
1201 * This is called to write back a folio that's being invalidated when an inode
1202 * is getting torn down. Ideally, writepages would be used instead.
1203 */
1204int netfs_launder_folio(struct folio *folio)
1205{
1206 struct netfs_io_request *wreq;
1207 struct address_space *mapping = folio->mapping;
1208 struct netfs_folio *finfo = netfs_folio_info(folio);
1209 struct netfs_group *group = netfs_folio_group(folio);
1210 struct bio_vec bvec;
1211 unsigned long long i_size = i_size_read(mapping->host);
1212 unsigned long long start = folio_pos(folio);
1213 size_t offset = 0, len;
1214 int ret = 0;
1215
1216 if (finfo) {
1217 offset = finfo->dirty_offset;
1218 start += offset;
1219 len = finfo->dirty_len;
1220 } else {
1221 len = folio_size(folio);
1222 }
1223 len = min_t(unsigned long long, len, i_size - start);
1224
1225 wreq = netfs_alloc_request(mapping, NULL, start, len, NETFS_LAUNDER_WRITE);
1226 if (IS_ERR(wreq)) {
1227 ret = PTR_ERR(wreq);
1228 goto out;
1229 }
1230
1231 if (!folio_clear_dirty_for_io(folio))
1232 goto out_put;
1233
1234 trace_netfs_folio(folio, netfs_folio_trace_launder);
1235
1236 _debug("launder %llx-%llx", start, start + len - 1);
1237
1238 /* Speculatively write to the cache. We have to fix this up later if
1239 * the store fails.
1240 */
1241 wreq->cleanup = netfs_cleanup_launder_folio;
1242
1243 bvec_set_folio(&bvec, folio, len, offset);
1244 iov_iter_bvec(&wreq->iter, ITER_SOURCE, &bvec, 1, len);
1245 __set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags);
1246 ret = netfs_begin_write(wreq, true, netfs_write_trace_launder);
1247
1248out_put:
1249 folio_detach_private(folio);
1250 netfs_put_group(group);
1251 kfree(finfo);
1252 netfs_put_request(wreq, false, netfs_rreq_trace_put_return);
1253out:
1254 folio_wait_fscache(folio);
1255 _leave(" = %d", ret);
1256 return ret;
1257}
1258EXPORT_SYMBOL(netfs_launder_folio);