Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/* Network filesystem high-level write support.
3 *
4 * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8#include <linux/export.h>
9#include <linux/fs.h>
10#include <linux/mm.h>
11#include <linux/pagemap.h>
12#include <linux/slab.h>
13#include <linux/pagevec.h>
14#include "internal.h"
15
16/*
17 * Determined write method. Adjust netfs_folio_traces if this is changed.
18 */
19enum netfs_how_to_modify {
20 NETFS_FOLIO_IS_UPTODATE, /* Folio is uptodate already */
21 NETFS_JUST_PREFETCH, /* We have to read the folio anyway */
22 NETFS_WHOLE_FOLIO_MODIFY, /* We're going to overwrite the whole folio */
23 NETFS_MODIFY_AND_CLEAR, /* We can assume there is no data to be downloaded. */
24 NETFS_STREAMING_WRITE, /* Store incomplete data in non-uptodate page. */
25 NETFS_STREAMING_WRITE_CONT, /* Continue streaming write. */
26 NETFS_FLUSH_CONTENT, /* Flush incompatible content. */
27};
28
29static void netfs_cleanup_buffered_write(struct netfs_io_request *wreq);
30
31static void netfs_set_group(struct folio *folio, struct netfs_group *netfs_group)
32{
33 if (netfs_group && !folio_get_private(folio))
34 folio_attach_private(folio, netfs_get_group(netfs_group));
35}
36
37#if IS_ENABLED(CONFIG_FSCACHE)
38static void netfs_folio_start_fscache(bool caching, struct folio *folio)
39{
40 if (caching)
41 folio_start_fscache(folio);
42}
43#else
44static void netfs_folio_start_fscache(bool caching, struct folio *folio)
45{
46}
47#endif
48
49/*
50 * Decide how we should modify a folio. We might be attempting to do
51 * write-streaming, in which case we don't want to a local RMW cycle if we can
52 * avoid it. If we're doing local caching or content crypto, we award that
53 * priority over avoiding RMW. If the file is open readably, then we also
54 * assume that we may want to read what we wrote.
55 */
56static enum netfs_how_to_modify netfs_how_to_modify(struct netfs_inode *ctx,
57 struct file *file,
58 struct folio *folio,
59 void *netfs_group,
60 size_t flen,
61 size_t offset,
62 size_t len,
63 bool maybe_trouble)
64{
65 struct netfs_folio *finfo = netfs_folio_info(folio);
66 loff_t pos = folio_file_pos(folio);
67
68 _enter("");
69
70 if (netfs_folio_group(folio) != netfs_group)
71 return NETFS_FLUSH_CONTENT;
72
73 if (folio_test_uptodate(folio))
74 return NETFS_FOLIO_IS_UPTODATE;
75
76 if (pos >= ctx->zero_point)
77 return NETFS_MODIFY_AND_CLEAR;
78
79 if (!maybe_trouble && offset == 0 && len >= flen)
80 return NETFS_WHOLE_FOLIO_MODIFY;
81
82 if (file->f_mode & FMODE_READ)
83 goto no_write_streaming;
84 if (test_bit(NETFS_ICTX_NO_WRITE_STREAMING, &ctx->flags))
85 goto no_write_streaming;
86
87 if (netfs_is_cache_enabled(ctx)) {
88 /* We don't want to get a streaming write on a file that loses
89 * caching service temporarily because the backing store got
90 * culled.
91 */
92 if (!test_bit(NETFS_ICTX_NO_WRITE_STREAMING, &ctx->flags))
93 set_bit(NETFS_ICTX_NO_WRITE_STREAMING, &ctx->flags);
94 goto no_write_streaming;
95 }
96
97 if (!finfo)
98 return NETFS_STREAMING_WRITE;
99
100 /* We can continue a streaming write only if it continues on from the
101 * previous. If it overlaps, we must flush lest we suffer a partial
102 * copy and disjoint dirty regions.
103 */
104 if (offset == finfo->dirty_offset + finfo->dirty_len)
105 return NETFS_STREAMING_WRITE_CONT;
106 return NETFS_FLUSH_CONTENT;
107
108no_write_streaming:
109 if (finfo) {
110 netfs_stat(&netfs_n_wh_wstream_conflict);
111 return NETFS_FLUSH_CONTENT;
112 }
113 return NETFS_JUST_PREFETCH;
114}
115
116/*
117 * Grab a folio for writing and lock it. Attempt to allocate as large a folio
118 * as possible to hold as much of the remaining length as possible in one go.
119 */
120static struct folio *netfs_grab_folio_for_write(struct address_space *mapping,
121 loff_t pos, size_t part)
122{
123 pgoff_t index = pos / PAGE_SIZE;
124 fgf_t fgp_flags = FGP_WRITEBEGIN;
125
126 if (mapping_large_folio_support(mapping))
127 fgp_flags |= fgf_set_order(pos % PAGE_SIZE + part);
128
129 return __filemap_get_folio(mapping, index, fgp_flags,
130 mapping_gfp_mask(mapping));
131}
132
133/**
134 * netfs_perform_write - Copy data into the pagecache.
135 * @iocb: The operation parameters
136 * @iter: The source buffer
137 * @netfs_group: Grouping for dirty pages (eg. ceph snaps).
138 *
139 * Copy data into pagecache pages attached to the inode specified by @iocb.
140 * The caller must hold appropriate inode locks.
141 *
142 * Dirty pages are tagged with a netfs_folio struct if they're not up to date
143 * to indicate the range modified. Dirty pages may also be tagged with a
144 * netfs-specific grouping such that data from an old group gets flushed before
145 * a new one is started.
146 */
147ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
148 struct netfs_group *netfs_group)
149{
150 struct file *file = iocb->ki_filp;
151 struct inode *inode = file_inode(file);
152 struct address_space *mapping = inode->i_mapping;
153 struct netfs_inode *ctx = netfs_inode(inode);
154 struct writeback_control wbc = {
155 .sync_mode = WB_SYNC_NONE,
156 .for_sync = true,
157 .nr_to_write = LONG_MAX,
158 .range_start = iocb->ki_pos,
159 .range_end = iocb->ki_pos + iter->count,
160 };
161 struct netfs_io_request *wreq = NULL;
162 struct netfs_folio *finfo;
163 struct folio *folio;
164 enum netfs_how_to_modify howto;
165 enum netfs_folio_trace trace;
166 unsigned int bdp_flags = (iocb->ki_flags & IOCB_SYNC) ? 0: BDP_ASYNC;
167 ssize_t written = 0, ret;
168 loff_t i_size, pos = iocb->ki_pos, from, to;
169 size_t max_chunk = PAGE_SIZE << MAX_PAGECACHE_ORDER;
170 bool maybe_trouble = false;
171
172 if (unlikely(test_bit(NETFS_ICTX_WRITETHROUGH, &ctx->flags) ||
173 iocb->ki_flags & (IOCB_DSYNC | IOCB_SYNC))
174 ) {
175 if (pos < i_size_read(inode)) {
176 ret = filemap_write_and_wait_range(mapping, pos, pos + iter->count);
177 if (ret < 0) {
178 goto out;
179 }
180 }
181
182 wbc_attach_fdatawrite_inode(&wbc, mapping->host);
183
184 wreq = netfs_begin_writethrough(iocb, iter->count);
185 if (IS_ERR(wreq)) {
186 wbc_detach_inode(&wbc);
187 ret = PTR_ERR(wreq);
188 wreq = NULL;
189 goto out;
190 }
191 if (!is_sync_kiocb(iocb))
192 wreq->iocb = iocb;
193 wreq->cleanup = netfs_cleanup_buffered_write;
194 }
195
196 do {
197 size_t flen;
198 size_t offset; /* Offset into pagecache folio */
199 size_t part; /* Bytes to write to folio */
200 size_t copied; /* Bytes copied from user */
201
202 ret = balance_dirty_pages_ratelimited_flags(mapping, bdp_flags);
203 if (unlikely(ret < 0))
204 break;
205
206 offset = pos & (max_chunk - 1);
207 part = min(max_chunk - offset, iov_iter_count(iter));
208
209 /* Bring in the user pages that we will copy from _first_ lest
210 * we hit a nasty deadlock on copying from the same page as
211 * we're writing to, without it being marked uptodate.
212 *
213 * Not only is this an optimisation, but it is also required to
214 * check that the address is actually valid, when atomic
215 * usercopies are used below.
216 *
217 * We rely on the page being held onto long enough by the LRU
218 * that we can grab it below if this causes it to be read.
219 */
220 ret = -EFAULT;
221 if (unlikely(fault_in_iov_iter_readable(iter, part) == part))
222 break;
223
224 folio = netfs_grab_folio_for_write(mapping, pos, part);
225 if (IS_ERR(folio)) {
226 ret = PTR_ERR(folio);
227 break;
228 }
229
230 flen = folio_size(folio);
231 offset = pos & (flen - 1);
232 part = min_t(size_t, flen - offset, part);
233
234 if (signal_pending(current)) {
235 ret = written ? -EINTR : -ERESTARTSYS;
236 goto error_folio_unlock;
237 }
238
239 /* See if we need to prefetch the area we're going to modify.
240 * We need to do this before we get a lock on the folio in case
241 * there's more than one writer competing for the same cache
242 * block.
243 */
244 howto = netfs_how_to_modify(ctx, file, folio, netfs_group,
245 flen, offset, part, maybe_trouble);
246 _debug("howto %u", howto);
247 switch (howto) {
248 case NETFS_JUST_PREFETCH:
249 ret = netfs_prefetch_for_write(file, folio, offset, part);
250 if (ret < 0) {
251 _debug("prefetch = %zd", ret);
252 goto error_folio_unlock;
253 }
254 break;
255 case NETFS_FOLIO_IS_UPTODATE:
256 case NETFS_WHOLE_FOLIO_MODIFY:
257 case NETFS_STREAMING_WRITE_CONT:
258 break;
259 case NETFS_MODIFY_AND_CLEAR:
260 zero_user_segment(&folio->page, 0, offset);
261 break;
262 case NETFS_STREAMING_WRITE:
263 ret = -EIO;
264 if (WARN_ON(folio_get_private(folio)))
265 goto error_folio_unlock;
266 break;
267 case NETFS_FLUSH_CONTENT:
268 trace_netfs_folio(folio, netfs_flush_content);
269 from = folio_pos(folio);
270 to = from + folio_size(folio) - 1;
271 folio_unlock(folio);
272 folio_put(folio);
273 ret = filemap_write_and_wait_range(mapping, from, to);
274 if (ret < 0)
275 goto error_folio_unlock;
276 continue;
277 }
278
279 if (mapping_writably_mapped(mapping))
280 flush_dcache_folio(folio);
281
282 copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
283
284 flush_dcache_folio(folio);
285
286 /* Deal with a (partially) failed copy */
287 if (copied == 0) {
288 ret = -EFAULT;
289 goto error_folio_unlock;
290 }
291
292 trace = (enum netfs_folio_trace)howto;
293 switch (howto) {
294 case NETFS_FOLIO_IS_UPTODATE:
295 case NETFS_JUST_PREFETCH:
296 netfs_set_group(folio, netfs_group);
297 break;
298 case NETFS_MODIFY_AND_CLEAR:
299 zero_user_segment(&folio->page, offset + copied, flen);
300 netfs_set_group(folio, netfs_group);
301 folio_mark_uptodate(folio);
302 break;
303 case NETFS_WHOLE_FOLIO_MODIFY:
304 if (unlikely(copied < part)) {
305 maybe_trouble = true;
306 iov_iter_revert(iter, copied);
307 copied = 0;
308 goto retry;
309 }
310 netfs_set_group(folio, netfs_group);
311 folio_mark_uptodate(folio);
312 break;
313 case NETFS_STREAMING_WRITE:
314 if (offset == 0 && copied == flen) {
315 netfs_set_group(folio, netfs_group);
316 folio_mark_uptodate(folio);
317 trace = netfs_streaming_filled_page;
318 break;
319 }
320 finfo = kzalloc(sizeof(*finfo), GFP_KERNEL);
321 if (!finfo) {
322 iov_iter_revert(iter, copied);
323 ret = -ENOMEM;
324 goto error_folio_unlock;
325 }
326 finfo->netfs_group = netfs_get_group(netfs_group);
327 finfo->dirty_offset = offset;
328 finfo->dirty_len = copied;
329 folio_attach_private(folio, (void *)((unsigned long)finfo |
330 NETFS_FOLIO_INFO));
331 break;
332 case NETFS_STREAMING_WRITE_CONT:
333 finfo = netfs_folio_info(folio);
334 finfo->dirty_len += copied;
335 if (finfo->dirty_offset == 0 && finfo->dirty_len == flen) {
336 if (finfo->netfs_group)
337 folio_change_private(folio, finfo->netfs_group);
338 else
339 folio_detach_private(folio);
340 folio_mark_uptodate(folio);
341 kfree(finfo);
342 trace = netfs_streaming_cont_filled_page;
343 }
344 break;
345 default:
346 WARN(true, "Unexpected modify type %u ix=%lx\n",
347 howto, folio->index);
348 ret = -EIO;
349 goto error_folio_unlock;
350 }
351
352 trace_netfs_folio(folio, trace);
353
354 /* Update the inode size if we moved the EOF marker */
355 i_size = i_size_read(inode);
356 pos += copied;
357 if (pos > i_size) {
358 if (ctx->ops->update_i_size) {
359 ctx->ops->update_i_size(inode, pos);
360 } else {
361 i_size_write(inode, pos);
362#if IS_ENABLED(CONFIG_FSCACHE)
363 fscache_update_cookie(ctx->cache, NULL, &pos);
364#endif
365 }
366 }
367 written += copied;
368
369 if (likely(!wreq)) {
370 folio_mark_dirty(folio);
371 } else {
372 if (folio_test_dirty(folio))
373 /* Sigh. mmap. */
374 folio_clear_dirty_for_io(folio);
375 /* We make multiple writes to the folio... */
376 if (!folio_test_writeback(folio)) {
377 folio_wait_fscache(folio);
378 folio_start_writeback(folio);
379 folio_start_fscache(folio);
380 if (wreq->iter.count == 0)
381 trace_netfs_folio(folio, netfs_folio_trace_wthru);
382 else
383 trace_netfs_folio(folio, netfs_folio_trace_wthru_plus);
384 }
385 netfs_advance_writethrough(wreq, copied,
386 offset + copied == flen);
387 }
388 retry:
389 folio_unlock(folio);
390 folio_put(folio);
391 folio = NULL;
392
393 cond_resched();
394 } while (iov_iter_count(iter));
395
396out:
397 if (unlikely(wreq)) {
398 ret = netfs_end_writethrough(wreq, iocb);
399 wbc_detach_inode(&wbc);
400 if (ret == -EIOCBQUEUED)
401 return ret;
402 }
403
404 iocb->ki_pos += written;
405 _leave(" = %zd [%zd]", written, ret);
406 return written ? written : ret;
407
408error_folio_unlock:
409 folio_unlock(folio);
410 folio_put(folio);
411 goto out;
412}
413EXPORT_SYMBOL(netfs_perform_write);
414
415/**
416 * netfs_buffered_write_iter_locked - write data to a file
417 * @iocb: IO state structure (file, offset, etc.)
418 * @from: iov_iter with data to write
419 * @netfs_group: Grouping for dirty pages (eg. ceph snaps).
420 *
421 * This function does all the work needed for actually writing data to a
422 * file. It does all basic checks, removes SUID from the file, updates
423 * modification times and calls proper subroutines depending on whether we
424 * do direct IO or a standard buffered write.
425 *
426 * The caller must hold appropriate locks around this function and have called
427 * generic_write_checks() already. The caller is also responsible for doing
428 * any necessary syncing afterwards.
429 *
430 * This function does *not* take care of syncing data in case of O_SYNC write.
431 * A caller has to handle it. This is mainly due to the fact that we want to
432 * avoid syncing under i_rwsem.
433 *
434 * Return:
435 * * number of bytes written, even for truncated writes
436 * * negative error code if no data has been written at all
437 */
438ssize_t netfs_buffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *from,
439 struct netfs_group *netfs_group)
440{
441 struct file *file = iocb->ki_filp;
442 ssize_t ret;
443
444 trace_netfs_write_iter(iocb, from);
445
446 ret = file_remove_privs(file);
447 if (ret)
448 return ret;
449
450 ret = file_update_time(file);
451 if (ret)
452 return ret;
453
454 return netfs_perform_write(iocb, from, netfs_group);
455}
456EXPORT_SYMBOL(netfs_buffered_write_iter_locked);
457
458/**
459 * netfs_file_write_iter - write data to a file
460 * @iocb: IO state structure
461 * @from: iov_iter with data to write
462 *
463 * Perform a write to a file, writing into the pagecache if possible and doing
464 * an unbuffered write instead if not.
465 *
466 * Return:
467 * * Negative error code if no data has been written at all of
468 * vfs_fsync_range() failed for a synchronous write
469 * * Number of bytes written, even for truncated writes
470 */
471ssize_t netfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
472{
473 struct file *file = iocb->ki_filp;
474 struct inode *inode = file->f_mapping->host;
475 struct netfs_inode *ictx = netfs_inode(inode);
476 ssize_t ret;
477
478 _enter("%llx,%zx,%llx", iocb->ki_pos, iov_iter_count(from), i_size_read(inode));
479
480 if (!iov_iter_count(from))
481 return 0;
482
483 if ((iocb->ki_flags & IOCB_DIRECT) ||
484 test_bit(NETFS_ICTX_UNBUFFERED, &ictx->flags))
485 return netfs_unbuffered_write_iter(iocb, from);
486
487 ret = netfs_start_io_write(inode);
488 if (ret < 0)
489 return ret;
490
491 ret = generic_write_checks(iocb, from);
492 if (ret > 0)
493 ret = netfs_buffered_write_iter_locked(iocb, from, NULL);
494 netfs_end_io_write(inode);
495 if (ret > 0)
496 ret = generic_write_sync(iocb, ret);
497 return ret;
498}
499EXPORT_SYMBOL(netfs_file_write_iter);
500
501/*
502 * Notification that a previously read-only page is about to become writable.
503 * Note that the caller indicates a single page of a multipage folio.
504 */
505vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_group)
506{
507 struct folio *folio = page_folio(vmf->page);
508 struct file *file = vmf->vma->vm_file;
509 struct inode *inode = file_inode(file);
510 vm_fault_t ret = VM_FAULT_RETRY;
511 int err;
512
513 _enter("%lx", folio->index);
514
515 sb_start_pagefault(inode->i_sb);
516
517 if (folio_wait_writeback_killable(folio))
518 goto out;
519
520 if (folio_lock_killable(folio) < 0)
521 goto out;
522
523 /* Can we see a streaming write here? */
524 if (WARN_ON(!folio_test_uptodate(folio))) {
525 ret = VM_FAULT_SIGBUS | VM_FAULT_LOCKED;
526 goto out;
527 }
528
529 if (netfs_folio_group(folio) != netfs_group) {
530 folio_unlock(folio);
531 err = filemap_fdatawait_range(inode->i_mapping,
532 folio_pos(folio),
533 folio_pos(folio) + folio_size(folio));
534 switch (err) {
535 case 0:
536 ret = VM_FAULT_RETRY;
537 goto out;
538 case -ENOMEM:
539 ret = VM_FAULT_OOM;
540 goto out;
541 default:
542 ret = VM_FAULT_SIGBUS;
543 goto out;
544 }
545 }
546
547 if (folio_test_dirty(folio))
548 trace_netfs_folio(folio, netfs_folio_trace_mkwrite_plus);
549 else
550 trace_netfs_folio(folio, netfs_folio_trace_mkwrite);
551 netfs_set_group(folio, netfs_group);
552 file_update_time(file);
553 ret = VM_FAULT_LOCKED;
554out:
555 sb_end_pagefault(inode->i_sb);
556 return ret;
557}
558EXPORT_SYMBOL(netfs_page_mkwrite);
559
560/*
561 * Kill all the pages in the given range
562 */
563static void netfs_kill_pages(struct address_space *mapping,
564 loff_t start, loff_t len)
565{
566 struct folio *folio;
567 pgoff_t index = start / PAGE_SIZE;
568 pgoff_t last = (start + len - 1) / PAGE_SIZE, next;
569
570 _enter("%llx-%llx", start, start + len - 1);
571
572 do {
573 _debug("kill %lx (to %lx)", index, last);
574
575 folio = filemap_get_folio(mapping, index);
576 if (IS_ERR(folio)) {
577 next = index + 1;
578 continue;
579 }
580
581 next = folio_next_index(folio);
582
583 trace_netfs_folio(folio, netfs_folio_trace_kill);
584 folio_clear_uptodate(folio);
585 if (folio_test_fscache(folio))
586 folio_end_fscache(folio);
587 folio_end_writeback(folio);
588 folio_lock(folio);
589 generic_error_remove_folio(mapping, folio);
590 folio_unlock(folio);
591 folio_put(folio);
592
593 } while (index = next, index <= last);
594
595 _leave("");
596}
597
598/*
599 * Redirty all the pages in a given range.
600 */
601static void netfs_redirty_pages(struct address_space *mapping,
602 loff_t start, loff_t len)
603{
604 struct folio *folio;
605 pgoff_t index = start / PAGE_SIZE;
606 pgoff_t last = (start + len - 1) / PAGE_SIZE, next;
607
608 _enter("%llx-%llx", start, start + len - 1);
609
610 do {
611 _debug("redirty %llx @%llx", len, start);
612
613 folio = filemap_get_folio(mapping, index);
614 if (IS_ERR(folio)) {
615 next = index + 1;
616 continue;
617 }
618
619 next = folio_next_index(folio);
620 trace_netfs_folio(folio, netfs_folio_trace_redirty);
621 filemap_dirty_folio(mapping, folio);
622 if (folio_test_fscache(folio))
623 folio_end_fscache(folio);
624 folio_end_writeback(folio);
625 folio_put(folio);
626 } while (index = next, index <= last);
627
628 balance_dirty_pages_ratelimited(mapping);
629
630 _leave("");
631}
632
633/*
634 * Completion of write to server
635 */
636static void netfs_pages_written_back(struct netfs_io_request *wreq)
637{
638 struct address_space *mapping = wreq->mapping;
639 struct netfs_folio *finfo;
640 struct netfs_group *group = NULL;
641 struct folio *folio;
642 pgoff_t last;
643 int gcount = 0;
644
645 XA_STATE(xas, &mapping->i_pages, wreq->start / PAGE_SIZE);
646
647 _enter("%llx-%llx", wreq->start, wreq->start + wreq->len);
648
649 rcu_read_lock();
650
651 last = (wreq->start + wreq->len - 1) / PAGE_SIZE;
652 xas_for_each(&xas, folio, last) {
653 WARN(!folio_test_writeback(folio),
654 "bad %zx @%llx page %lx %lx\n",
655 wreq->len, wreq->start, folio->index, last);
656
657 if ((finfo = netfs_folio_info(folio))) {
658 /* Streaming writes cannot be redirtied whilst under
659 * writeback, so discard the streaming record.
660 */
661 folio_detach_private(folio);
662 group = finfo->netfs_group;
663 gcount++;
664 trace_netfs_folio(folio, netfs_folio_trace_clear_s);
665 kfree(finfo);
666 } else if ((group = netfs_folio_group(folio))) {
667 /* Need to detach the group pointer if the page didn't
668 * get redirtied. If it has been redirtied, then it
669 * must be within the same group.
670 */
671 if (folio_test_dirty(folio)) {
672 trace_netfs_folio(folio, netfs_folio_trace_redirtied);
673 goto end_wb;
674 }
675 if (folio_trylock(folio)) {
676 if (!folio_test_dirty(folio)) {
677 folio_detach_private(folio);
678 gcount++;
679 trace_netfs_folio(folio, netfs_folio_trace_clear_g);
680 } else {
681 trace_netfs_folio(folio, netfs_folio_trace_redirtied);
682 }
683 folio_unlock(folio);
684 goto end_wb;
685 }
686
687 xas_pause(&xas);
688 rcu_read_unlock();
689 folio_lock(folio);
690 if (!folio_test_dirty(folio)) {
691 folio_detach_private(folio);
692 gcount++;
693 trace_netfs_folio(folio, netfs_folio_trace_clear_g);
694 } else {
695 trace_netfs_folio(folio, netfs_folio_trace_redirtied);
696 }
697 folio_unlock(folio);
698 rcu_read_lock();
699 } else {
700 trace_netfs_folio(folio, netfs_folio_trace_clear);
701 }
702 end_wb:
703 if (folio_test_fscache(folio))
704 folio_end_fscache(folio);
705 xas_advance(&xas, folio_next_index(folio) - 1);
706 folio_end_writeback(folio);
707 }
708
709 rcu_read_unlock();
710 netfs_put_group_many(group, gcount);
711 _leave("");
712}
713
714/*
715 * Deal with the disposition of the folios that are under writeback to close
716 * out the operation.
717 */
718static void netfs_cleanup_buffered_write(struct netfs_io_request *wreq)
719{
720 struct address_space *mapping = wreq->mapping;
721
722 _enter("");
723
724 switch (wreq->error) {
725 case 0:
726 netfs_pages_written_back(wreq);
727 break;
728
729 default:
730 pr_notice("R=%08x Unexpected error %d\n", wreq->debug_id, wreq->error);
731 fallthrough;
732 case -EACCES:
733 case -EPERM:
734 case -ENOKEY:
735 case -EKEYEXPIRED:
736 case -EKEYREJECTED:
737 case -EKEYREVOKED:
738 case -ENETRESET:
739 case -EDQUOT:
740 case -ENOSPC:
741 netfs_redirty_pages(mapping, wreq->start, wreq->len);
742 break;
743
744 case -EROFS:
745 case -EIO:
746 case -EREMOTEIO:
747 case -EFBIG:
748 case -ENOENT:
749 case -ENOMEDIUM:
750 case -ENXIO:
751 netfs_kill_pages(mapping, wreq->start, wreq->len);
752 break;
753 }
754
755 if (wreq->error)
756 mapping_set_error(mapping, wreq->error);
757 if (wreq->netfs_ops->done)
758 wreq->netfs_ops->done(wreq);
759}
760
761/*
762 * Extend the region to be written back to include subsequent contiguously
763 * dirty pages if possible, but don't sleep while doing so.
764 *
765 * If this page holds new content, then we can include filler zeros in the
766 * writeback.
767 */
768static void netfs_extend_writeback(struct address_space *mapping,
769 struct netfs_group *group,
770 struct xa_state *xas,
771 long *_count,
772 loff_t start,
773 loff_t max_len,
774 bool caching,
775 size_t *_len,
776 size_t *_top)
777{
778 struct netfs_folio *finfo;
779 struct folio_batch fbatch;
780 struct folio *folio;
781 unsigned int i;
782 pgoff_t index = (start + *_len) / PAGE_SIZE;
783 size_t len;
784 void *priv;
785 bool stop = true;
786
787 folio_batch_init(&fbatch);
788
789 do {
790 /* Firstly, we gather up a batch of contiguous dirty pages
791 * under the RCU read lock - but we can't clear the dirty flags
792 * there if any of those pages are mapped.
793 */
794 rcu_read_lock();
795
796 xas_for_each(xas, folio, ULONG_MAX) {
797 stop = true;
798 if (xas_retry(xas, folio))
799 continue;
800 if (xa_is_value(folio))
801 break;
802 if (folio->index != index) {
803 xas_reset(xas);
804 break;
805 }
806
807 if (!folio_try_get_rcu(folio)) {
808 xas_reset(xas);
809 continue;
810 }
811
812 /* Has the folio moved or been split? */
813 if (unlikely(folio != xas_reload(xas))) {
814 folio_put(folio);
815 xas_reset(xas);
816 break;
817 }
818
819 if (!folio_trylock(folio)) {
820 folio_put(folio);
821 xas_reset(xas);
822 break;
823 }
824 if (!folio_test_dirty(folio) ||
825 folio_test_writeback(folio) ||
826 folio_test_fscache(folio)) {
827 folio_unlock(folio);
828 folio_put(folio);
829 xas_reset(xas);
830 break;
831 }
832
833 stop = false;
834 len = folio_size(folio);
835 priv = folio_get_private(folio);
836 if ((const struct netfs_group *)priv != group) {
837 stop = true;
838 finfo = netfs_folio_info(folio);
839 if (finfo->netfs_group != group ||
840 finfo->dirty_offset > 0) {
841 folio_unlock(folio);
842 folio_put(folio);
843 xas_reset(xas);
844 break;
845 }
846 len = finfo->dirty_len;
847 }
848
849 *_top += folio_size(folio);
850 index += folio_nr_pages(folio);
851 *_count -= folio_nr_pages(folio);
852 *_len += len;
853 if (*_len >= max_len || *_count <= 0)
854 stop = true;
855
856 if (!folio_batch_add(&fbatch, folio))
857 break;
858 if (stop)
859 break;
860 }
861
862 xas_pause(xas);
863 rcu_read_unlock();
864
865 /* Now, if we obtained any folios, we can shift them to being
866 * writable and mark them for caching.
867 */
868 if (!folio_batch_count(&fbatch))
869 break;
870
871 for (i = 0; i < folio_batch_count(&fbatch); i++) {
872 folio = fbatch.folios[i];
873 trace_netfs_folio(folio, netfs_folio_trace_store_plus);
874
875 if (!folio_clear_dirty_for_io(folio))
876 BUG();
877 folio_start_writeback(folio);
878 netfs_folio_start_fscache(caching, folio);
879 folio_unlock(folio);
880 }
881
882 folio_batch_release(&fbatch);
883 cond_resched();
884 } while (!stop);
885}
886
887/*
888 * Synchronously write back the locked page and any subsequent non-locked dirty
889 * pages.
890 */
891static ssize_t netfs_write_back_from_locked_folio(struct address_space *mapping,
892 struct writeback_control *wbc,
893 struct netfs_group *group,
894 struct xa_state *xas,
895 struct folio *folio,
896 unsigned long long start,
897 unsigned long long end)
898{
899 struct netfs_io_request *wreq;
900 struct netfs_folio *finfo;
901 struct netfs_inode *ctx = netfs_inode(mapping->host);
902 unsigned long long i_size = i_size_read(&ctx->inode);
903 size_t len, max_len;
904 bool caching = netfs_is_cache_enabled(ctx);
905 long count = wbc->nr_to_write;
906 int ret;
907
908 _enter(",%lx,%llx-%llx,%u", folio->index, start, end, caching);
909
910 wreq = netfs_alloc_request(mapping, NULL, start, folio_size(folio),
911 NETFS_WRITEBACK);
912 if (IS_ERR(wreq)) {
913 folio_unlock(folio);
914 return PTR_ERR(wreq);
915 }
916
917 if (!folio_clear_dirty_for_io(folio))
918 BUG();
919 folio_start_writeback(folio);
920 netfs_folio_start_fscache(caching, folio);
921
922 count -= folio_nr_pages(folio);
923
924 /* Find all consecutive lockable dirty pages that have contiguous
925 * written regions, stopping when we find a page that is not
926 * immediately lockable, is not dirty or is missing, or we reach the
927 * end of the range.
928 */
929 trace_netfs_folio(folio, netfs_folio_trace_store);
930
931 len = wreq->len;
932 finfo = netfs_folio_info(folio);
933 if (finfo) {
934 start += finfo->dirty_offset;
935 if (finfo->dirty_offset + finfo->dirty_len != len) {
936 len = finfo->dirty_len;
937 goto cant_expand;
938 }
939 len = finfo->dirty_len;
940 }
941
942 if (start < i_size) {
943 /* Trim the write to the EOF; the extra data is ignored. Also
944 * put an upper limit on the size of a single storedata op.
945 */
946 max_len = 65536 * 4096;
947 max_len = min_t(unsigned long long, max_len, end - start + 1);
948 max_len = min_t(unsigned long long, max_len, i_size - start);
949
950 if (len < max_len)
951 netfs_extend_writeback(mapping, group, xas, &count, start,
952 max_len, caching, &len, &wreq->upper_len);
953 }
954
955cant_expand:
956 len = min_t(unsigned long long, len, i_size - start);
957
958 /* We now have a contiguous set of dirty pages, each with writeback
959 * set; the first page is still locked at this point, but all the rest
960 * have been unlocked.
961 */
962 folio_unlock(folio);
963 wreq->start = start;
964 wreq->len = len;
965
966 if (start < i_size) {
967 _debug("write back %zx @%llx [%llx]", len, start, i_size);
968
969 /* Speculatively write to the cache. We have to fix this up
970 * later if the store fails.
971 */
972 wreq->cleanup = netfs_cleanup_buffered_write;
973
974 iov_iter_xarray(&wreq->iter, ITER_SOURCE, &mapping->i_pages, start,
975 wreq->upper_len);
976 __set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags);
977 ret = netfs_begin_write(wreq, true, netfs_write_trace_writeback);
978 if (ret == 0 || ret == -EIOCBQUEUED)
979 wbc->nr_to_write -= len / PAGE_SIZE;
980 } else {
981 _debug("write discard %zx @%llx [%llx]", len, start, i_size);
982
983 /* The dirty region was entirely beyond the EOF. */
984 fscache_clear_page_bits(mapping, start, len, caching);
985 netfs_pages_written_back(wreq);
986 ret = 0;
987 }
988
989 netfs_put_request(wreq, false, netfs_rreq_trace_put_return);
990 _leave(" = 1");
991 return 1;
992}
993
994/*
995 * Write a region of pages back to the server
996 */
997static ssize_t netfs_writepages_begin(struct address_space *mapping,
998 struct writeback_control *wbc,
999 struct netfs_group *group,
1000 struct xa_state *xas,
1001 unsigned long long *_start,
1002 unsigned long long end)
1003{
1004 const struct netfs_folio *finfo;
1005 struct folio *folio;
1006 unsigned long long start = *_start;
1007 ssize_t ret;
1008 void *priv;
1009 int skips = 0;
1010
1011 _enter("%llx,%llx,", start, end);
1012
1013search_again:
1014 /* Find the first dirty page in the group. */
1015 rcu_read_lock();
1016
1017 for (;;) {
1018 folio = xas_find_marked(xas, end / PAGE_SIZE, PAGECACHE_TAG_DIRTY);
1019 if (xas_retry(xas, folio) || xa_is_value(folio))
1020 continue;
1021 if (!folio)
1022 break;
1023
1024 if (!folio_try_get_rcu(folio)) {
1025 xas_reset(xas);
1026 continue;
1027 }
1028
1029 if (unlikely(folio != xas_reload(xas))) {
1030 folio_put(folio);
1031 xas_reset(xas);
1032 continue;
1033 }
1034
1035 /* Skip any dirty folio that's not in the group of interest. */
1036 priv = folio_get_private(folio);
1037 if ((const struct netfs_group *)priv != group) {
1038 finfo = netfs_folio_info(folio);
1039 if (finfo->netfs_group != group) {
1040 folio_put(folio);
1041 continue;
1042 }
1043 }
1044
1045 xas_pause(xas);
1046 break;
1047 }
1048 rcu_read_unlock();
1049 if (!folio)
1050 return 0;
1051
1052 start = folio_pos(folio); /* May regress with THPs */
1053
1054 _debug("wback %lx", folio->index);
1055
1056 /* At this point we hold neither the i_pages lock nor the page lock:
1057 * the page may be truncated or invalidated (changing page->mapping to
1058 * NULL), or even swizzled back from swapper_space to tmpfs file
1059 * mapping
1060 */
1061lock_again:
1062 if (wbc->sync_mode != WB_SYNC_NONE) {
1063 ret = folio_lock_killable(folio);
1064 if (ret < 0)
1065 return ret;
1066 } else {
1067 if (!folio_trylock(folio))
1068 goto search_again;
1069 }
1070
1071 if (folio->mapping != mapping ||
1072 !folio_test_dirty(folio)) {
1073 start += folio_size(folio);
1074 folio_unlock(folio);
1075 goto search_again;
1076 }
1077
1078 if (folio_test_writeback(folio) ||
1079 folio_test_fscache(folio)) {
1080 folio_unlock(folio);
1081 if (wbc->sync_mode != WB_SYNC_NONE) {
1082 folio_wait_writeback(folio);
1083#ifdef CONFIG_FSCACHE
1084 folio_wait_fscache(folio);
1085#endif
1086 goto lock_again;
1087 }
1088
1089 start += folio_size(folio);
1090 if (wbc->sync_mode == WB_SYNC_NONE) {
1091 if (skips >= 5 || need_resched()) {
1092 ret = 0;
1093 goto out;
1094 }
1095 skips++;
1096 }
1097 goto search_again;
1098 }
1099
1100 ret = netfs_write_back_from_locked_folio(mapping, wbc, group, xas,
1101 folio, start, end);
1102out:
1103 if (ret > 0)
1104 *_start = start + ret;
1105 _leave(" = %zd [%llx]", ret, *_start);
1106 return ret;
1107}
1108
1109/*
1110 * Write a region of pages back to the server
1111 */
1112static int netfs_writepages_region(struct address_space *mapping,
1113 struct writeback_control *wbc,
1114 struct netfs_group *group,
1115 unsigned long long *_start,
1116 unsigned long long end)
1117{
1118 ssize_t ret;
1119
1120 XA_STATE(xas, &mapping->i_pages, *_start / PAGE_SIZE);
1121
1122 do {
1123 ret = netfs_writepages_begin(mapping, wbc, group, &xas,
1124 _start, end);
1125 if (ret > 0 && wbc->nr_to_write > 0)
1126 cond_resched();
1127 } while (ret > 0 && wbc->nr_to_write > 0);
1128
1129 return ret > 0 ? 0 : ret;
1130}
1131
1132/*
1133 * write some of the pending data back to the server
1134 */
1135int netfs_writepages(struct address_space *mapping,
1136 struct writeback_control *wbc)
1137{
1138 struct netfs_group *group = NULL;
1139 loff_t start, end;
1140 int ret;
1141
1142 _enter("");
1143
1144 /* We have to be careful as we can end up racing with setattr()
1145 * truncating the pagecache since the caller doesn't take a lock here
1146 * to prevent it.
1147 */
1148
1149 if (wbc->range_cyclic && mapping->writeback_index) {
1150 start = mapping->writeback_index * PAGE_SIZE;
1151 ret = netfs_writepages_region(mapping, wbc, group,
1152 &start, LLONG_MAX);
1153 if (ret < 0)
1154 goto out;
1155
1156 if (wbc->nr_to_write <= 0) {
1157 mapping->writeback_index = start / PAGE_SIZE;
1158 goto out;
1159 }
1160
1161 start = 0;
1162 end = mapping->writeback_index * PAGE_SIZE;
1163 mapping->writeback_index = 0;
1164 ret = netfs_writepages_region(mapping, wbc, group, &start, end);
1165 if (ret == 0)
1166 mapping->writeback_index = start / PAGE_SIZE;
1167 } else if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) {
1168 start = 0;
1169 ret = netfs_writepages_region(mapping, wbc, group,
1170 &start, LLONG_MAX);
1171 if (wbc->nr_to_write > 0 && ret == 0)
1172 mapping->writeback_index = start / PAGE_SIZE;
1173 } else {
1174 start = wbc->range_start;
1175 ret = netfs_writepages_region(mapping, wbc, group,
1176 &start, wbc->range_end);
1177 }
1178
1179out:
1180 _leave(" = %d", ret);
1181 return ret;
1182}
1183EXPORT_SYMBOL(netfs_writepages);
1184
1185/*
1186 * Deal with the disposition of a laundered folio.
1187 */
1188static void netfs_cleanup_launder_folio(struct netfs_io_request *wreq)
1189{
1190 if (wreq->error) {
1191 pr_notice("R=%08x Laundering error %d\n", wreq->debug_id, wreq->error);
1192 mapping_set_error(wreq->mapping, wreq->error);
1193 }
1194}
1195
1196/**
1197 * netfs_launder_folio - Clean up a dirty folio that's being invalidated
1198 * @folio: The folio to clean
1199 *
1200 * This is called to write back a folio that's being invalidated when an inode
1201 * is getting torn down. Ideally, writepages would be used instead.
1202 */
1203int netfs_launder_folio(struct folio *folio)
1204{
1205 struct netfs_io_request *wreq;
1206 struct address_space *mapping = folio->mapping;
1207 struct netfs_folio *finfo = netfs_folio_info(folio);
1208 struct netfs_group *group = netfs_folio_group(folio);
1209 struct bio_vec bvec;
1210 unsigned long long i_size = i_size_read(mapping->host);
1211 unsigned long long start = folio_pos(folio);
1212 size_t offset = 0, len;
1213 int ret = 0;
1214
1215 if (finfo) {
1216 offset = finfo->dirty_offset;
1217 start += offset;
1218 len = finfo->dirty_len;
1219 } else {
1220 len = folio_size(folio);
1221 }
1222 len = min_t(unsigned long long, len, i_size - start);
1223
1224 wreq = netfs_alloc_request(mapping, NULL, start, len, NETFS_LAUNDER_WRITE);
1225 if (IS_ERR(wreq)) {
1226 ret = PTR_ERR(wreq);
1227 goto out;
1228 }
1229
1230 if (!folio_clear_dirty_for_io(folio))
1231 goto out_put;
1232
1233 trace_netfs_folio(folio, netfs_folio_trace_launder);
1234
1235 _debug("launder %llx-%llx", start, start + len - 1);
1236
1237 /* Speculatively write to the cache. We have to fix this up later if
1238 * the store fails.
1239 */
1240 wreq->cleanup = netfs_cleanup_launder_folio;
1241
1242 bvec_set_folio(&bvec, folio, len, offset);
1243 iov_iter_bvec(&wreq->iter, ITER_SOURCE, &bvec, 1, len);
1244 __set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags);
1245 ret = netfs_begin_write(wreq, true, netfs_write_trace_launder);
1246
1247out_put:
1248 folio_detach_private(folio);
1249 netfs_put_group(group);
1250 kfree(finfo);
1251 netfs_put_request(wreq, false, netfs_rreq_trace_put_return);
1252out:
1253 folio_wait_fscache(folio);
1254 _leave(" = %d", ret);
1255 return ret;
1256}
1257EXPORT_SYMBOL(netfs_launder_folio);