Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/* Network filesystem high-level buffered write support.
3 *
4 * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8#include <linux/export.h>
9#include <linux/fs.h>
10#include <linux/mm.h>
11#include <linux/pagemap.h>
12#include <linux/slab.h>
13#include <linux/pagevec.h>
14#include "internal.h"
15
16static void __netfs_set_group(struct folio *folio, struct netfs_group *netfs_group)
17{
18 if (netfs_group)
19 folio_attach_private(folio, netfs_get_group(netfs_group));
20}
21
22static void netfs_set_group(struct folio *folio, struct netfs_group *netfs_group)
23{
24 void *priv = folio_get_private(folio);
25
26 if (unlikely(priv != netfs_group)) {
27 if (netfs_group && (!priv || priv == NETFS_FOLIO_COPY_TO_CACHE))
28 folio_attach_private(folio, netfs_get_group(netfs_group));
29 else if (!netfs_group && priv == NETFS_FOLIO_COPY_TO_CACHE)
30 folio_detach_private(folio);
31 }
32}
33
34/*
35 * Grab a folio for writing and lock it. Attempt to allocate as large a folio
36 * as possible to hold as much of the remaining length as possible in one go.
37 */
38static struct folio *netfs_grab_folio_for_write(struct address_space *mapping,
39 loff_t pos, size_t part)
40{
41 pgoff_t index = pos / PAGE_SIZE;
42 fgf_t fgp_flags = FGP_WRITEBEGIN;
43
44 if (mapping_large_folio_support(mapping))
45 fgp_flags |= fgf_set_order(pos % PAGE_SIZE + part);
46
47 return __filemap_get_folio(mapping, index, fgp_flags,
48 mapping_gfp_mask(mapping));
49}
50
51/*
52 * Update i_size and estimate the update to i_blocks to reflect the additional
53 * data written into the pagecache until we can find out from the server what
54 * the values actually are.
55 */
56static void netfs_update_i_size(struct netfs_inode *ctx, struct inode *inode,
57 loff_t i_size, loff_t pos, size_t copied)
58{
59 blkcnt_t add;
60 size_t gap;
61
62 if (ctx->ops->update_i_size) {
63 ctx->ops->update_i_size(inode, pos);
64 return;
65 }
66
67 i_size_write(inode, pos);
68#if IS_ENABLED(CONFIG_FSCACHE)
69 fscache_update_cookie(ctx->cache, NULL, &pos);
70#endif
71
72 gap = SECTOR_SIZE - (i_size & (SECTOR_SIZE - 1));
73 if (copied > gap) {
74 add = DIV_ROUND_UP(copied - gap, SECTOR_SIZE);
75
76 inode->i_blocks = min_t(blkcnt_t,
77 DIV_ROUND_UP(pos, SECTOR_SIZE),
78 inode->i_blocks + add);
79 }
80}
81
82/**
83 * netfs_perform_write - Copy data into the pagecache.
84 * @iocb: The operation parameters
85 * @iter: The source buffer
86 * @netfs_group: Grouping for dirty folios (eg. ceph snaps).
87 *
88 * Copy data into pagecache folios attached to the inode specified by @iocb.
89 * The caller must hold appropriate inode locks.
90 *
91 * Dirty folios are tagged with a netfs_folio struct if they're not up to date
92 * to indicate the range modified. Dirty folios may also be tagged with a
93 * netfs-specific grouping such that data from an old group gets flushed before
94 * a new one is started.
95 */
96ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
97 struct netfs_group *netfs_group)
98{
99 struct file *file = iocb->ki_filp;
100 struct inode *inode = file_inode(file);
101 struct address_space *mapping = inode->i_mapping;
102 struct netfs_inode *ctx = netfs_inode(inode);
103 struct writeback_control wbc = {
104 .sync_mode = WB_SYNC_NONE,
105 .for_sync = true,
106 .nr_to_write = LONG_MAX,
107 .range_start = iocb->ki_pos,
108 .range_end = iocb->ki_pos + iter->count,
109 };
110 struct netfs_io_request *wreq = NULL;
111 struct folio *folio = NULL, *writethrough = NULL;
112 unsigned int bdp_flags = (iocb->ki_flags & IOCB_NOWAIT) ? BDP_ASYNC : 0;
113 ssize_t written = 0, ret, ret2;
114 loff_t i_size, pos = iocb->ki_pos;
115 size_t max_chunk = mapping_max_folio_size(mapping);
116 bool maybe_trouble = false;
117
118 if (unlikely(test_bit(NETFS_ICTX_WRITETHROUGH, &ctx->flags) ||
119 iocb->ki_flags & (IOCB_DSYNC | IOCB_SYNC))
120 ) {
121 wbc_attach_fdatawrite_inode(&wbc, mapping->host);
122
123 ret = filemap_write_and_wait_range(mapping, pos, pos + iter->count);
124 if (ret < 0) {
125 wbc_detach_inode(&wbc);
126 goto out;
127 }
128
129 wreq = netfs_begin_writethrough(iocb, iter->count);
130 if (IS_ERR(wreq)) {
131 wbc_detach_inode(&wbc);
132 ret = PTR_ERR(wreq);
133 wreq = NULL;
134 goto out;
135 }
136 if (!is_sync_kiocb(iocb))
137 wreq->iocb = iocb;
138 netfs_stat(&netfs_n_wh_writethrough);
139 } else {
140 netfs_stat(&netfs_n_wh_buffered_write);
141 }
142
143 do {
144 struct netfs_folio *finfo;
145 struct netfs_group *group;
146 unsigned long long fpos;
147 size_t flen;
148 size_t offset; /* Offset into pagecache folio */
149 size_t part; /* Bytes to write to folio */
150 size_t copied; /* Bytes copied from user */
151
152 offset = pos & (max_chunk - 1);
153 part = min(max_chunk - offset, iov_iter_count(iter));
154
155 /* Bring in the user pages that we will copy from _first_ lest
156 * we hit a nasty deadlock on copying from the same page as
157 * we're writing to, without it being marked uptodate.
158 *
159 * Not only is this an optimisation, but it is also required to
160 * check that the address is actually valid, when atomic
161 * usercopies are used below.
162 *
163 * We rely on the page being held onto long enough by the LRU
164 * that we can grab it below if this causes it to be read.
165 */
166 ret = -EFAULT;
167 if (unlikely(fault_in_iov_iter_readable(iter, part) == part))
168 break;
169
170 folio = netfs_grab_folio_for_write(mapping, pos, part);
171 if (IS_ERR(folio)) {
172 ret = PTR_ERR(folio);
173 break;
174 }
175
176 flen = folio_size(folio);
177 fpos = folio_pos(folio);
178 offset = pos - fpos;
179 part = min_t(size_t, flen - offset, part);
180
181 /* Wait for writeback to complete. The writeback engine owns
182 * the info in folio->private and may change it until it
183 * removes the WB mark.
184 */
185 if (folio_get_private(folio) &&
186 folio_wait_writeback_killable(folio)) {
187 ret = written ? -EINTR : -ERESTARTSYS;
188 goto error_folio_unlock;
189 }
190
191 if (signal_pending(current)) {
192 ret = written ? -EINTR : -ERESTARTSYS;
193 goto error_folio_unlock;
194 }
195
196 /* Decide how we should modify a folio. We might be attempting
197 * to do write-streaming, in which case we don't want to a
198 * local RMW cycle if we can avoid it. If we're doing local
199 * caching or content crypto, we award that priority over
200 * avoiding RMW. If the file is open readably, then we also
201 * assume that we may want to read what we wrote.
202 */
203 finfo = netfs_folio_info(folio);
204 group = netfs_folio_group(folio);
205
206 if (unlikely(group != netfs_group) &&
207 group != NETFS_FOLIO_COPY_TO_CACHE)
208 goto flush_content;
209
210 if (folio_test_uptodate(folio)) {
211 if (mapping_writably_mapped(mapping))
212 flush_dcache_folio(folio);
213 copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
214 if (unlikely(copied == 0))
215 goto copy_failed;
216 netfs_set_group(folio, netfs_group);
217 trace_netfs_folio(folio, netfs_folio_is_uptodate);
218 goto copied;
219 }
220
221 /* If the page is above the zero-point then we assume that the
222 * server would just return a block of zeros or a short read if
223 * we try to read it.
224 */
225 if (fpos >= ctx->zero_point) {
226 folio_zero_segment(folio, 0, offset);
227 copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
228 if (unlikely(copied == 0))
229 goto copy_failed;
230 folio_zero_segment(folio, offset + copied, flen);
231 __netfs_set_group(folio, netfs_group);
232 folio_mark_uptodate(folio);
233 trace_netfs_folio(folio, netfs_modify_and_clear);
234 goto copied;
235 }
236
237 /* See if we can write a whole folio in one go. */
238 if (!maybe_trouble && offset == 0 && part >= flen) {
239 copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
240 if (unlikely(copied == 0))
241 goto copy_failed;
242 if (unlikely(copied < part)) {
243 maybe_trouble = true;
244 iov_iter_revert(iter, copied);
245 copied = 0;
246 folio_unlock(folio);
247 goto retry;
248 }
249 __netfs_set_group(folio, netfs_group);
250 folio_mark_uptodate(folio);
251 trace_netfs_folio(folio, netfs_whole_folio_modify);
252 goto copied;
253 }
254
255 /* We don't want to do a streaming write on a file that loses
256 * caching service temporarily because the backing store got
257 * culled and we don't really want to get a streaming write on
258 * a file that's open for reading as ->read_folio() then has to
259 * be able to flush it.
260 */
261 if ((file->f_mode & FMODE_READ) ||
262 netfs_is_cache_enabled(ctx)) {
263 if (finfo) {
264 netfs_stat(&netfs_n_wh_wstream_conflict);
265 goto flush_content;
266 }
267 ret = netfs_prefetch_for_write(file, folio, offset, part);
268 if (ret < 0) {
269 _debug("prefetch = %zd", ret);
270 goto error_folio_unlock;
271 }
272 /* Note that copy-to-cache may have been set. */
273
274 copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
275 if (unlikely(copied == 0))
276 goto copy_failed;
277 netfs_set_group(folio, netfs_group);
278 trace_netfs_folio(folio, netfs_just_prefetch);
279 goto copied;
280 }
281
282 if (!finfo) {
283 ret = -EIO;
284 if (WARN_ON(folio_get_private(folio)))
285 goto error_folio_unlock;
286 copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
287 if (unlikely(copied == 0))
288 goto copy_failed;
289 if (offset == 0 && copied == flen) {
290 __netfs_set_group(folio, netfs_group);
291 folio_mark_uptodate(folio);
292 trace_netfs_folio(folio, netfs_streaming_filled_page);
293 goto copied;
294 }
295
296 finfo = kzalloc(sizeof(*finfo), GFP_KERNEL);
297 if (!finfo) {
298 iov_iter_revert(iter, copied);
299 ret = -ENOMEM;
300 goto error_folio_unlock;
301 }
302 finfo->netfs_group = netfs_get_group(netfs_group);
303 finfo->dirty_offset = offset;
304 finfo->dirty_len = copied;
305 folio_attach_private(folio, (void *)((unsigned long)finfo |
306 NETFS_FOLIO_INFO));
307 trace_netfs_folio(folio, netfs_streaming_write);
308 goto copied;
309 }
310
311 /* We can continue a streaming write only if it continues on
312 * from the previous. If it overlaps, we must flush lest we
313 * suffer a partial copy and disjoint dirty regions.
314 */
315 if (offset == finfo->dirty_offset + finfo->dirty_len) {
316 copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
317 if (unlikely(copied == 0))
318 goto copy_failed;
319 finfo->dirty_len += copied;
320 if (finfo->dirty_offset == 0 && finfo->dirty_len == flen) {
321 if (finfo->netfs_group)
322 folio_change_private(folio, finfo->netfs_group);
323 else
324 folio_detach_private(folio);
325 folio_mark_uptodate(folio);
326 kfree(finfo);
327 trace_netfs_folio(folio, netfs_streaming_cont_filled_page);
328 } else {
329 trace_netfs_folio(folio, netfs_streaming_write_cont);
330 }
331 goto copied;
332 }
333
334 /* Incompatible write; flush the folio and try again. */
335 flush_content:
336 trace_netfs_folio(folio, netfs_flush_content);
337 folio_unlock(folio);
338 folio_put(folio);
339 ret = filemap_write_and_wait_range(mapping, fpos, fpos + flen - 1);
340 if (ret < 0)
341 goto error_folio_unlock;
342 continue;
343
344 copied:
345 flush_dcache_folio(folio);
346
347 /* Update the inode size if we moved the EOF marker */
348 pos += copied;
349 i_size = i_size_read(inode);
350 if (pos > i_size)
351 netfs_update_i_size(ctx, inode, i_size, pos, copied);
352 written += copied;
353
354 if (likely(!wreq)) {
355 folio_mark_dirty(folio);
356 folio_unlock(folio);
357 } else {
358 netfs_advance_writethrough(wreq, &wbc, folio, copied,
359 offset + copied == flen,
360 &writethrough);
361 /* Folio unlocked */
362 }
363 retry:
364 folio_put(folio);
365 folio = NULL;
366
367 ret = balance_dirty_pages_ratelimited_flags(mapping, bdp_flags);
368 if (unlikely(ret < 0))
369 break;
370
371 cond_resched();
372 } while (iov_iter_count(iter));
373
374out:
375 if (likely(written)) {
376 /* Set indication that ctime and mtime got updated in case
377 * close is deferred.
378 */
379 set_bit(NETFS_ICTX_MODIFIED_ATTR, &ctx->flags);
380 if (unlikely(ctx->ops->post_modify))
381 ctx->ops->post_modify(inode);
382 }
383
384 if (unlikely(wreq)) {
385 ret2 = netfs_end_writethrough(wreq, &wbc, writethrough);
386 wbc_detach_inode(&wbc);
387 if (ret2 == -EIOCBQUEUED)
388 return ret2;
389 if (ret == 0)
390 ret = ret2;
391 }
392
393 iocb->ki_pos += written;
394 _leave(" = %zd [%zd]", written, ret);
395 return written ? written : ret;
396
397copy_failed:
398 ret = -EFAULT;
399error_folio_unlock:
400 folio_unlock(folio);
401 folio_put(folio);
402 goto out;
403}
404EXPORT_SYMBOL(netfs_perform_write);
405
406/**
407 * netfs_buffered_write_iter_locked - write data to a file
408 * @iocb: IO state structure (file, offset, etc.)
409 * @from: iov_iter with data to write
410 * @netfs_group: Grouping for dirty folios (eg. ceph snaps).
411 *
412 * This function does all the work needed for actually writing data to a
413 * file. It does all basic checks, removes SUID from the file, updates
414 * modification times and calls proper subroutines depending on whether we
415 * do direct IO or a standard buffered write.
416 *
417 * The caller must hold appropriate locks around this function and have called
418 * generic_write_checks() already. The caller is also responsible for doing
419 * any necessary syncing afterwards.
420 *
421 * This function does *not* take care of syncing data in case of O_SYNC write.
422 * A caller has to handle it. This is mainly due to the fact that we want to
423 * avoid syncing under i_rwsem.
424 *
425 * Return:
426 * * number of bytes written, even for truncated writes
427 * * negative error code if no data has been written at all
428 */
429ssize_t netfs_buffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *from,
430 struct netfs_group *netfs_group)
431{
432 struct file *file = iocb->ki_filp;
433 ssize_t ret;
434
435 trace_netfs_write_iter(iocb, from);
436
437 ret = file_remove_privs(file);
438 if (ret)
439 return ret;
440
441 ret = file_update_time(file);
442 if (ret)
443 return ret;
444
445 return netfs_perform_write(iocb, from, netfs_group);
446}
447EXPORT_SYMBOL(netfs_buffered_write_iter_locked);
448
449/**
450 * netfs_file_write_iter - write data to a file
451 * @iocb: IO state structure
452 * @from: iov_iter with data to write
453 *
454 * Perform a write to a file, writing into the pagecache if possible and doing
455 * an unbuffered write instead if not.
456 *
457 * Return:
458 * * Negative error code if no data has been written at all of
459 * vfs_fsync_range() failed for a synchronous write
460 * * Number of bytes written, even for truncated writes
461 */
462ssize_t netfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
463{
464 struct file *file = iocb->ki_filp;
465 struct inode *inode = file->f_mapping->host;
466 struct netfs_inode *ictx = netfs_inode(inode);
467 ssize_t ret;
468
469 _enter("%llx,%zx,%llx", iocb->ki_pos, iov_iter_count(from), i_size_read(inode));
470
471 if (!iov_iter_count(from))
472 return 0;
473
474 if ((iocb->ki_flags & IOCB_DIRECT) ||
475 test_bit(NETFS_ICTX_UNBUFFERED, &ictx->flags))
476 return netfs_unbuffered_write_iter(iocb, from);
477
478 ret = netfs_start_io_write(inode);
479 if (ret < 0)
480 return ret;
481
482 ret = generic_write_checks(iocb, from);
483 if (ret > 0)
484 ret = netfs_buffered_write_iter_locked(iocb, from, NULL);
485 netfs_end_io_write(inode);
486 if (ret > 0)
487 ret = generic_write_sync(iocb, ret);
488 return ret;
489}
490EXPORT_SYMBOL(netfs_file_write_iter);
491
492/*
493 * Notification that a previously read-only page is about to become writable.
494 * The caller indicates the precise page that needs to be written to, but
495 * we only track group on a per-folio basis, so we block more often than
496 * we might otherwise.
497 */
498vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_group)
499{
500 struct netfs_group *group;
501 struct folio *folio = page_folio(vmf->page);
502 struct file *file = vmf->vma->vm_file;
503 struct address_space *mapping = file->f_mapping;
504 struct inode *inode = file_inode(file);
505 struct netfs_inode *ictx = netfs_inode(inode);
506 vm_fault_t ret = VM_FAULT_NOPAGE;
507 int err;
508
509 _enter("%lx", folio->index);
510
511 sb_start_pagefault(inode->i_sb);
512
513 if (folio_lock_killable(folio) < 0)
514 goto out;
515 if (folio->mapping != mapping)
516 goto unlock;
517 if (folio_wait_writeback_killable(folio) < 0)
518 goto unlock;
519
520 /* Can we see a streaming write here? */
521 if (WARN_ON(!folio_test_uptodate(folio))) {
522 ret = VM_FAULT_SIGBUS;
523 goto unlock;
524 }
525
526 group = netfs_folio_group(folio);
527 if (group != netfs_group && group != NETFS_FOLIO_COPY_TO_CACHE) {
528 folio_unlock(folio);
529 err = filemap_fdatawrite_range(mapping,
530 folio_pos(folio),
531 folio_pos(folio) + folio_size(folio));
532 switch (err) {
533 case 0:
534 ret = VM_FAULT_RETRY;
535 goto out;
536 case -ENOMEM:
537 ret = VM_FAULT_OOM;
538 goto out;
539 default:
540 ret = VM_FAULT_SIGBUS;
541 goto out;
542 }
543 }
544
545 if (folio_test_dirty(folio))
546 trace_netfs_folio(folio, netfs_folio_trace_mkwrite_plus);
547 else
548 trace_netfs_folio(folio, netfs_folio_trace_mkwrite);
549 netfs_set_group(folio, netfs_group);
550 file_update_time(file);
551 set_bit(NETFS_ICTX_MODIFIED_ATTR, &ictx->flags);
552 if (ictx->ops->post_modify)
553 ictx->ops->post_modify(inode);
554 ret = VM_FAULT_LOCKED;
555out:
556 sb_end_pagefault(inode->i_sb);
557 return ret;
558unlock:
559 folio_unlock(folio);
560 goto out;
561}
562EXPORT_SYMBOL(netfs_page_mkwrite);