Loading...
Note: File does not exist in v3.1.
1/*
2 * fs/f2fs/data.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11#include <linux/fs.h>
12#include <linux/f2fs_fs.h>
13#include <linux/buffer_head.h>
14#include <linux/mpage.h>
15#include <linux/writeback.h>
16#include <linux/backing-dev.h>
17#include <linux/pagevec.h>
18#include <linux/blkdev.h>
19#include <linux/bio.h>
20#include <linux/prefetch.h>
21#include <linux/uio.h>
22#include <linux/cleancache.h>
23
24#include "f2fs.h"
25#include "node.h"
26#include "segment.h"
27#include "trace.h"
28#include <trace/events/f2fs.h>
29
30static void f2fs_read_end_io(struct bio *bio)
31{
32 struct bio_vec *bvec;
33 int i;
34
35 if (f2fs_bio_encrypted(bio)) {
36 if (bio->bi_error) {
37 fscrypt_release_ctx(bio->bi_private);
38 } else {
39 fscrypt_decrypt_bio_pages(bio->bi_private, bio);
40 return;
41 }
42 }
43
44 bio_for_each_segment_all(bvec, bio, i) {
45 struct page *page = bvec->bv_page;
46
47 if (!bio->bi_error) {
48 SetPageUptodate(page);
49 } else {
50 ClearPageUptodate(page);
51 SetPageError(page);
52 }
53 unlock_page(page);
54 }
55 bio_put(bio);
56}
57
58static void f2fs_write_end_io(struct bio *bio)
59{
60 struct f2fs_sb_info *sbi = bio->bi_private;
61 struct bio_vec *bvec;
62 int i;
63
64 bio_for_each_segment_all(bvec, bio, i) {
65 struct page *page = bvec->bv_page;
66
67 fscrypt_pullback_bio_page(&page, true);
68
69 if (unlikely(bio->bi_error)) {
70 set_bit(AS_EIO, &page->mapping->flags);
71 f2fs_stop_checkpoint(sbi);
72 }
73 end_page_writeback(page);
74 dec_page_count(sbi, F2FS_WRITEBACK);
75 }
76
77 if (!get_pages(sbi, F2FS_WRITEBACK) && wq_has_sleeper(&sbi->cp_wait))
78 wake_up(&sbi->cp_wait);
79
80 bio_put(bio);
81}
82
83/*
84 * Low-level block read/write IO operations.
85 */
86static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
87 int npages, bool is_read)
88{
89 struct bio *bio;
90
91 bio = f2fs_bio_alloc(npages);
92
93 bio->bi_bdev = sbi->sb->s_bdev;
94 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
95 bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io;
96 bio->bi_private = is_read ? NULL : sbi;
97
98 return bio;
99}
100
101static void __submit_merged_bio(struct f2fs_bio_info *io)
102{
103 struct f2fs_io_info *fio = &io->fio;
104
105 if (!io->bio)
106 return;
107
108 if (is_read_io(fio->rw))
109 trace_f2fs_submit_read_bio(io->sbi->sb, fio, io->bio);
110 else
111 trace_f2fs_submit_write_bio(io->sbi->sb, fio, io->bio);
112
113 submit_bio(fio->rw, io->bio);
114 io->bio = NULL;
115}
116
117static bool __has_merged_page(struct f2fs_bio_info *io, struct inode *inode,
118 struct page *page, nid_t ino)
119{
120 struct bio_vec *bvec;
121 struct page *target;
122 int i;
123
124 if (!io->bio)
125 return false;
126
127 if (!inode && !page && !ino)
128 return true;
129
130 bio_for_each_segment_all(bvec, io->bio, i) {
131
132 if (bvec->bv_page->mapping)
133 target = bvec->bv_page;
134 else
135 target = fscrypt_control_page(bvec->bv_page);
136
137 if (inode && inode == target->mapping->host)
138 return true;
139 if (page && page == target)
140 return true;
141 if (ino && ino == ino_of_node(target))
142 return true;
143 }
144
145 return false;
146}
147
148static bool has_merged_page(struct f2fs_sb_info *sbi, struct inode *inode,
149 struct page *page, nid_t ino,
150 enum page_type type)
151{
152 enum page_type btype = PAGE_TYPE_OF_BIO(type);
153 struct f2fs_bio_info *io = &sbi->write_io[btype];
154 bool ret;
155
156 down_read(&io->io_rwsem);
157 ret = __has_merged_page(io, inode, page, ino);
158 up_read(&io->io_rwsem);
159 return ret;
160}
161
162static void __f2fs_submit_merged_bio(struct f2fs_sb_info *sbi,
163 struct inode *inode, struct page *page,
164 nid_t ino, enum page_type type, int rw)
165{
166 enum page_type btype = PAGE_TYPE_OF_BIO(type);
167 struct f2fs_bio_info *io;
168
169 io = is_read_io(rw) ? &sbi->read_io : &sbi->write_io[btype];
170
171 down_write(&io->io_rwsem);
172
173 if (!__has_merged_page(io, inode, page, ino))
174 goto out;
175
176 /* change META to META_FLUSH in the checkpoint procedure */
177 if (type >= META_FLUSH) {
178 io->fio.type = META_FLUSH;
179 if (test_opt(sbi, NOBARRIER))
180 io->fio.rw = WRITE_FLUSH | REQ_META | REQ_PRIO;
181 else
182 io->fio.rw = WRITE_FLUSH_FUA | REQ_META | REQ_PRIO;
183 }
184 __submit_merged_bio(io);
185out:
186 up_write(&io->io_rwsem);
187}
188
189void f2fs_submit_merged_bio(struct f2fs_sb_info *sbi, enum page_type type,
190 int rw)
191{
192 __f2fs_submit_merged_bio(sbi, NULL, NULL, 0, type, rw);
193}
194
195void f2fs_submit_merged_bio_cond(struct f2fs_sb_info *sbi,
196 struct inode *inode, struct page *page,
197 nid_t ino, enum page_type type, int rw)
198{
199 if (has_merged_page(sbi, inode, page, ino, type))
200 __f2fs_submit_merged_bio(sbi, inode, page, ino, type, rw);
201}
202
203void f2fs_flush_merged_bios(struct f2fs_sb_info *sbi)
204{
205 f2fs_submit_merged_bio(sbi, DATA, WRITE);
206 f2fs_submit_merged_bio(sbi, NODE, WRITE);
207 f2fs_submit_merged_bio(sbi, META, WRITE);
208}
209
210/*
211 * Fill the locked page with data located in the block address.
212 * Return unlocked page.
213 */
214int f2fs_submit_page_bio(struct f2fs_io_info *fio)
215{
216 struct bio *bio;
217 struct page *page = fio->encrypted_page ?
218 fio->encrypted_page : fio->page;
219
220 trace_f2fs_submit_page_bio(page, fio);
221 f2fs_trace_ios(fio, 0);
222
223 /* Allocate a new bio */
224 bio = __bio_alloc(fio->sbi, fio->new_blkaddr, 1, is_read_io(fio->rw));
225
226 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
227 bio_put(bio);
228 return -EFAULT;
229 }
230
231 submit_bio(fio->rw, bio);
232 return 0;
233}
234
235void f2fs_submit_page_mbio(struct f2fs_io_info *fio)
236{
237 struct f2fs_sb_info *sbi = fio->sbi;
238 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
239 struct f2fs_bio_info *io;
240 bool is_read = is_read_io(fio->rw);
241 struct page *bio_page;
242
243 io = is_read ? &sbi->read_io : &sbi->write_io[btype];
244
245 if (fio->old_blkaddr != NEW_ADDR)
246 verify_block_addr(sbi, fio->old_blkaddr);
247 verify_block_addr(sbi, fio->new_blkaddr);
248
249 down_write(&io->io_rwsem);
250
251 if (!is_read)
252 inc_page_count(sbi, F2FS_WRITEBACK);
253
254 if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 ||
255 io->fio.rw != fio->rw))
256 __submit_merged_bio(io);
257alloc_new:
258 if (io->bio == NULL) {
259 int bio_blocks = MAX_BIO_BLOCKS(sbi);
260
261 io->bio = __bio_alloc(sbi, fio->new_blkaddr,
262 bio_blocks, is_read);
263 io->fio = *fio;
264 }
265
266 bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;
267
268 if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) <
269 PAGE_SIZE) {
270 __submit_merged_bio(io);
271 goto alloc_new;
272 }
273
274 io->last_block_in_bio = fio->new_blkaddr;
275 f2fs_trace_ios(fio, 0);
276
277 up_write(&io->io_rwsem);
278 trace_f2fs_submit_page_mbio(fio->page, fio);
279}
280
281/*
282 * Lock ordering for the change of data block address:
283 * ->data_page
284 * ->node_page
285 * update block addresses in the node page
286 */
287void set_data_blkaddr(struct dnode_of_data *dn)
288{
289 struct f2fs_node *rn;
290 __le32 *addr_array;
291 struct page *node_page = dn->node_page;
292 unsigned int ofs_in_node = dn->ofs_in_node;
293
294 f2fs_wait_on_page_writeback(node_page, NODE, true);
295
296 rn = F2FS_NODE(node_page);
297
298 /* Get physical address of data block */
299 addr_array = blkaddr_in_node(rn);
300 addr_array[ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
301 if (set_page_dirty(node_page))
302 dn->node_changed = true;
303}
304
305void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
306{
307 dn->data_blkaddr = blkaddr;
308 set_data_blkaddr(dn);
309 f2fs_update_extent_cache(dn);
310}
311
312int reserve_new_block(struct dnode_of_data *dn)
313{
314 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
315
316 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
317 return -EPERM;
318 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
319 return -ENOSPC;
320
321 trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node);
322
323 dn->data_blkaddr = NEW_ADDR;
324 set_data_blkaddr(dn);
325 mark_inode_dirty(dn->inode);
326 sync_inode_page(dn);
327 return 0;
328}
329
330int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
331{
332 bool need_put = dn->inode_page ? false : true;
333 int err;
334
335 err = get_dnode_of_data(dn, index, ALLOC_NODE);
336 if (err)
337 return err;
338
339 if (dn->data_blkaddr == NULL_ADDR)
340 err = reserve_new_block(dn);
341 if (err || need_put)
342 f2fs_put_dnode(dn);
343 return err;
344}
345
346int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
347{
348 struct extent_info ei;
349 struct inode *inode = dn->inode;
350
351 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
352 dn->data_blkaddr = ei.blk + index - ei.fofs;
353 return 0;
354 }
355
356 return f2fs_reserve_block(dn, index);
357}
358
359struct page *get_read_data_page(struct inode *inode, pgoff_t index,
360 int rw, bool for_write)
361{
362 struct address_space *mapping = inode->i_mapping;
363 struct dnode_of_data dn;
364 struct page *page;
365 struct extent_info ei;
366 int err;
367 struct f2fs_io_info fio = {
368 .sbi = F2FS_I_SB(inode),
369 .type = DATA,
370 .rw = rw,
371 .encrypted_page = NULL,
372 };
373
374 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
375 return read_mapping_page(mapping, index, NULL);
376
377 page = f2fs_grab_cache_page(mapping, index, for_write);
378 if (!page)
379 return ERR_PTR(-ENOMEM);
380
381 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
382 dn.data_blkaddr = ei.blk + index - ei.fofs;
383 goto got_it;
384 }
385
386 set_new_dnode(&dn, inode, NULL, NULL, 0);
387 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
388 if (err)
389 goto put_err;
390 f2fs_put_dnode(&dn);
391
392 if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
393 err = -ENOENT;
394 goto put_err;
395 }
396got_it:
397 if (PageUptodate(page)) {
398 unlock_page(page);
399 return page;
400 }
401
402 /*
403 * A new dentry page is allocated but not able to be written, since its
404 * new inode page couldn't be allocated due to -ENOSPC.
405 * In such the case, its blkaddr can be remained as NEW_ADDR.
406 * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata.
407 */
408 if (dn.data_blkaddr == NEW_ADDR) {
409 zero_user_segment(page, 0, PAGE_SIZE);
410 SetPageUptodate(page);
411 unlock_page(page);
412 return page;
413 }
414
415 fio.new_blkaddr = fio.old_blkaddr = dn.data_blkaddr;
416 fio.page = page;
417 err = f2fs_submit_page_bio(&fio);
418 if (err)
419 goto put_err;
420 return page;
421
422put_err:
423 f2fs_put_page(page, 1);
424 return ERR_PTR(err);
425}
426
427struct page *find_data_page(struct inode *inode, pgoff_t index)
428{
429 struct address_space *mapping = inode->i_mapping;
430 struct page *page;
431
432 page = find_get_page(mapping, index);
433 if (page && PageUptodate(page))
434 return page;
435 f2fs_put_page(page, 0);
436
437 page = get_read_data_page(inode, index, READ_SYNC, false);
438 if (IS_ERR(page))
439 return page;
440
441 if (PageUptodate(page))
442 return page;
443
444 wait_on_page_locked(page);
445 if (unlikely(!PageUptodate(page))) {
446 f2fs_put_page(page, 0);
447 return ERR_PTR(-EIO);
448 }
449 return page;
450}
451
452/*
453 * If it tries to access a hole, return an error.
454 * Because, the callers, functions in dir.c and GC, should be able to know
455 * whether this page exists or not.
456 */
457struct page *get_lock_data_page(struct inode *inode, pgoff_t index,
458 bool for_write)
459{
460 struct address_space *mapping = inode->i_mapping;
461 struct page *page;
462repeat:
463 page = get_read_data_page(inode, index, READ_SYNC, for_write);
464 if (IS_ERR(page))
465 return page;
466
467 /* wait for read completion */
468 lock_page(page);
469 if (unlikely(!PageUptodate(page))) {
470 f2fs_put_page(page, 1);
471 return ERR_PTR(-EIO);
472 }
473 if (unlikely(page->mapping != mapping)) {
474 f2fs_put_page(page, 1);
475 goto repeat;
476 }
477 return page;
478}
479
480/*
481 * Caller ensures that this data page is never allocated.
482 * A new zero-filled data page is allocated in the page cache.
483 *
484 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
485 * f2fs_unlock_op().
486 * Note that, ipage is set only by make_empty_dir, and if any error occur,
487 * ipage should be released by this function.
488 */
489struct page *get_new_data_page(struct inode *inode,
490 struct page *ipage, pgoff_t index, bool new_i_size)
491{
492 struct address_space *mapping = inode->i_mapping;
493 struct page *page;
494 struct dnode_of_data dn;
495 int err;
496
497 page = f2fs_grab_cache_page(mapping, index, true);
498 if (!page) {
499 /*
500 * before exiting, we should make sure ipage will be released
501 * if any error occur.
502 */
503 f2fs_put_page(ipage, 1);
504 return ERR_PTR(-ENOMEM);
505 }
506
507 set_new_dnode(&dn, inode, ipage, NULL, 0);
508 err = f2fs_reserve_block(&dn, index);
509 if (err) {
510 f2fs_put_page(page, 1);
511 return ERR_PTR(err);
512 }
513 if (!ipage)
514 f2fs_put_dnode(&dn);
515
516 if (PageUptodate(page))
517 goto got_it;
518
519 if (dn.data_blkaddr == NEW_ADDR) {
520 zero_user_segment(page, 0, PAGE_SIZE);
521 SetPageUptodate(page);
522 } else {
523 f2fs_put_page(page, 1);
524
525 /* if ipage exists, blkaddr should be NEW_ADDR */
526 f2fs_bug_on(F2FS_I_SB(inode), ipage);
527 page = get_lock_data_page(inode, index, true);
528 if (IS_ERR(page))
529 return page;
530 }
531got_it:
532 if (new_i_size && i_size_read(inode) <
533 ((loff_t)(index + 1) << PAGE_SHIFT)) {
534 i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT));
535 /* Only the directory inode sets new_i_size */
536 set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR);
537 }
538 return page;
539}
540
541static int __allocate_data_block(struct dnode_of_data *dn)
542{
543 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
544 struct f2fs_summary sum;
545 struct node_info ni;
546 int seg = CURSEG_WARM_DATA;
547 pgoff_t fofs;
548
549 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
550 return -EPERM;
551
552 dn->data_blkaddr = datablock_addr(dn->node_page, dn->ofs_in_node);
553 if (dn->data_blkaddr == NEW_ADDR)
554 goto alloc;
555
556 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
557 return -ENOSPC;
558
559alloc:
560 get_node_info(sbi, dn->nid, &ni);
561 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
562
563 if (dn->ofs_in_node == 0 && dn->inode_page == dn->node_page)
564 seg = CURSEG_DIRECT_IO;
565
566 allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,
567 &sum, seg);
568 set_data_blkaddr(dn);
569
570 /* update i_size */
571 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) +
572 dn->ofs_in_node;
573 if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_SHIFT))
574 i_size_write(dn->inode,
575 ((loff_t)(fofs + 1) << PAGE_SHIFT));
576 return 0;
577}
578
579ssize_t f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
580{
581 struct inode *inode = file_inode(iocb->ki_filp);
582 struct f2fs_map_blocks map;
583 ssize_t ret = 0;
584
585 map.m_lblk = F2FS_BYTES_TO_BLK(iocb->ki_pos);
586 map.m_len = F2FS_BLK_ALIGN(iov_iter_count(from));
587 map.m_next_pgofs = NULL;
588
589 if (f2fs_encrypted_inode(inode))
590 return 0;
591
592 if (iocb->ki_flags & IOCB_DIRECT) {
593 ret = f2fs_convert_inline_inode(inode);
594 if (ret)
595 return ret;
596 return f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_DIO);
597 }
598 if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA) {
599 ret = f2fs_convert_inline_inode(inode);
600 if (ret)
601 return ret;
602 }
603 if (!f2fs_has_inline_data(inode))
604 return f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_AIO);
605 return ret;
606}
607
608/*
609 * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with
610 * f2fs_map_blocks structure.
611 * If original data blocks are allocated, then give them to blockdev.
612 * Otherwise,
613 * a. preallocate requested block addresses
614 * b. do not use extent cache for better performance
615 * c. give the block addresses to blockdev
616 */
617int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
618 int create, int flag)
619{
620 unsigned int maxblocks = map->m_len;
621 struct dnode_of_data dn;
622 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
623 int mode = create ? ALLOC_NODE : LOOKUP_NODE_RA;
624 pgoff_t pgofs, end_offset;
625 int err = 0, ofs = 1;
626 struct extent_info ei;
627 bool allocated = false;
628 block_t blkaddr;
629
630 map->m_len = 0;
631 map->m_flags = 0;
632
633 /* it only supports block size == page size */
634 pgofs = (pgoff_t)map->m_lblk;
635
636 if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
637 map->m_pblk = ei.blk + pgofs - ei.fofs;
638 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
639 map->m_flags = F2FS_MAP_MAPPED;
640 goto out;
641 }
642
643next_dnode:
644 if (create)
645 f2fs_lock_op(sbi);
646
647 /* When reading holes, we need its node page */
648 set_new_dnode(&dn, inode, NULL, NULL, 0);
649 err = get_dnode_of_data(&dn, pgofs, mode);
650 if (err) {
651 if (err == -ENOENT) {
652 err = 0;
653 if (map->m_next_pgofs)
654 *map->m_next_pgofs =
655 get_next_page_offset(&dn, pgofs);
656 }
657 goto unlock_out;
658 }
659
660 end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
661
662next_block:
663 blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node);
664
665 if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR) {
666 if (create) {
667 if (unlikely(f2fs_cp_error(sbi))) {
668 err = -EIO;
669 goto sync_out;
670 }
671 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
672 if (blkaddr == NULL_ADDR)
673 err = reserve_new_block(&dn);
674 } else {
675 err = __allocate_data_block(&dn);
676 }
677 if (err)
678 goto sync_out;
679 allocated = true;
680 map->m_flags = F2FS_MAP_NEW;
681 blkaddr = dn.data_blkaddr;
682 } else {
683 if (flag == F2FS_GET_BLOCK_FIEMAP &&
684 blkaddr == NULL_ADDR) {
685 if (map->m_next_pgofs)
686 *map->m_next_pgofs = pgofs + 1;
687 }
688 if (flag != F2FS_GET_BLOCK_FIEMAP ||
689 blkaddr != NEW_ADDR) {
690 if (flag == F2FS_GET_BLOCK_BMAP)
691 err = -ENOENT;
692 goto sync_out;
693 }
694 }
695 }
696
697 if (map->m_len == 0) {
698 /* preallocated unwritten block should be mapped for fiemap. */
699 if (blkaddr == NEW_ADDR)
700 map->m_flags |= F2FS_MAP_UNWRITTEN;
701 map->m_flags |= F2FS_MAP_MAPPED;
702
703 map->m_pblk = blkaddr;
704 map->m_len = 1;
705 } else if ((map->m_pblk != NEW_ADDR &&
706 blkaddr == (map->m_pblk + ofs)) ||
707 (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) ||
708 flag == F2FS_GET_BLOCK_PRE_DIO ||
709 flag == F2FS_GET_BLOCK_PRE_AIO) {
710 ofs++;
711 map->m_len++;
712 } else {
713 goto sync_out;
714 }
715
716 dn.ofs_in_node++;
717 pgofs++;
718
719 if (map->m_len < maxblocks) {
720 if (dn.ofs_in_node < end_offset)
721 goto next_block;
722
723 if (allocated)
724 sync_inode_page(&dn);
725 f2fs_put_dnode(&dn);
726
727 if (create) {
728 f2fs_unlock_op(sbi);
729 f2fs_balance_fs(sbi, allocated);
730 }
731 allocated = false;
732 goto next_dnode;
733 }
734
735sync_out:
736 if (allocated)
737 sync_inode_page(&dn);
738 f2fs_put_dnode(&dn);
739unlock_out:
740 if (create) {
741 f2fs_unlock_op(sbi);
742 f2fs_balance_fs(sbi, allocated);
743 }
744out:
745 trace_f2fs_map_blocks(inode, map, err);
746 return err;
747}
748
749static int __get_data_block(struct inode *inode, sector_t iblock,
750 struct buffer_head *bh, int create, int flag,
751 pgoff_t *next_pgofs)
752{
753 struct f2fs_map_blocks map;
754 int ret;
755
756 map.m_lblk = iblock;
757 map.m_len = bh->b_size >> inode->i_blkbits;
758 map.m_next_pgofs = next_pgofs;
759
760 ret = f2fs_map_blocks(inode, &map, create, flag);
761 if (!ret) {
762 map_bh(bh, inode->i_sb, map.m_pblk);
763 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
764 bh->b_size = map.m_len << inode->i_blkbits;
765 }
766 return ret;
767}
768
769static int get_data_block(struct inode *inode, sector_t iblock,
770 struct buffer_head *bh_result, int create, int flag,
771 pgoff_t *next_pgofs)
772{
773 return __get_data_block(inode, iblock, bh_result, create,
774 flag, next_pgofs);
775}
776
777static int get_data_block_dio(struct inode *inode, sector_t iblock,
778 struct buffer_head *bh_result, int create)
779{
780 return __get_data_block(inode, iblock, bh_result, create,
781 F2FS_GET_BLOCK_DIO, NULL);
782}
783
784static int get_data_block_bmap(struct inode *inode, sector_t iblock,
785 struct buffer_head *bh_result, int create)
786{
787 /* Block number less than F2FS MAX BLOCKS */
788 if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks))
789 return -EFBIG;
790
791 return __get_data_block(inode, iblock, bh_result, create,
792 F2FS_GET_BLOCK_BMAP, NULL);
793}
794
795static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
796{
797 return (offset >> inode->i_blkbits);
798}
799
800static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
801{
802 return (blk << inode->i_blkbits);
803}
804
805int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
806 u64 start, u64 len)
807{
808 struct buffer_head map_bh;
809 sector_t start_blk, last_blk;
810 pgoff_t next_pgofs;
811 loff_t isize;
812 u64 logical = 0, phys = 0, size = 0;
813 u32 flags = 0;
814 int ret = 0;
815
816 ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC);
817 if (ret)
818 return ret;
819
820 if (f2fs_has_inline_data(inode)) {
821 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
822 if (ret != -EAGAIN)
823 return ret;
824 }
825
826 inode_lock(inode);
827
828 isize = i_size_read(inode);
829 if (start >= isize)
830 goto out;
831
832 if (start + len > isize)
833 len = isize - start;
834
835 if (logical_to_blk(inode, len) == 0)
836 len = blk_to_logical(inode, 1);
837
838 start_blk = logical_to_blk(inode, start);
839 last_blk = logical_to_blk(inode, start + len - 1);
840
841next:
842 memset(&map_bh, 0, sizeof(struct buffer_head));
843 map_bh.b_size = len;
844
845 ret = get_data_block(inode, start_blk, &map_bh, 0,
846 F2FS_GET_BLOCK_FIEMAP, &next_pgofs);
847 if (ret)
848 goto out;
849
850 /* HOLE */
851 if (!buffer_mapped(&map_bh)) {
852 start_blk = next_pgofs;
853 /* Go through holes util pass the EOF */
854 if (blk_to_logical(inode, start_blk) < isize)
855 goto prep_next;
856 /* Found a hole beyond isize means no more extents.
857 * Note that the premise is that filesystems don't
858 * punch holes beyond isize and keep size unchanged.
859 */
860 flags |= FIEMAP_EXTENT_LAST;
861 }
862
863 if (size) {
864 if (f2fs_encrypted_inode(inode))
865 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
866
867 ret = fiemap_fill_next_extent(fieinfo, logical,
868 phys, size, flags);
869 }
870
871 if (start_blk > last_blk || ret)
872 goto out;
873
874 logical = blk_to_logical(inode, start_blk);
875 phys = blk_to_logical(inode, map_bh.b_blocknr);
876 size = map_bh.b_size;
877 flags = 0;
878 if (buffer_unwritten(&map_bh))
879 flags = FIEMAP_EXTENT_UNWRITTEN;
880
881 start_blk += logical_to_blk(inode, size);
882
883prep_next:
884 cond_resched();
885 if (fatal_signal_pending(current))
886 ret = -EINTR;
887 else
888 goto next;
889out:
890 if (ret == 1)
891 ret = 0;
892
893 inode_unlock(inode);
894 return ret;
895}
896
897/*
898 * This function was originally taken from fs/mpage.c, and customized for f2fs.
899 * Major change was from block_size == page_size in f2fs by default.
900 */
901static int f2fs_mpage_readpages(struct address_space *mapping,
902 struct list_head *pages, struct page *page,
903 unsigned nr_pages)
904{
905 struct bio *bio = NULL;
906 unsigned page_idx;
907 sector_t last_block_in_bio = 0;
908 struct inode *inode = mapping->host;
909 const unsigned blkbits = inode->i_blkbits;
910 const unsigned blocksize = 1 << blkbits;
911 sector_t block_in_file;
912 sector_t last_block;
913 sector_t last_block_in_file;
914 sector_t block_nr;
915 struct block_device *bdev = inode->i_sb->s_bdev;
916 struct f2fs_map_blocks map;
917
918 map.m_pblk = 0;
919 map.m_lblk = 0;
920 map.m_len = 0;
921 map.m_flags = 0;
922 map.m_next_pgofs = NULL;
923
924 for (page_idx = 0; nr_pages; page_idx++, nr_pages--) {
925
926 prefetchw(&page->flags);
927 if (pages) {
928 page = list_entry(pages->prev, struct page, lru);
929 list_del(&page->lru);
930 if (add_to_page_cache_lru(page, mapping,
931 page->index, GFP_KERNEL))
932 goto next_page;
933 }
934
935 block_in_file = (sector_t)page->index;
936 last_block = block_in_file + nr_pages;
937 last_block_in_file = (i_size_read(inode) + blocksize - 1) >>
938 blkbits;
939 if (last_block > last_block_in_file)
940 last_block = last_block_in_file;
941
942 /*
943 * Map blocks using the previous result first.
944 */
945 if ((map.m_flags & F2FS_MAP_MAPPED) &&
946 block_in_file > map.m_lblk &&
947 block_in_file < (map.m_lblk + map.m_len))
948 goto got_it;
949
950 /*
951 * Then do more f2fs_map_blocks() calls until we are
952 * done with this page.
953 */
954 map.m_flags = 0;
955
956 if (block_in_file < last_block) {
957 map.m_lblk = block_in_file;
958 map.m_len = last_block - block_in_file;
959
960 if (f2fs_map_blocks(inode, &map, 0,
961 F2FS_GET_BLOCK_READ))
962 goto set_error_page;
963 }
964got_it:
965 if ((map.m_flags & F2FS_MAP_MAPPED)) {
966 block_nr = map.m_pblk + block_in_file - map.m_lblk;
967 SetPageMappedToDisk(page);
968
969 if (!PageUptodate(page) && !cleancache_get_page(page)) {
970 SetPageUptodate(page);
971 goto confused;
972 }
973 } else {
974 zero_user_segment(page, 0, PAGE_SIZE);
975 SetPageUptodate(page);
976 unlock_page(page);
977 goto next_page;
978 }
979
980 /*
981 * This page will go to BIO. Do we need to send this
982 * BIO off first?
983 */
984 if (bio && (last_block_in_bio != block_nr - 1)) {
985submit_and_realloc:
986 submit_bio(READ, bio);
987 bio = NULL;
988 }
989 if (bio == NULL) {
990 struct fscrypt_ctx *ctx = NULL;
991
992 if (f2fs_encrypted_inode(inode) &&
993 S_ISREG(inode->i_mode)) {
994
995 ctx = fscrypt_get_ctx(inode, GFP_NOFS);
996 if (IS_ERR(ctx))
997 goto set_error_page;
998
999 /* wait the page to be moved by cleaning */
1000 f2fs_wait_on_encrypted_page_writeback(
1001 F2FS_I_SB(inode), block_nr);
1002 }
1003
1004 bio = bio_alloc(GFP_KERNEL,
1005 min_t(int, nr_pages, BIO_MAX_PAGES));
1006 if (!bio) {
1007 if (ctx)
1008 fscrypt_release_ctx(ctx);
1009 goto set_error_page;
1010 }
1011 bio->bi_bdev = bdev;
1012 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(block_nr);
1013 bio->bi_end_io = f2fs_read_end_io;
1014 bio->bi_private = ctx;
1015 }
1016
1017 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
1018 goto submit_and_realloc;
1019
1020 last_block_in_bio = block_nr;
1021 goto next_page;
1022set_error_page:
1023 SetPageError(page);
1024 zero_user_segment(page, 0, PAGE_SIZE);
1025 unlock_page(page);
1026 goto next_page;
1027confused:
1028 if (bio) {
1029 submit_bio(READ, bio);
1030 bio = NULL;
1031 }
1032 unlock_page(page);
1033next_page:
1034 if (pages)
1035 put_page(page);
1036 }
1037 BUG_ON(pages && !list_empty(pages));
1038 if (bio)
1039 submit_bio(READ, bio);
1040 return 0;
1041}
1042
1043static int f2fs_read_data_page(struct file *file, struct page *page)
1044{
1045 struct inode *inode = page->mapping->host;
1046 int ret = -EAGAIN;
1047
1048 trace_f2fs_readpage(page, DATA);
1049
1050 /* If the file has inline data, try to read it directly */
1051 if (f2fs_has_inline_data(inode))
1052 ret = f2fs_read_inline_data(inode, page);
1053 if (ret == -EAGAIN)
1054 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1);
1055 return ret;
1056}
1057
1058static int f2fs_read_data_pages(struct file *file,
1059 struct address_space *mapping,
1060 struct list_head *pages, unsigned nr_pages)
1061{
1062 struct inode *inode = file->f_mapping->host;
1063 struct page *page = list_entry(pages->prev, struct page, lru);
1064
1065 trace_f2fs_readpages(inode, page, nr_pages);
1066
1067 /* If the file has inline data, skip readpages */
1068 if (f2fs_has_inline_data(inode))
1069 return 0;
1070
1071 return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages);
1072}
1073
1074int do_write_data_page(struct f2fs_io_info *fio)
1075{
1076 struct page *page = fio->page;
1077 struct inode *inode = page->mapping->host;
1078 struct dnode_of_data dn;
1079 int err = 0;
1080
1081 set_new_dnode(&dn, inode, NULL, NULL, 0);
1082 err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
1083 if (err)
1084 return err;
1085
1086 fio->old_blkaddr = dn.data_blkaddr;
1087
1088 /* This page is already truncated */
1089 if (fio->old_blkaddr == NULL_ADDR) {
1090 ClearPageUptodate(page);
1091 goto out_writepage;
1092 }
1093
1094 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) {
1095 gfp_t gfp_flags = GFP_NOFS;
1096
1097 /* wait for GCed encrypted page writeback */
1098 f2fs_wait_on_encrypted_page_writeback(F2FS_I_SB(inode),
1099 fio->old_blkaddr);
1100retry_encrypt:
1101 fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page,
1102 gfp_flags);
1103 if (IS_ERR(fio->encrypted_page)) {
1104 err = PTR_ERR(fio->encrypted_page);
1105 if (err == -ENOMEM) {
1106 /* flush pending ios and wait for a while */
1107 f2fs_flush_merged_bios(F2FS_I_SB(inode));
1108 congestion_wait(BLK_RW_ASYNC, HZ/50);
1109 gfp_flags |= __GFP_NOFAIL;
1110 err = 0;
1111 goto retry_encrypt;
1112 }
1113 goto out_writepage;
1114 }
1115 }
1116
1117 set_page_writeback(page);
1118
1119 /*
1120 * If current allocation needs SSR,
1121 * it had better in-place writes for updated data.
1122 */
1123 if (unlikely(fio->old_blkaddr != NEW_ADDR &&
1124 !is_cold_data(page) &&
1125 !IS_ATOMIC_WRITTEN_PAGE(page) &&
1126 need_inplace_update(inode))) {
1127 rewrite_data_page(fio);
1128 set_inode_flag(F2FS_I(inode), FI_UPDATE_WRITE);
1129 trace_f2fs_do_write_data_page(page, IPU);
1130 } else {
1131 write_data_page(&dn, fio);
1132 trace_f2fs_do_write_data_page(page, OPU);
1133 set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE);
1134 if (page->index == 0)
1135 set_inode_flag(F2FS_I(inode), FI_FIRST_BLOCK_WRITTEN);
1136 }
1137out_writepage:
1138 f2fs_put_dnode(&dn);
1139 return err;
1140}
1141
1142static int f2fs_write_data_page(struct page *page,
1143 struct writeback_control *wbc)
1144{
1145 struct inode *inode = page->mapping->host;
1146 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1147 loff_t i_size = i_size_read(inode);
1148 const pgoff_t end_index = ((unsigned long long) i_size)
1149 >> PAGE_SHIFT;
1150 unsigned offset = 0;
1151 bool need_balance_fs = false;
1152 int err = 0;
1153 struct f2fs_io_info fio = {
1154 .sbi = sbi,
1155 .type = DATA,
1156 .rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE,
1157 .page = page,
1158 .encrypted_page = NULL,
1159 };
1160
1161 trace_f2fs_writepage(page, DATA);
1162
1163 if (page->index < end_index)
1164 goto write;
1165
1166 /*
1167 * If the offset is out-of-range of file size,
1168 * this page does not have to be written to disk.
1169 */
1170 offset = i_size & (PAGE_SIZE - 1);
1171 if ((page->index >= end_index + 1) || !offset)
1172 goto out;
1173
1174 zero_user_segment(page, offset, PAGE_SIZE);
1175write:
1176 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1177 goto redirty_out;
1178 if (f2fs_is_drop_cache(inode))
1179 goto out;
1180 if (f2fs_is_volatile_file(inode) && !wbc->for_reclaim &&
1181 available_free_memory(sbi, BASE_CHECK))
1182 goto redirty_out;
1183
1184 /* Dentry blocks are controlled by checkpoint */
1185 if (S_ISDIR(inode->i_mode)) {
1186 if (unlikely(f2fs_cp_error(sbi)))
1187 goto redirty_out;
1188 err = do_write_data_page(&fio);
1189 goto done;
1190 }
1191
1192 /* we should bypass data pages to proceed the kworkder jobs */
1193 if (unlikely(f2fs_cp_error(sbi))) {
1194 SetPageError(page);
1195 goto out;
1196 }
1197
1198 if (!wbc->for_reclaim)
1199 need_balance_fs = true;
1200 else if (has_not_enough_free_secs(sbi, 0))
1201 goto redirty_out;
1202
1203 err = -EAGAIN;
1204 f2fs_lock_op(sbi);
1205 if (f2fs_has_inline_data(inode))
1206 err = f2fs_write_inline_data(inode, page);
1207 if (err == -EAGAIN)
1208 err = do_write_data_page(&fio);
1209 f2fs_unlock_op(sbi);
1210done:
1211 if (err && err != -ENOENT)
1212 goto redirty_out;
1213
1214 clear_cold_data(page);
1215out:
1216 inode_dec_dirty_pages(inode);
1217 if (err)
1218 ClearPageUptodate(page);
1219
1220 if (wbc->for_reclaim) {
1221 f2fs_submit_merged_bio_cond(sbi, NULL, page, 0, DATA, WRITE);
1222 remove_dirty_inode(inode);
1223 }
1224
1225 unlock_page(page);
1226 f2fs_balance_fs(sbi, need_balance_fs);
1227
1228 if (unlikely(f2fs_cp_error(sbi)))
1229 f2fs_submit_merged_bio(sbi, DATA, WRITE);
1230
1231 return 0;
1232
1233redirty_out:
1234 redirty_page_for_writepage(wbc, page);
1235 return AOP_WRITEPAGE_ACTIVATE;
1236}
1237
1238static int __f2fs_writepage(struct page *page, struct writeback_control *wbc,
1239 void *data)
1240{
1241 struct address_space *mapping = data;
1242 int ret = mapping->a_ops->writepage(page, wbc);
1243 mapping_set_error(mapping, ret);
1244 return ret;
1245}
1246
1247/*
1248 * This function was copied from write_cche_pages from mm/page-writeback.c.
1249 * The major change is making write step of cold data page separately from
1250 * warm/hot data page.
1251 */
1252static int f2fs_write_cache_pages(struct address_space *mapping,
1253 struct writeback_control *wbc, writepage_t writepage,
1254 void *data)
1255{
1256 int ret = 0;
1257 int done = 0;
1258 struct pagevec pvec;
1259 int nr_pages;
1260 pgoff_t uninitialized_var(writeback_index);
1261 pgoff_t index;
1262 pgoff_t end; /* Inclusive */
1263 pgoff_t done_index;
1264 int cycled;
1265 int range_whole = 0;
1266 int tag;
1267 int step = 0;
1268
1269 pagevec_init(&pvec, 0);
1270next:
1271 if (wbc->range_cyclic) {
1272 writeback_index = mapping->writeback_index; /* prev offset */
1273 index = writeback_index;
1274 if (index == 0)
1275 cycled = 1;
1276 else
1277 cycled = 0;
1278 end = -1;
1279 } else {
1280 index = wbc->range_start >> PAGE_SHIFT;
1281 end = wbc->range_end >> PAGE_SHIFT;
1282 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
1283 range_whole = 1;
1284 cycled = 1; /* ignore range_cyclic tests */
1285 }
1286 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1287 tag = PAGECACHE_TAG_TOWRITE;
1288 else
1289 tag = PAGECACHE_TAG_DIRTY;
1290retry:
1291 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1292 tag_pages_for_writeback(mapping, index, end);
1293 done_index = index;
1294 while (!done && (index <= end)) {
1295 int i;
1296
1297 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
1298 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1);
1299 if (nr_pages == 0)
1300 break;
1301
1302 for (i = 0; i < nr_pages; i++) {
1303 struct page *page = pvec.pages[i];
1304
1305 if (page->index > end) {
1306 done = 1;
1307 break;
1308 }
1309
1310 done_index = page->index;
1311
1312 lock_page(page);
1313
1314 if (unlikely(page->mapping != mapping)) {
1315continue_unlock:
1316 unlock_page(page);
1317 continue;
1318 }
1319
1320 if (!PageDirty(page)) {
1321 /* someone wrote it for us */
1322 goto continue_unlock;
1323 }
1324
1325 if (step == is_cold_data(page))
1326 goto continue_unlock;
1327
1328 if (PageWriteback(page)) {
1329 if (wbc->sync_mode != WB_SYNC_NONE)
1330 f2fs_wait_on_page_writeback(page,
1331 DATA, true);
1332 else
1333 goto continue_unlock;
1334 }
1335
1336 BUG_ON(PageWriteback(page));
1337 if (!clear_page_dirty_for_io(page))
1338 goto continue_unlock;
1339
1340 ret = (*writepage)(page, wbc, data);
1341 if (unlikely(ret)) {
1342 if (ret == AOP_WRITEPAGE_ACTIVATE) {
1343 unlock_page(page);
1344 ret = 0;
1345 } else {
1346 done_index = page->index + 1;
1347 done = 1;
1348 break;
1349 }
1350 }
1351
1352 if (--wbc->nr_to_write <= 0 &&
1353 wbc->sync_mode == WB_SYNC_NONE) {
1354 done = 1;
1355 break;
1356 }
1357 }
1358 pagevec_release(&pvec);
1359 cond_resched();
1360 }
1361
1362 if (step < 1) {
1363 step++;
1364 goto next;
1365 }
1366
1367 if (!cycled && !done) {
1368 cycled = 1;
1369 index = 0;
1370 end = writeback_index - 1;
1371 goto retry;
1372 }
1373 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
1374 mapping->writeback_index = done_index;
1375
1376 return ret;
1377}
1378
1379static int f2fs_write_data_pages(struct address_space *mapping,
1380 struct writeback_control *wbc)
1381{
1382 struct inode *inode = mapping->host;
1383 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1384 bool locked = false;
1385 int ret;
1386 long diff;
1387
1388 /* deal with chardevs and other special file */
1389 if (!mapping->a_ops->writepage)
1390 return 0;
1391
1392 /* skip writing if there is no dirty page in this inode */
1393 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
1394 return 0;
1395
1396 if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE &&
1397 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
1398 available_free_memory(sbi, DIRTY_DENTS))
1399 goto skip_write;
1400
1401 /* skip writing during file defragment */
1402 if (is_inode_flag_set(F2FS_I(inode), FI_DO_DEFRAG))
1403 goto skip_write;
1404
1405 /* during POR, we don't need to trigger writepage at all. */
1406 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1407 goto skip_write;
1408
1409 trace_f2fs_writepages(mapping->host, wbc, DATA);
1410
1411 diff = nr_pages_to_write(sbi, DATA, wbc);
1412
1413 if (!S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_ALL) {
1414 mutex_lock(&sbi->writepages);
1415 locked = true;
1416 }
1417 ret = f2fs_write_cache_pages(mapping, wbc, __f2fs_writepage, mapping);
1418 f2fs_submit_merged_bio_cond(sbi, inode, NULL, 0, DATA, WRITE);
1419 if (locked)
1420 mutex_unlock(&sbi->writepages);
1421
1422 remove_dirty_inode(inode);
1423
1424 wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff);
1425 return ret;
1426
1427skip_write:
1428 wbc->pages_skipped += get_dirty_pages(inode);
1429 trace_f2fs_writepages(mapping->host, wbc, DATA);
1430 return 0;
1431}
1432
1433static void f2fs_write_failed(struct address_space *mapping, loff_t to)
1434{
1435 struct inode *inode = mapping->host;
1436 loff_t i_size = i_size_read(inode);
1437
1438 if (to > i_size) {
1439 truncate_pagecache(inode, i_size);
1440 truncate_blocks(inode, i_size, true);
1441 }
1442}
1443
1444static int prepare_write_begin(struct f2fs_sb_info *sbi,
1445 struct page *page, loff_t pos, unsigned len,
1446 block_t *blk_addr, bool *node_changed)
1447{
1448 struct inode *inode = page->mapping->host;
1449 pgoff_t index = page->index;
1450 struct dnode_of_data dn;
1451 struct page *ipage;
1452 bool locked = false;
1453 struct extent_info ei;
1454 int err = 0;
1455
1456 /*
1457 * we already allocated all the blocks, so we don't need to get
1458 * the block addresses when there is no need to fill the page.
1459 */
1460 if (!f2fs_has_inline_data(inode) && !f2fs_encrypted_inode(inode) &&
1461 len == PAGE_SIZE)
1462 return 0;
1463
1464 if (f2fs_has_inline_data(inode) ||
1465 (pos & PAGE_MASK) >= i_size_read(inode)) {
1466 f2fs_lock_op(sbi);
1467 locked = true;
1468 }
1469restart:
1470 /* check inline_data */
1471 ipage = get_node_page(sbi, inode->i_ino);
1472 if (IS_ERR(ipage)) {
1473 err = PTR_ERR(ipage);
1474 goto unlock_out;
1475 }
1476
1477 set_new_dnode(&dn, inode, ipage, ipage, 0);
1478
1479 if (f2fs_has_inline_data(inode)) {
1480 if (pos + len <= MAX_INLINE_DATA) {
1481 read_inline_data(page, ipage);
1482 set_inode_flag(F2FS_I(inode), FI_DATA_EXIST);
1483 set_inline_node(ipage);
1484 } else {
1485 err = f2fs_convert_inline_page(&dn, page);
1486 if (err)
1487 goto out;
1488 if (dn.data_blkaddr == NULL_ADDR)
1489 err = f2fs_get_block(&dn, index);
1490 }
1491 } else if (locked) {
1492 err = f2fs_get_block(&dn, index);
1493 } else {
1494 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
1495 dn.data_blkaddr = ei.blk + index - ei.fofs;
1496 } else {
1497 /* hole case */
1498 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
1499 if (err || (!err && dn.data_blkaddr == NULL_ADDR)) {
1500 f2fs_put_dnode(&dn);
1501 f2fs_lock_op(sbi);
1502 locked = true;
1503 goto restart;
1504 }
1505 }
1506 }
1507
1508 /* convert_inline_page can make node_changed */
1509 *blk_addr = dn.data_blkaddr;
1510 *node_changed = dn.node_changed;
1511out:
1512 f2fs_put_dnode(&dn);
1513unlock_out:
1514 if (locked)
1515 f2fs_unlock_op(sbi);
1516 return err;
1517}
1518
1519static int f2fs_write_begin(struct file *file, struct address_space *mapping,
1520 loff_t pos, unsigned len, unsigned flags,
1521 struct page **pagep, void **fsdata)
1522{
1523 struct inode *inode = mapping->host;
1524 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1525 struct page *page = NULL;
1526 pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
1527 bool need_balance = false;
1528 block_t blkaddr = NULL_ADDR;
1529 int err = 0;
1530
1531 trace_f2fs_write_begin(inode, pos, len, flags);
1532
1533 /*
1534 * We should check this at this moment to avoid deadlock on inode page
1535 * and #0 page. The locking rule for inline_data conversion should be:
1536 * lock_page(page #0) -> lock_page(inode_page)
1537 */
1538 if (index != 0) {
1539 err = f2fs_convert_inline_inode(inode);
1540 if (err)
1541 goto fail;
1542 }
1543repeat:
1544 page = grab_cache_page_write_begin(mapping, index, flags);
1545 if (!page) {
1546 err = -ENOMEM;
1547 goto fail;
1548 }
1549
1550 *pagep = page;
1551
1552 err = prepare_write_begin(sbi, page, pos, len,
1553 &blkaddr, &need_balance);
1554 if (err)
1555 goto fail;
1556
1557 if (need_balance && has_not_enough_free_secs(sbi, 0)) {
1558 unlock_page(page);
1559 f2fs_balance_fs(sbi, true);
1560 lock_page(page);
1561 if (page->mapping != mapping) {
1562 /* The page got truncated from under us */
1563 f2fs_put_page(page, 1);
1564 goto repeat;
1565 }
1566 }
1567
1568 f2fs_wait_on_page_writeback(page, DATA, false);
1569
1570 /* wait for GCed encrypted page writeback */
1571 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
1572 f2fs_wait_on_encrypted_page_writeback(sbi, blkaddr);
1573
1574 if (len == PAGE_SIZE)
1575 goto out_update;
1576 if (PageUptodate(page))
1577 goto out_clear;
1578
1579 if ((pos & PAGE_MASK) >= i_size_read(inode)) {
1580 unsigned start = pos & (PAGE_SIZE - 1);
1581 unsigned end = start + len;
1582
1583 /* Reading beyond i_size is simple: memset to zero */
1584 zero_user_segments(page, 0, start, end, PAGE_SIZE);
1585 goto out_update;
1586 }
1587
1588 if (blkaddr == NEW_ADDR) {
1589 zero_user_segment(page, 0, PAGE_SIZE);
1590 } else {
1591 struct f2fs_io_info fio = {
1592 .sbi = sbi,
1593 .type = DATA,
1594 .rw = READ_SYNC,
1595 .old_blkaddr = blkaddr,
1596 .new_blkaddr = blkaddr,
1597 .page = page,
1598 .encrypted_page = NULL,
1599 };
1600 err = f2fs_submit_page_bio(&fio);
1601 if (err)
1602 goto fail;
1603
1604 lock_page(page);
1605 if (unlikely(!PageUptodate(page))) {
1606 err = -EIO;
1607 goto fail;
1608 }
1609 if (unlikely(page->mapping != mapping)) {
1610 f2fs_put_page(page, 1);
1611 goto repeat;
1612 }
1613
1614 /* avoid symlink page */
1615 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) {
1616 err = fscrypt_decrypt_page(page);
1617 if (err)
1618 goto fail;
1619 }
1620 }
1621out_update:
1622 SetPageUptodate(page);
1623out_clear:
1624 clear_cold_data(page);
1625 return 0;
1626
1627fail:
1628 f2fs_put_page(page, 1);
1629 f2fs_write_failed(mapping, pos + len);
1630 return err;
1631}
1632
1633static int f2fs_write_end(struct file *file,
1634 struct address_space *mapping,
1635 loff_t pos, unsigned len, unsigned copied,
1636 struct page *page, void *fsdata)
1637{
1638 struct inode *inode = page->mapping->host;
1639
1640 trace_f2fs_write_end(inode, pos, len, copied);
1641
1642 set_page_dirty(page);
1643
1644 if (pos + copied > i_size_read(inode)) {
1645 i_size_write(inode, pos + copied);
1646 mark_inode_dirty(inode);
1647 }
1648
1649 f2fs_put_page(page, 1);
1650 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
1651 return copied;
1652}
1653
1654static int check_direct_IO(struct inode *inode, struct iov_iter *iter,
1655 loff_t offset)
1656{
1657 unsigned blocksize_mask = inode->i_sb->s_blocksize - 1;
1658
1659 if (offset & blocksize_mask)
1660 return -EINVAL;
1661
1662 if (iov_iter_alignment(iter) & blocksize_mask)
1663 return -EINVAL;
1664
1665 return 0;
1666}
1667
1668static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
1669 loff_t offset)
1670{
1671 struct address_space *mapping = iocb->ki_filp->f_mapping;
1672 struct inode *inode = mapping->host;
1673 size_t count = iov_iter_count(iter);
1674 int err;
1675
1676 err = check_direct_IO(inode, iter, offset);
1677 if (err)
1678 return err;
1679
1680 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
1681 return 0;
1682
1683 trace_f2fs_direct_IO_enter(inode, offset, count, iov_iter_rw(iter));
1684
1685 err = blockdev_direct_IO(iocb, inode, iter, offset, get_data_block_dio);
1686 if (err < 0 && iov_iter_rw(iter) == WRITE)
1687 f2fs_write_failed(mapping, offset + count);
1688
1689 trace_f2fs_direct_IO_exit(inode, offset, count, iov_iter_rw(iter), err);
1690
1691 return err;
1692}
1693
1694void f2fs_invalidate_page(struct page *page, unsigned int offset,
1695 unsigned int length)
1696{
1697 struct inode *inode = page->mapping->host;
1698 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1699
1700 if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
1701 (offset % PAGE_SIZE || length != PAGE_SIZE))
1702 return;
1703
1704 if (PageDirty(page)) {
1705 if (inode->i_ino == F2FS_META_INO(sbi))
1706 dec_page_count(sbi, F2FS_DIRTY_META);
1707 else if (inode->i_ino == F2FS_NODE_INO(sbi))
1708 dec_page_count(sbi, F2FS_DIRTY_NODES);
1709 else
1710 inode_dec_dirty_pages(inode);
1711 }
1712
1713 /* This is atomic written page, keep Private */
1714 if (IS_ATOMIC_WRITTEN_PAGE(page))
1715 return;
1716
1717 ClearPagePrivate(page);
1718}
1719
1720int f2fs_release_page(struct page *page, gfp_t wait)
1721{
1722 /* If this is dirty page, keep PagePrivate */
1723 if (PageDirty(page))
1724 return 0;
1725
1726 /* This is atomic written page, keep Private */
1727 if (IS_ATOMIC_WRITTEN_PAGE(page))
1728 return 0;
1729
1730 ClearPagePrivate(page);
1731 return 1;
1732}
1733
1734static int f2fs_set_data_page_dirty(struct page *page)
1735{
1736 struct address_space *mapping = page->mapping;
1737 struct inode *inode = mapping->host;
1738
1739 trace_f2fs_set_page_dirty(page, DATA);
1740
1741 SetPageUptodate(page);
1742
1743 if (f2fs_is_atomic_file(inode)) {
1744 if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
1745 register_inmem_page(inode, page);
1746 return 1;
1747 }
1748 /*
1749 * Previously, this page has been registered, we just
1750 * return here.
1751 */
1752 return 0;
1753 }
1754
1755 if (!PageDirty(page)) {
1756 __set_page_dirty_nobuffers(page);
1757 update_dirty_page(inode, page);
1758 return 1;
1759 }
1760 return 0;
1761}
1762
1763static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
1764{
1765 struct inode *inode = mapping->host;
1766
1767 if (f2fs_has_inline_data(inode))
1768 return 0;
1769
1770 /* make sure allocating whole blocks */
1771 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
1772 filemap_write_and_wait(mapping);
1773
1774 return generic_block_bmap(mapping, block, get_data_block_bmap);
1775}
1776
1777const struct address_space_operations f2fs_dblock_aops = {
1778 .readpage = f2fs_read_data_page,
1779 .readpages = f2fs_read_data_pages,
1780 .writepage = f2fs_write_data_page,
1781 .writepages = f2fs_write_data_pages,
1782 .write_begin = f2fs_write_begin,
1783 .write_end = f2fs_write_end,
1784 .set_page_dirty = f2fs_set_data_page_dirty,
1785 .invalidatepage = f2fs_invalidate_page,
1786 .releasepage = f2fs_release_page,
1787 .direct_IO = f2fs_direct_IO,
1788 .bmap = f2fs_bmap,
1789};