Loading...
1/*
2 * Resizable virtual memory filesystem for Linux.
3 *
4 * Copyright (C) 2000 Linus Torvalds.
5 * 2000 Transmeta Corp.
6 * 2000-2001 Christoph Rohland
7 * 2000-2001 SAP AG
8 * 2002 Red Hat Inc.
9 * Copyright (C) 2002-2011 Hugh Dickins.
10 * Copyright (C) 2011 Google Inc.
11 * Copyright (C) 2002-2005 VERITAS Software Corporation.
12 * Copyright (C) 2004 Andi Kleen, SuSE Labs
13 *
14 * Extended attribute support for tmpfs:
15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17 *
18 * tiny-shmem:
19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
20 *
21 * This file is released under the GPL.
22 */
23
24#include <linux/fs.h>
25#include <linux/init.h>
26#include <linux/vfs.h>
27#include <linux/mount.h>
28#include <linux/ramfs.h>
29#include <linux/pagemap.h>
30#include <linux/file.h>
31#include <linux/mm.h>
32#include <linux/export.h>
33#include <linux/swap.h>
34#include <linux/uio.h>
35
36static struct vfsmount *shm_mnt;
37
38#ifdef CONFIG_SHMEM
39/*
40 * This virtual memory filesystem is heavily based on the ramfs. It
41 * extends ramfs by the ability to use swap and honor resource limits
42 * which makes it a completely usable filesystem.
43 */
44
45#include <linux/xattr.h>
46#include <linux/exportfs.h>
47#include <linux/posix_acl.h>
48#include <linux/posix_acl_xattr.h>
49#include <linux/mman.h>
50#include <linux/string.h>
51#include <linux/slab.h>
52#include <linux/backing-dev.h>
53#include <linux/shmem_fs.h>
54#include <linux/writeback.h>
55#include <linux/blkdev.h>
56#include <linux/pagevec.h>
57#include <linux/percpu_counter.h>
58#include <linux/falloc.h>
59#include <linux/splice.h>
60#include <linux/security.h>
61#include <linux/swapops.h>
62#include <linux/mempolicy.h>
63#include <linux/namei.h>
64#include <linux/ctype.h>
65#include <linux/migrate.h>
66#include <linux/highmem.h>
67#include <linux/seq_file.h>
68#include <linux/magic.h>
69#include <linux/syscalls.h>
70#include <linux/fcntl.h>
71#include <uapi/linux/memfd.h>
72
73#include <asm/uaccess.h>
74#include <asm/pgtable.h>
75
76#include "internal.h"
77
78#define BLOCKS_PER_PAGE (PAGE_SIZE/512)
79#define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT)
80
81/* Pretend that each entry is of this size in directory's i_size */
82#define BOGO_DIRENT_SIZE 20
83
84/* Symlink up to this size is kmalloc'ed instead of using a swappable page */
85#define SHORT_SYMLINK_LEN 128
86
87/*
88 * shmem_fallocate communicates with shmem_fault or shmem_writepage via
89 * inode->i_private (with i_mutex making sure that it has only one user at
90 * a time): we would prefer not to enlarge the shmem inode just for that.
91 */
92struct shmem_falloc {
93 wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
94 pgoff_t start; /* start of range currently being fallocated */
95 pgoff_t next; /* the next page offset to be fallocated */
96 pgoff_t nr_falloced; /* how many new pages have been fallocated */
97 pgoff_t nr_unswapped; /* how often writepage refused to swap out */
98};
99
100/* Flag allocation requirements to shmem_getpage */
101enum sgp_type {
102 SGP_READ, /* don't exceed i_size, don't allocate page */
103 SGP_CACHE, /* don't exceed i_size, may allocate page */
104 SGP_DIRTY, /* like SGP_CACHE, but set new page dirty */
105 SGP_WRITE, /* may exceed i_size, may allocate !Uptodate page */
106 SGP_FALLOC, /* like SGP_WRITE, but make existing page Uptodate */
107};
108
109#ifdef CONFIG_TMPFS
110static unsigned long shmem_default_max_blocks(void)
111{
112 return totalram_pages / 2;
113}
114
115static unsigned long shmem_default_max_inodes(void)
116{
117 return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
118}
119#endif
120
121static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
122static int shmem_replace_page(struct page **pagep, gfp_t gfp,
123 struct shmem_inode_info *info, pgoff_t index);
124static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
125 struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type);
126
127static inline int shmem_getpage(struct inode *inode, pgoff_t index,
128 struct page **pagep, enum sgp_type sgp, int *fault_type)
129{
130 return shmem_getpage_gfp(inode, index, pagep, sgp,
131 mapping_gfp_mask(inode->i_mapping), fault_type);
132}
133
134static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
135{
136 return sb->s_fs_info;
137}
138
139/*
140 * shmem_file_setup pre-accounts the whole fixed size of a VM object,
141 * for shared memory and for shared anonymous (/dev/zero) mappings
142 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
143 * consistent with the pre-accounting of private mappings ...
144 */
145static inline int shmem_acct_size(unsigned long flags, loff_t size)
146{
147 return (flags & VM_NORESERVE) ?
148 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
149}
150
151static inline void shmem_unacct_size(unsigned long flags, loff_t size)
152{
153 if (!(flags & VM_NORESERVE))
154 vm_unacct_memory(VM_ACCT(size));
155}
156
157static inline int shmem_reacct_size(unsigned long flags,
158 loff_t oldsize, loff_t newsize)
159{
160 if (!(flags & VM_NORESERVE)) {
161 if (VM_ACCT(newsize) > VM_ACCT(oldsize))
162 return security_vm_enough_memory_mm(current->mm,
163 VM_ACCT(newsize) - VM_ACCT(oldsize));
164 else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
165 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
166 }
167 return 0;
168}
169
170/*
171 * ... whereas tmpfs objects are accounted incrementally as
172 * pages are allocated, in order to allow huge sparse files.
173 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
174 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
175 */
176static inline int shmem_acct_block(unsigned long flags)
177{
178 return (flags & VM_NORESERVE) ?
179 security_vm_enough_memory_mm(current->mm, VM_ACCT(PAGE_SIZE)) : 0;
180}
181
182static inline void shmem_unacct_blocks(unsigned long flags, long pages)
183{
184 if (flags & VM_NORESERVE)
185 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
186}
187
188static const struct super_operations shmem_ops;
189static const struct address_space_operations shmem_aops;
190static const struct file_operations shmem_file_operations;
191static const struct inode_operations shmem_inode_operations;
192static const struct inode_operations shmem_dir_inode_operations;
193static const struct inode_operations shmem_special_inode_operations;
194static const struct vm_operations_struct shmem_vm_ops;
195
196static LIST_HEAD(shmem_swaplist);
197static DEFINE_MUTEX(shmem_swaplist_mutex);
198
199static int shmem_reserve_inode(struct super_block *sb)
200{
201 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
202 if (sbinfo->max_inodes) {
203 spin_lock(&sbinfo->stat_lock);
204 if (!sbinfo->free_inodes) {
205 spin_unlock(&sbinfo->stat_lock);
206 return -ENOSPC;
207 }
208 sbinfo->free_inodes--;
209 spin_unlock(&sbinfo->stat_lock);
210 }
211 return 0;
212}
213
214static void shmem_free_inode(struct super_block *sb)
215{
216 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
217 if (sbinfo->max_inodes) {
218 spin_lock(&sbinfo->stat_lock);
219 sbinfo->free_inodes++;
220 spin_unlock(&sbinfo->stat_lock);
221 }
222}
223
224/**
225 * shmem_recalc_inode - recalculate the block usage of an inode
226 * @inode: inode to recalc
227 *
228 * We have to calculate the free blocks since the mm can drop
229 * undirtied hole pages behind our back.
230 *
231 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped
232 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
233 *
234 * It has to be called with the spinlock held.
235 */
236static void shmem_recalc_inode(struct inode *inode)
237{
238 struct shmem_inode_info *info = SHMEM_I(inode);
239 long freed;
240
241 freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
242 if (freed > 0) {
243 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
244 if (sbinfo->max_blocks)
245 percpu_counter_add(&sbinfo->used_blocks, -freed);
246 info->alloced -= freed;
247 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
248 shmem_unacct_blocks(info->flags, freed);
249 }
250}
251
252/*
253 * Replace item expected in radix tree by a new item, while holding tree lock.
254 */
255static int shmem_radix_tree_replace(struct address_space *mapping,
256 pgoff_t index, void *expected, void *replacement)
257{
258 void **pslot;
259 void *item;
260
261 VM_BUG_ON(!expected);
262 VM_BUG_ON(!replacement);
263 pslot = radix_tree_lookup_slot(&mapping->page_tree, index);
264 if (!pslot)
265 return -ENOENT;
266 item = radix_tree_deref_slot_protected(pslot, &mapping->tree_lock);
267 if (item != expected)
268 return -ENOENT;
269 radix_tree_replace_slot(pslot, replacement);
270 return 0;
271}
272
273/*
274 * Sometimes, before we decide whether to proceed or to fail, we must check
275 * that an entry was not already brought back from swap by a racing thread.
276 *
277 * Checking page is not enough: by the time a SwapCache page is locked, it
278 * might be reused, and again be SwapCache, using the same swap as before.
279 */
280static bool shmem_confirm_swap(struct address_space *mapping,
281 pgoff_t index, swp_entry_t swap)
282{
283 void *item;
284
285 rcu_read_lock();
286 item = radix_tree_lookup(&mapping->page_tree, index);
287 rcu_read_unlock();
288 return item == swp_to_radix_entry(swap);
289}
290
291/*
292 * Like add_to_page_cache_locked, but error if expected item has gone.
293 */
294static int shmem_add_to_page_cache(struct page *page,
295 struct address_space *mapping,
296 pgoff_t index, void *expected)
297{
298 int error;
299
300 VM_BUG_ON_PAGE(!PageLocked(page), page);
301 VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
302
303 get_page(page);
304 page->mapping = mapping;
305 page->index = index;
306
307 spin_lock_irq(&mapping->tree_lock);
308 if (!expected)
309 error = radix_tree_insert(&mapping->page_tree, index, page);
310 else
311 error = shmem_radix_tree_replace(mapping, index, expected,
312 page);
313 if (!error) {
314 mapping->nrpages++;
315 __inc_zone_page_state(page, NR_FILE_PAGES);
316 __inc_zone_page_state(page, NR_SHMEM);
317 spin_unlock_irq(&mapping->tree_lock);
318 } else {
319 page->mapping = NULL;
320 spin_unlock_irq(&mapping->tree_lock);
321 put_page(page);
322 }
323 return error;
324}
325
326/*
327 * Like delete_from_page_cache, but substitutes swap for page.
328 */
329static void shmem_delete_from_page_cache(struct page *page, void *radswap)
330{
331 struct address_space *mapping = page->mapping;
332 int error;
333
334 spin_lock_irq(&mapping->tree_lock);
335 error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
336 page->mapping = NULL;
337 mapping->nrpages--;
338 __dec_zone_page_state(page, NR_FILE_PAGES);
339 __dec_zone_page_state(page, NR_SHMEM);
340 spin_unlock_irq(&mapping->tree_lock);
341 put_page(page);
342 BUG_ON(error);
343}
344
345/*
346 * Remove swap entry from radix tree, free the swap and its page cache.
347 */
348static int shmem_free_swap(struct address_space *mapping,
349 pgoff_t index, void *radswap)
350{
351 void *old;
352
353 spin_lock_irq(&mapping->tree_lock);
354 old = radix_tree_delete_item(&mapping->page_tree, index, radswap);
355 spin_unlock_irq(&mapping->tree_lock);
356 if (old != radswap)
357 return -ENOENT;
358 free_swap_and_cache(radix_to_swp_entry(radswap));
359 return 0;
360}
361
362/*
363 * Determine (in bytes) how many of the shmem object's pages mapped by the
364 * given offsets are swapped out.
365 *
366 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU,
367 * as long as the inode doesn't go away and racy results are not a problem.
368 */
369unsigned long shmem_partial_swap_usage(struct address_space *mapping,
370 pgoff_t start, pgoff_t end)
371{
372 struct radix_tree_iter iter;
373 void **slot;
374 struct page *page;
375 unsigned long swapped = 0;
376
377 rcu_read_lock();
378
379 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
380 if (iter.index >= end)
381 break;
382
383 page = radix_tree_deref_slot(slot);
384
385 if (radix_tree_deref_retry(page)) {
386 slot = radix_tree_iter_retry(&iter);
387 continue;
388 }
389
390 if (radix_tree_exceptional_entry(page))
391 swapped++;
392
393 if (need_resched()) {
394 cond_resched_rcu();
395 slot = radix_tree_iter_next(&iter);
396 }
397 }
398
399 rcu_read_unlock();
400
401 return swapped << PAGE_SHIFT;
402}
403
404/*
405 * Determine (in bytes) how many of the shmem object's pages mapped by the
406 * given vma is swapped out.
407 *
408 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU,
409 * as long as the inode doesn't go away and racy results are not a problem.
410 */
411unsigned long shmem_swap_usage(struct vm_area_struct *vma)
412{
413 struct inode *inode = file_inode(vma->vm_file);
414 struct shmem_inode_info *info = SHMEM_I(inode);
415 struct address_space *mapping = inode->i_mapping;
416 unsigned long swapped;
417
418 /* Be careful as we don't hold info->lock */
419 swapped = READ_ONCE(info->swapped);
420
421 /*
422 * The easier cases are when the shmem object has nothing in swap, or
423 * the vma maps it whole. Then we can simply use the stats that we
424 * already track.
425 */
426 if (!swapped)
427 return 0;
428
429 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
430 return swapped << PAGE_SHIFT;
431
432 /* Here comes the more involved part */
433 return shmem_partial_swap_usage(mapping,
434 linear_page_index(vma, vma->vm_start),
435 linear_page_index(vma, vma->vm_end));
436}
437
438/*
439 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
440 */
441void shmem_unlock_mapping(struct address_space *mapping)
442{
443 struct pagevec pvec;
444 pgoff_t indices[PAGEVEC_SIZE];
445 pgoff_t index = 0;
446
447 pagevec_init(&pvec, 0);
448 /*
449 * Minor point, but we might as well stop if someone else SHM_LOCKs it.
450 */
451 while (!mapping_unevictable(mapping)) {
452 /*
453 * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it
454 * has finished, if it hits a row of PAGEVEC_SIZE swap entries.
455 */
456 pvec.nr = find_get_entries(mapping, index,
457 PAGEVEC_SIZE, pvec.pages, indices);
458 if (!pvec.nr)
459 break;
460 index = indices[pvec.nr - 1] + 1;
461 pagevec_remove_exceptionals(&pvec);
462 check_move_unevictable_pages(pvec.pages, pvec.nr);
463 pagevec_release(&pvec);
464 cond_resched();
465 }
466}
467
468/*
469 * Remove range of pages and swap entries from radix tree, and free them.
470 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
471 */
472static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
473 bool unfalloc)
474{
475 struct address_space *mapping = inode->i_mapping;
476 struct shmem_inode_info *info = SHMEM_I(inode);
477 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
478 pgoff_t end = (lend + 1) >> PAGE_SHIFT;
479 unsigned int partial_start = lstart & (PAGE_SIZE - 1);
480 unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1);
481 struct pagevec pvec;
482 pgoff_t indices[PAGEVEC_SIZE];
483 long nr_swaps_freed = 0;
484 pgoff_t index;
485 int i;
486
487 if (lend == -1)
488 end = -1; /* unsigned, so actually very big */
489
490 pagevec_init(&pvec, 0);
491 index = start;
492 while (index < end) {
493 pvec.nr = find_get_entries(mapping, index,
494 min(end - index, (pgoff_t)PAGEVEC_SIZE),
495 pvec.pages, indices);
496 if (!pvec.nr)
497 break;
498 for (i = 0; i < pagevec_count(&pvec); i++) {
499 struct page *page = pvec.pages[i];
500
501 index = indices[i];
502 if (index >= end)
503 break;
504
505 if (radix_tree_exceptional_entry(page)) {
506 if (unfalloc)
507 continue;
508 nr_swaps_freed += !shmem_free_swap(mapping,
509 index, page);
510 continue;
511 }
512
513 if (!trylock_page(page))
514 continue;
515 if (!unfalloc || !PageUptodate(page)) {
516 if (page->mapping == mapping) {
517 VM_BUG_ON_PAGE(PageWriteback(page), page);
518 truncate_inode_page(mapping, page);
519 }
520 }
521 unlock_page(page);
522 }
523 pagevec_remove_exceptionals(&pvec);
524 pagevec_release(&pvec);
525 cond_resched();
526 index++;
527 }
528
529 if (partial_start) {
530 struct page *page = NULL;
531 shmem_getpage(inode, start - 1, &page, SGP_READ, NULL);
532 if (page) {
533 unsigned int top = PAGE_SIZE;
534 if (start > end) {
535 top = partial_end;
536 partial_end = 0;
537 }
538 zero_user_segment(page, partial_start, top);
539 set_page_dirty(page);
540 unlock_page(page);
541 put_page(page);
542 }
543 }
544 if (partial_end) {
545 struct page *page = NULL;
546 shmem_getpage(inode, end, &page, SGP_READ, NULL);
547 if (page) {
548 zero_user_segment(page, 0, partial_end);
549 set_page_dirty(page);
550 unlock_page(page);
551 put_page(page);
552 }
553 }
554 if (start >= end)
555 return;
556
557 index = start;
558 while (index < end) {
559 cond_resched();
560
561 pvec.nr = find_get_entries(mapping, index,
562 min(end - index, (pgoff_t)PAGEVEC_SIZE),
563 pvec.pages, indices);
564 if (!pvec.nr) {
565 /* If all gone or hole-punch or unfalloc, we're done */
566 if (index == start || end != -1)
567 break;
568 /* But if truncating, restart to make sure all gone */
569 index = start;
570 continue;
571 }
572 for (i = 0; i < pagevec_count(&pvec); i++) {
573 struct page *page = pvec.pages[i];
574
575 index = indices[i];
576 if (index >= end)
577 break;
578
579 if (radix_tree_exceptional_entry(page)) {
580 if (unfalloc)
581 continue;
582 if (shmem_free_swap(mapping, index, page)) {
583 /* Swap was replaced by page: retry */
584 index--;
585 break;
586 }
587 nr_swaps_freed++;
588 continue;
589 }
590
591 lock_page(page);
592 if (!unfalloc || !PageUptodate(page)) {
593 if (page->mapping == mapping) {
594 VM_BUG_ON_PAGE(PageWriteback(page), page);
595 truncate_inode_page(mapping, page);
596 } else {
597 /* Page was replaced by swap: retry */
598 unlock_page(page);
599 index--;
600 break;
601 }
602 }
603 unlock_page(page);
604 }
605 pagevec_remove_exceptionals(&pvec);
606 pagevec_release(&pvec);
607 index++;
608 }
609
610 spin_lock(&info->lock);
611 info->swapped -= nr_swaps_freed;
612 shmem_recalc_inode(inode);
613 spin_unlock(&info->lock);
614}
615
616void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
617{
618 shmem_undo_range(inode, lstart, lend, false);
619 inode->i_ctime = inode->i_mtime = CURRENT_TIME;
620}
621EXPORT_SYMBOL_GPL(shmem_truncate_range);
622
623static int shmem_getattr(struct vfsmount *mnt, struct dentry *dentry,
624 struct kstat *stat)
625{
626 struct inode *inode = dentry->d_inode;
627 struct shmem_inode_info *info = SHMEM_I(inode);
628
629 if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
630 spin_lock(&info->lock);
631 shmem_recalc_inode(inode);
632 spin_unlock(&info->lock);
633 }
634 generic_fillattr(inode, stat);
635 return 0;
636}
637
638static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
639{
640 struct inode *inode = d_inode(dentry);
641 struct shmem_inode_info *info = SHMEM_I(inode);
642 int error;
643
644 error = inode_change_ok(inode, attr);
645 if (error)
646 return error;
647
648 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
649 loff_t oldsize = inode->i_size;
650 loff_t newsize = attr->ia_size;
651
652 /* protected by i_mutex */
653 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
654 (newsize > oldsize && (info->seals & F_SEAL_GROW)))
655 return -EPERM;
656
657 if (newsize != oldsize) {
658 error = shmem_reacct_size(SHMEM_I(inode)->flags,
659 oldsize, newsize);
660 if (error)
661 return error;
662 i_size_write(inode, newsize);
663 inode->i_ctime = inode->i_mtime = CURRENT_TIME;
664 }
665 if (newsize <= oldsize) {
666 loff_t holebegin = round_up(newsize, PAGE_SIZE);
667 if (oldsize > holebegin)
668 unmap_mapping_range(inode->i_mapping,
669 holebegin, 0, 1);
670 if (info->alloced)
671 shmem_truncate_range(inode,
672 newsize, (loff_t)-1);
673 /* unmap again to remove racily COWed private pages */
674 if (oldsize > holebegin)
675 unmap_mapping_range(inode->i_mapping,
676 holebegin, 0, 1);
677 }
678 }
679
680 setattr_copy(inode, attr);
681 if (attr->ia_valid & ATTR_MODE)
682 error = posix_acl_chmod(inode, inode->i_mode);
683 return error;
684}
685
686static void shmem_evict_inode(struct inode *inode)
687{
688 struct shmem_inode_info *info = SHMEM_I(inode);
689
690 if (inode->i_mapping->a_ops == &shmem_aops) {
691 shmem_unacct_size(info->flags, inode->i_size);
692 inode->i_size = 0;
693 shmem_truncate_range(inode, 0, (loff_t)-1);
694 if (!list_empty(&info->swaplist)) {
695 mutex_lock(&shmem_swaplist_mutex);
696 list_del_init(&info->swaplist);
697 mutex_unlock(&shmem_swaplist_mutex);
698 }
699 }
700
701 simple_xattrs_free(&info->xattrs);
702 WARN_ON(inode->i_blocks);
703 shmem_free_inode(inode->i_sb);
704 clear_inode(inode);
705}
706
707/*
708 * If swap found in inode, free it and move page from swapcache to filecache.
709 */
710static int shmem_unuse_inode(struct shmem_inode_info *info,
711 swp_entry_t swap, struct page **pagep)
712{
713 struct address_space *mapping = info->vfs_inode.i_mapping;
714 void *radswap;
715 pgoff_t index;
716 gfp_t gfp;
717 int error = 0;
718
719 radswap = swp_to_radix_entry(swap);
720 index = radix_tree_locate_item(&mapping->page_tree, radswap);
721 if (index == -1)
722 return -EAGAIN; /* tell shmem_unuse we found nothing */
723
724 /*
725 * Move _head_ to start search for next from here.
726 * But be careful: shmem_evict_inode checks list_empty without taking
727 * mutex, and there's an instant in list_move_tail when info->swaplist
728 * would appear empty, if it were the only one on shmem_swaplist.
729 */
730 if (shmem_swaplist.next != &info->swaplist)
731 list_move_tail(&shmem_swaplist, &info->swaplist);
732
733 gfp = mapping_gfp_mask(mapping);
734 if (shmem_should_replace_page(*pagep, gfp)) {
735 mutex_unlock(&shmem_swaplist_mutex);
736 error = shmem_replace_page(pagep, gfp, info, index);
737 mutex_lock(&shmem_swaplist_mutex);
738 /*
739 * We needed to drop mutex to make that restrictive page
740 * allocation, but the inode might have been freed while we
741 * dropped it: although a racing shmem_evict_inode() cannot
742 * complete without emptying the radix_tree, our page lock
743 * on this swapcache page is not enough to prevent that -
744 * free_swap_and_cache() of our swap entry will only
745 * trylock_page(), removing swap from radix_tree whatever.
746 *
747 * We must not proceed to shmem_add_to_page_cache() if the
748 * inode has been freed, but of course we cannot rely on
749 * inode or mapping or info to check that. However, we can
750 * safely check if our swap entry is still in use (and here
751 * it can't have got reused for another page): if it's still
752 * in use, then the inode cannot have been freed yet, and we
753 * can safely proceed (if it's no longer in use, that tells
754 * nothing about the inode, but we don't need to unuse swap).
755 */
756 if (!page_swapcount(*pagep))
757 error = -ENOENT;
758 }
759
760 /*
761 * We rely on shmem_swaplist_mutex, not only to protect the swaplist,
762 * but also to hold up shmem_evict_inode(): so inode cannot be freed
763 * beneath us (pagelock doesn't help until the page is in pagecache).
764 */
765 if (!error)
766 error = shmem_add_to_page_cache(*pagep, mapping, index,
767 radswap);
768 if (error != -ENOMEM) {
769 /*
770 * Truncation and eviction use free_swap_and_cache(), which
771 * only does trylock page: if we raced, best clean up here.
772 */
773 delete_from_swap_cache(*pagep);
774 set_page_dirty(*pagep);
775 if (!error) {
776 spin_lock(&info->lock);
777 info->swapped--;
778 spin_unlock(&info->lock);
779 swap_free(swap);
780 }
781 }
782 return error;
783}
784
785/*
786 * Search through swapped inodes to find and replace swap by page.
787 */
788int shmem_unuse(swp_entry_t swap, struct page *page)
789{
790 struct list_head *this, *next;
791 struct shmem_inode_info *info;
792 struct mem_cgroup *memcg;
793 int error = 0;
794
795 /*
796 * There's a faint possibility that swap page was replaced before
797 * caller locked it: caller will come back later with the right page.
798 */
799 if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val))
800 goto out;
801
802 /*
803 * Charge page using GFP_KERNEL while we can wait, before taking
804 * the shmem_swaplist_mutex which might hold up shmem_writepage().
805 * Charged back to the user (not to caller) when swap account is used.
806 */
807 error = mem_cgroup_try_charge(page, current->mm, GFP_KERNEL, &memcg,
808 false);
809 if (error)
810 goto out;
811 /* No radix_tree_preload: swap entry keeps a place for page in tree */
812 error = -EAGAIN;
813
814 mutex_lock(&shmem_swaplist_mutex);
815 list_for_each_safe(this, next, &shmem_swaplist) {
816 info = list_entry(this, struct shmem_inode_info, swaplist);
817 if (info->swapped)
818 error = shmem_unuse_inode(info, swap, &page);
819 else
820 list_del_init(&info->swaplist);
821 cond_resched();
822 if (error != -EAGAIN)
823 break;
824 /* found nothing in this: move on to search the next */
825 }
826 mutex_unlock(&shmem_swaplist_mutex);
827
828 if (error) {
829 if (error != -ENOMEM)
830 error = 0;
831 mem_cgroup_cancel_charge(page, memcg, false);
832 } else
833 mem_cgroup_commit_charge(page, memcg, true, false);
834out:
835 unlock_page(page);
836 put_page(page);
837 return error;
838}
839
840/*
841 * Move the page from the page cache to the swap cache.
842 */
843static int shmem_writepage(struct page *page, struct writeback_control *wbc)
844{
845 struct shmem_inode_info *info;
846 struct address_space *mapping;
847 struct inode *inode;
848 swp_entry_t swap;
849 pgoff_t index;
850
851 BUG_ON(!PageLocked(page));
852 mapping = page->mapping;
853 index = page->index;
854 inode = mapping->host;
855 info = SHMEM_I(inode);
856 if (info->flags & VM_LOCKED)
857 goto redirty;
858 if (!total_swap_pages)
859 goto redirty;
860
861 /*
862 * Our capabilities prevent regular writeback or sync from ever calling
863 * shmem_writepage; but a stacking filesystem might use ->writepage of
864 * its underlying filesystem, in which case tmpfs should write out to
865 * swap only in response to memory pressure, and not for the writeback
866 * threads or sync.
867 */
868 if (!wbc->for_reclaim) {
869 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */
870 goto redirty;
871 }
872
873 /*
874 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
875 * value into swapfile.c, the only way we can correctly account for a
876 * fallocated page arriving here is now to initialize it and write it.
877 *
878 * That's okay for a page already fallocated earlier, but if we have
879 * not yet completed the fallocation, then (a) we want to keep track
880 * of this page in case we have to undo it, and (b) it may not be a
881 * good idea to continue anyway, once we're pushing into swap. So
882 * reactivate the page, and let shmem_fallocate() quit when too many.
883 */
884 if (!PageUptodate(page)) {
885 if (inode->i_private) {
886 struct shmem_falloc *shmem_falloc;
887 spin_lock(&inode->i_lock);
888 shmem_falloc = inode->i_private;
889 if (shmem_falloc &&
890 !shmem_falloc->waitq &&
891 index >= shmem_falloc->start &&
892 index < shmem_falloc->next)
893 shmem_falloc->nr_unswapped++;
894 else
895 shmem_falloc = NULL;
896 spin_unlock(&inode->i_lock);
897 if (shmem_falloc)
898 goto redirty;
899 }
900 clear_highpage(page);
901 flush_dcache_page(page);
902 SetPageUptodate(page);
903 }
904
905 swap = get_swap_page();
906 if (!swap.val)
907 goto redirty;
908
909 if (mem_cgroup_try_charge_swap(page, swap))
910 goto free_swap;
911
912 /*
913 * Add inode to shmem_unuse()'s list of swapped-out inodes,
914 * if it's not already there. Do it now before the page is
915 * moved to swap cache, when its pagelock no longer protects
916 * the inode from eviction. But don't unlock the mutex until
917 * we've incremented swapped, because shmem_unuse_inode() will
918 * prune a !swapped inode from the swaplist under this mutex.
919 */
920 mutex_lock(&shmem_swaplist_mutex);
921 if (list_empty(&info->swaplist))
922 list_add_tail(&info->swaplist, &shmem_swaplist);
923
924 if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
925 spin_lock(&info->lock);
926 shmem_recalc_inode(inode);
927 info->swapped++;
928 spin_unlock(&info->lock);
929
930 swap_shmem_alloc(swap);
931 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
932
933 mutex_unlock(&shmem_swaplist_mutex);
934 BUG_ON(page_mapped(page));
935 swap_writepage(page, wbc);
936 return 0;
937 }
938
939 mutex_unlock(&shmem_swaplist_mutex);
940free_swap:
941 swapcache_free(swap);
942redirty:
943 set_page_dirty(page);
944 if (wbc->for_reclaim)
945 return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */
946 unlock_page(page);
947 return 0;
948}
949
950#ifdef CONFIG_NUMA
951#ifdef CONFIG_TMPFS
952static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
953{
954 char buffer[64];
955
956 if (!mpol || mpol->mode == MPOL_DEFAULT)
957 return; /* show nothing */
958
959 mpol_to_str(buffer, sizeof(buffer), mpol);
960
961 seq_printf(seq, ",mpol=%s", buffer);
962}
963
964static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
965{
966 struct mempolicy *mpol = NULL;
967 if (sbinfo->mpol) {
968 spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */
969 mpol = sbinfo->mpol;
970 mpol_get(mpol);
971 spin_unlock(&sbinfo->stat_lock);
972 }
973 return mpol;
974}
975#endif /* CONFIG_TMPFS */
976
977static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
978 struct shmem_inode_info *info, pgoff_t index)
979{
980 struct vm_area_struct pvma;
981 struct page *page;
982
983 /* Create a pseudo vma that just contains the policy */
984 pvma.vm_start = 0;
985 /* Bias interleave by inode number to distribute better across nodes */
986 pvma.vm_pgoff = index + info->vfs_inode.i_ino;
987 pvma.vm_ops = NULL;
988 pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index);
989
990 page = swapin_readahead(swap, gfp, &pvma, 0);
991
992 /* Drop reference taken by mpol_shared_policy_lookup() */
993 mpol_cond_put(pvma.vm_policy);
994
995 return page;
996}
997
998static struct page *shmem_alloc_page(gfp_t gfp,
999 struct shmem_inode_info *info, pgoff_t index)
1000{
1001 struct vm_area_struct pvma;
1002 struct page *page;
1003
1004 /* Create a pseudo vma that just contains the policy */
1005 pvma.vm_start = 0;
1006 /* Bias interleave by inode number to distribute better across nodes */
1007 pvma.vm_pgoff = index + info->vfs_inode.i_ino;
1008 pvma.vm_ops = NULL;
1009 pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index);
1010
1011 page = alloc_page_vma(gfp, &pvma, 0);
1012
1013 /* Drop reference taken by mpol_shared_policy_lookup() */
1014 mpol_cond_put(pvma.vm_policy);
1015
1016 return page;
1017}
1018#else /* !CONFIG_NUMA */
1019#ifdef CONFIG_TMPFS
1020static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1021{
1022}
1023#endif /* CONFIG_TMPFS */
1024
1025static inline struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
1026 struct shmem_inode_info *info, pgoff_t index)
1027{
1028 return swapin_readahead(swap, gfp, NULL, 0);
1029}
1030
1031static inline struct page *shmem_alloc_page(gfp_t gfp,
1032 struct shmem_inode_info *info, pgoff_t index)
1033{
1034 return alloc_page(gfp);
1035}
1036#endif /* CONFIG_NUMA */
1037
1038#if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
1039static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1040{
1041 return NULL;
1042}
1043#endif
1044
1045/*
1046 * When a page is moved from swapcache to shmem filecache (either by the
1047 * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
1048 * shmem_unuse_inode()), it may have been read in earlier from swap, in
1049 * ignorance of the mapping it belongs to. If that mapping has special
1050 * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1051 * we may need to copy to a suitable page before moving to filecache.
1052 *
1053 * In a future release, this may well be extended to respect cpuset and
1054 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1055 * but for now it is a simple matter of zone.
1056 */
1057static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
1058{
1059 return page_zonenum(page) > gfp_zone(gfp);
1060}
1061
1062static int shmem_replace_page(struct page **pagep, gfp_t gfp,
1063 struct shmem_inode_info *info, pgoff_t index)
1064{
1065 struct page *oldpage, *newpage;
1066 struct address_space *swap_mapping;
1067 pgoff_t swap_index;
1068 int error;
1069
1070 oldpage = *pagep;
1071 swap_index = page_private(oldpage);
1072 swap_mapping = page_mapping(oldpage);
1073
1074 /*
1075 * We have arrived here because our zones are constrained, so don't
1076 * limit chance of success by further cpuset and node constraints.
1077 */
1078 gfp &= ~GFP_CONSTRAINT_MASK;
1079 newpage = shmem_alloc_page(gfp, info, index);
1080 if (!newpage)
1081 return -ENOMEM;
1082
1083 get_page(newpage);
1084 copy_highpage(newpage, oldpage);
1085 flush_dcache_page(newpage);
1086
1087 __SetPageLocked(newpage);
1088 SetPageUptodate(newpage);
1089 SetPageSwapBacked(newpage);
1090 set_page_private(newpage, swap_index);
1091 SetPageSwapCache(newpage);
1092
1093 /*
1094 * Our caller will very soon move newpage out of swapcache, but it's
1095 * a nice clean interface for us to replace oldpage by newpage there.
1096 */
1097 spin_lock_irq(&swap_mapping->tree_lock);
1098 error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage,
1099 newpage);
1100 if (!error) {
1101 __inc_zone_page_state(newpage, NR_FILE_PAGES);
1102 __dec_zone_page_state(oldpage, NR_FILE_PAGES);
1103 }
1104 spin_unlock_irq(&swap_mapping->tree_lock);
1105
1106 if (unlikely(error)) {
1107 /*
1108 * Is this possible? I think not, now that our callers check
1109 * both PageSwapCache and page_private after getting page lock;
1110 * but be defensive. Reverse old to newpage for clear and free.
1111 */
1112 oldpage = newpage;
1113 } else {
1114 mem_cgroup_migrate(oldpage, newpage);
1115 lru_cache_add_anon(newpage);
1116 *pagep = newpage;
1117 }
1118
1119 ClearPageSwapCache(oldpage);
1120 set_page_private(oldpage, 0);
1121
1122 unlock_page(oldpage);
1123 put_page(oldpage);
1124 put_page(oldpage);
1125 return error;
1126}
1127
1128/*
1129 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
1130 *
1131 * If we allocate a new one we do not mark it dirty. That's up to the
1132 * vm. If we swap it in we mark it dirty since we also free the swap
1133 * entry since a page cannot live in both the swap and page cache
1134 */
1135static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1136 struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type)
1137{
1138 struct address_space *mapping = inode->i_mapping;
1139 struct shmem_inode_info *info;
1140 struct shmem_sb_info *sbinfo;
1141 struct mem_cgroup *memcg;
1142 struct page *page;
1143 swp_entry_t swap;
1144 int error;
1145 int once = 0;
1146 int alloced = 0;
1147
1148 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1149 return -EFBIG;
1150repeat:
1151 swap.val = 0;
1152 page = find_lock_entry(mapping, index);
1153 if (radix_tree_exceptional_entry(page)) {
1154 swap = radix_to_swp_entry(page);
1155 page = NULL;
1156 }
1157
1158 if (sgp != SGP_WRITE && sgp != SGP_FALLOC &&
1159 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1160 error = -EINVAL;
1161 goto unlock;
1162 }
1163
1164 if (page && sgp == SGP_WRITE)
1165 mark_page_accessed(page);
1166
1167 /* fallocated page? */
1168 if (page && !PageUptodate(page)) {
1169 if (sgp != SGP_READ)
1170 goto clear;
1171 unlock_page(page);
1172 put_page(page);
1173 page = NULL;
1174 }
1175 if (page || (sgp == SGP_READ && !swap.val)) {
1176 *pagep = page;
1177 return 0;
1178 }
1179
1180 /*
1181 * Fast cache lookup did not find it:
1182 * bring it back from swap or allocate.
1183 */
1184 info = SHMEM_I(inode);
1185 sbinfo = SHMEM_SB(inode->i_sb);
1186
1187 if (swap.val) {
1188 /* Look it up and read it in.. */
1189 page = lookup_swap_cache(swap);
1190 if (!page) {
1191 /* here we actually do the io */
1192 if (fault_type)
1193 *fault_type |= VM_FAULT_MAJOR;
1194 page = shmem_swapin(swap, gfp, info, index);
1195 if (!page) {
1196 error = -ENOMEM;
1197 goto failed;
1198 }
1199 }
1200
1201 /* We have to do this with page locked to prevent races */
1202 lock_page(page);
1203 if (!PageSwapCache(page) || page_private(page) != swap.val ||
1204 !shmem_confirm_swap(mapping, index, swap)) {
1205 error = -EEXIST; /* try again */
1206 goto unlock;
1207 }
1208 if (!PageUptodate(page)) {
1209 error = -EIO;
1210 goto failed;
1211 }
1212 wait_on_page_writeback(page);
1213
1214 if (shmem_should_replace_page(page, gfp)) {
1215 error = shmem_replace_page(&page, gfp, info, index);
1216 if (error)
1217 goto failed;
1218 }
1219
1220 error = mem_cgroup_try_charge(page, current->mm, gfp, &memcg,
1221 false);
1222 if (!error) {
1223 error = shmem_add_to_page_cache(page, mapping, index,
1224 swp_to_radix_entry(swap));
1225 /*
1226 * We already confirmed swap under page lock, and make
1227 * no memory allocation here, so usually no possibility
1228 * of error; but free_swap_and_cache() only trylocks a
1229 * page, so it is just possible that the entry has been
1230 * truncated or holepunched since swap was confirmed.
1231 * shmem_undo_range() will have done some of the
1232 * unaccounting, now delete_from_swap_cache() will do
1233 * the rest.
1234 * Reset swap.val? No, leave it so "failed" goes back to
1235 * "repeat": reading a hole and writing should succeed.
1236 */
1237 if (error) {
1238 mem_cgroup_cancel_charge(page, memcg, false);
1239 delete_from_swap_cache(page);
1240 }
1241 }
1242 if (error)
1243 goto failed;
1244
1245 mem_cgroup_commit_charge(page, memcg, true, false);
1246
1247 spin_lock(&info->lock);
1248 info->swapped--;
1249 shmem_recalc_inode(inode);
1250 spin_unlock(&info->lock);
1251
1252 if (sgp == SGP_WRITE)
1253 mark_page_accessed(page);
1254
1255 delete_from_swap_cache(page);
1256 set_page_dirty(page);
1257 swap_free(swap);
1258
1259 } else {
1260 if (shmem_acct_block(info->flags)) {
1261 error = -ENOSPC;
1262 goto failed;
1263 }
1264 if (sbinfo->max_blocks) {
1265 if (percpu_counter_compare(&sbinfo->used_blocks,
1266 sbinfo->max_blocks) >= 0) {
1267 error = -ENOSPC;
1268 goto unacct;
1269 }
1270 percpu_counter_inc(&sbinfo->used_blocks);
1271 }
1272
1273 page = shmem_alloc_page(gfp, info, index);
1274 if (!page) {
1275 error = -ENOMEM;
1276 goto decused;
1277 }
1278
1279 __SetPageSwapBacked(page);
1280 __SetPageLocked(page);
1281 if (sgp == SGP_WRITE)
1282 __SetPageReferenced(page);
1283
1284 error = mem_cgroup_try_charge(page, current->mm, gfp, &memcg,
1285 false);
1286 if (error)
1287 goto decused;
1288 error = radix_tree_maybe_preload(gfp & GFP_RECLAIM_MASK);
1289 if (!error) {
1290 error = shmem_add_to_page_cache(page, mapping, index,
1291 NULL);
1292 radix_tree_preload_end();
1293 }
1294 if (error) {
1295 mem_cgroup_cancel_charge(page, memcg, false);
1296 goto decused;
1297 }
1298 mem_cgroup_commit_charge(page, memcg, false, false);
1299 lru_cache_add_anon(page);
1300
1301 spin_lock(&info->lock);
1302 info->alloced++;
1303 inode->i_blocks += BLOCKS_PER_PAGE;
1304 shmem_recalc_inode(inode);
1305 spin_unlock(&info->lock);
1306 alloced = true;
1307
1308 /*
1309 * Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
1310 */
1311 if (sgp == SGP_FALLOC)
1312 sgp = SGP_WRITE;
1313clear:
1314 /*
1315 * Let SGP_WRITE caller clear ends if write does not fill page;
1316 * but SGP_FALLOC on a page fallocated earlier must initialize
1317 * it now, lest undo on failure cancel our earlier guarantee.
1318 */
1319 if (sgp != SGP_WRITE) {
1320 clear_highpage(page);
1321 flush_dcache_page(page);
1322 SetPageUptodate(page);
1323 }
1324 if (sgp == SGP_DIRTY)
1325 set_page_dirty(page);
1326 }
1327
1328 /* Perhaps the file has been truncated since we checked */
1329 if (sgp != SGP_WRITE && sgp != SGP_FALLOC &&
1330 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1331 if (alloced) {
1332 ClearPageDirty(page);
1333 delete_from_page_cache(page);
1334 spin_lock(&info->lock);
1335 shmem_recalc_inode(inode);
1336 spin_unlock(&info->lock);
1337 }
1338 error = -EINVAL;
1339 goto unlock;
1340 }
1341 *pagep = page;
1342 return 0;
1343
1344 /*
1345 * Error recovery.
1346 */
1347decused:
1348 if (sbinfo->max_blocks)
1349 percpu_counter_add(&sbinfo->used_blocks, -1);
1350unacct:
1351 shmem_unacct_blocks(info->flags, 1);
1352failed:
1353 if (swap.val && !shmem_confirm_swap(mapping, index, swap))
1354 error = -EEXIST;
1355unlock:
1356 if (page) {
1357 unlock_page(page);
1358 put_page(page);
1359 }
1360 if (error == -ENOSPC && !once++) {
1361 info = SHMEM_I(inode);
1362 spin_lock(&info->lock);
1363 shmem_recalc_inode(inode);
1364 spin_unlock(&info->lock);
1365 goto repeat;
1366 }
1367 if (error == -EEXIST) /* from above or from radix_tree_insert */
1368 goto repeat;
1369 return error;
1370}
1371
1372static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1373{
1374 struct inode *inode = file_inode(vma->vm_file);
1375 int error;
1376 int ret = VM_FAULT_LOCKED;
1377
1378 /*
1379 * Trinity finds that probing a hole which tmpfs is punching can
1380 * prevent the hole-punch from ever completing: which in turn
1381 * locks writers out with its hold on i_mutex. So refrain from
1382 * faulting pages into the hole while it's being punched. Although
1383 * shmem_undo_range() does remove the additions, it may be unable to
1384 * keep up, as each new page needs its own unmap_mapping_range() call,
1385 * and the i_mmap tree grows ever slower to scan if new vmas are added.
1386 *
1387 * It does not matter if we sometimes reach this check just before the
1388 * hole-punch begins, so that one fault then races with the punch:
1389 * we just need to make racing faults a rare case.
1390 *
1391 * The implementation below would be much simpler if we just used a
1392 * standard mutex or completion: but we cannot take i_mutex in fault,
1393 * and bloating every shmem inode for this unlikely case would be sad.
1394 */
1395 if (unlikely(inode->i_private)) {
1396 struct shmem_falloc *shmem_falloc;
1397
1398 spin_lock(&inode->i_lock);
1399 shmem_falloc = inode->i_private;
1400 if (shmem_falloc &&
1401 shmem_falloc->waitq &&
1402 vmf->pgoff >= shmem_falloc->start &&
1403 vmf->pgoff < shmem_falloc->next) {
1404 wait_queue_head_t *shmem_falloc_waitq;
1405 DEFINE_WAIT(shmem_fault_wait);
1406
1407 ret = VM_FAULT_NOPAGE;
1408 if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) &&
1409 !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
1410 /* It's polite to up mmap_sem if we can */
1411 up_read(&vma->vm_mm->mmap_sem);
1412 ret = VM_FAULT_RETRY;
1413 }
1414
1415 shmem_falloc_waitq = shmem_falloc->waitq;
1416 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
1417 TASK_UNINTERRUPTIBLE);
1418 spin_unlock(&inode->i_lock);
1419 schedule();
1420
1421 /*
1422 * shmem_falloc_waitq points into the shmem_fallocate()
1423 * stack of the hole-punching task: shmem_falloc_waitq
1424 * is usually invalid by the time we reach here, but
1425 * finish_wait() does not dereference it in that case;
1426 * though i_lock needed lest racing with wake_up_all().
1427 */
1428 spin_lock(&inode->i_lock);
1429 finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
1430 spin_unlock(&inode->i_lock);
1431 return ret;
1432 }
1433 spin_unlock(&inode->i_lock);
1434 }
1435
1436 error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
1437 if (error)
1438 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1439
1440 if (ret & VM_FAULT_MAJOR) {
1441 count_vm_event(PGMAJFAULT);
1442 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
1443 }
1444 return ret;
1445}
1446
1447#ifdef CONFIG_NUMA
1448static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
1449{
1450 struct inode *inode = file_inode(vma->vm_file);
1451 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
1452}
1453
1454static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
1455 unsigned long addr)
1456{
1457 struct inode *inode = file_inode(vma->vm_file);
1458 pgoff_t index;
1459
1460 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1461 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
1462}
1463#endif
1464
1465int shmem_lock(struct file *file, int lock, struct user_struct *user)
1466{
1467 struct inode *inode = file_inode(file);
1468 struct shmem_inode_info *info = SHMEM_I(inode);
1469 int retval = -ENOMEM;
1470
1471 spin_lock(&info->lock);
1472 if (lock && !(info->flags & VM_LOCKED)) {
1473 if (!user_shm_lock(inode->i_size, user))
1474 goto out_nomem;
1475 info->flags |= VM_LOCKED;
1476 mapping_set_unevictable(file->f_mapping);
1477 }
1478 if (!lock && (info->flags & VM_LOCKED) && user) {
1479 user_shm_unlock(inode->i_size, user);
1480 info->flags &= ~VM_LOCKED;
1481 mapping_clear_unevictable(file->f_mapping);
1482 }
1483 retval = 0;
1484
1485out_nomem:
1486 spin_unlock(&info->lock);
1487 return retval;
1488}
1489
1490static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1491{
1492 file_accessed(file);
1493 vma->vm_ops = &shmem_vm_ops;
1494 return 0;
1495}
1496
1497static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
1498 umode_t mode, dev_t dev, unsigned long flags)
1499{
1500 struct inode *inode;
1501 struct shmem_inode_info *info;
1502 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1503
1504 if (shmem_reserve_inode(sb))
1505 return NULL;
1506
1507 inode = new_inode(sb);
1508 if (inode) {
1509 inode->i_ino = get_next_ino();
1510 inode_init_owner(inode, dir, mode);
1511 inode->i_blocks = 0;
1512 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1513 inode->i_generation = get_seconds();
1514 info = SHMEM_I(inode);
1515 memset(info, 0, (char *)inode - (char *)info);
1516 spin_lock_init(&info->lock);
1517 info->seals = F_SEAL_SEAL;
1518 info->flags = flags & VM_NORESERVE;
1519 INIT_LIST_HEAD(&info->swaplist);
1520 simple_xattrs_init(&info->xattrs);
1521 cache_no_acl(inode);
1522
1523 switch (mode & S_IFMT) {
1524 default:
1525 inode->i_op = &shmem_special_inode_operations;
1526 init_special_inode(inode, mode, dev);
1527 break;
1528 case S_IFREG:
1529 inode->i_mapping->a_ops = &shmem_aops;
1530 inode->i_op = &shmem_inode_operations;
1531 inode->i_fop = &shmem_file_operations;
1532 mpol_shared_policy_init(&info->policy,
1533 shmem_get_sbmpol(sbinfo));
1534 break;
1535 case S_IFDIR:
1536 inc_nlink(inode);
1537 /* Some things misbehave if size == 0 on a directory */
1538 inode->i_size = 2 * BOGO_DIRENT_SIZE;
1539 inode->i_op = &shmem_dir_inode_operations;
1540 inode->i_fop = &simple_dir_operations;
1541 break;
1542 case S_IFLNK:
1543 /*
1544 * Must not load anything in the rbtree,
1545 * mpol_free_shared_policy will not be called.
1546 */
1547 mpol_shared_policy_init(&info->policy, NULL);
1548 break;
1549 }
1550 } else
1551 shmem_free_inode(sb);
1552 return inode;
1553}
1554
1555bool shmem_mapping(struct address_space *mapping)
1556{
1557 if (!mapping->host)
1558 return false;
1559
1560 return mapping->host->i_sb->s_op == &shmem_ops;
1561}
1562
1563#ifdef CONFIG_TMPFS
1564static const struct inode_operations shmem_symlink_inode_operations;
1565static const struct inode_operations shmem_short_symlink_operations;
1566
1567#ifdef CONFIG_TMPFS_XATTR
1568static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
1569#else
1570#define shmem_initxattrs NULL
1571#endif
1572
1573static int
1574shmem_write_begin(struct file *file, struct address_space *mapping,
1575 loff_t pos, unsigned len, unsigned flags,
1576 struct page **pagep, void **fsdata)
1577{
1578 struct inode *inode = mapping->host;
1579 struct shmem_inode_info *info = SHMEM_I(inode);
1580 pgoff_t index = pos >> PAGE_SHIFT;
1581
1582 /* i_mutex is held by caller */
1583 if (unlikely(info->seals)) {
1584 if (info->seals & F_SEAL_WRITE)
1585 return -EPERM;
1586 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
1587 return -EPERM;
1588 }
1589
1590 return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
1591}
1592
1593static int
1594shmem_write_end(struct file *file, struct address_space *mapping,
1595 loff_t pos, unsigned len, unsigned copied,
1596 struct page *page, void *fsdata)
1597{
1598 struct inode *inode = mapping->host;
1599
1600 if (pos + copied > inode->i_size)
1601 i_size_write(inode, pos + copied);
1602
1603 if (!PageUptodate(page)) {
1604 if (copied < PAGE_SIZE) {
1605 unsigned from = pos & (PAGE_SIZE - 1);
1606 zero_user_segments(page, 0, from,
1607 from + copied, PAGE_SIZE);
1608 }
1609 SetPageUptodate(page);
1610 }
1611 set_page_dirty(page);
1612 unlock_page(page);
1613 put_page(page);
1614
1615 return copied;
1616}
1617
1618static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
1619{
1620 struct file *file = iocb->ki_filp;
1621 struct inode *inode = file_inode(file);
1622 struct address_space *mapping = inode->i_mapping;
1623 pgoff_t index;
1624 unsigned long offset;
1625 enum sgp_type sgp = SGP_READ;
1626 int error = 0;
1627 ssize_t retval = 0;
1628 loff_t *ppos = &iocb->ki_pos;
1629
1630 /*
1631 * Might this read be for a stacking filesystem? Then when reading
1632 * holes of a sparse file, we actually need to allocate those pages,
1633 * and even mark them dirty, so it cannot exceed the max_blocks limit.
1634 */
1635 if (!iter_is_iovec(to))
1636 sgp = SGP_DIRTY;
1637
1638 index = *ppos >> PAGE_SHIFT;
1639 offset = *ppos & ~PAGE_MASK;
1640
1641 for (;;) {
1642 struct page *page = NULL;
1643 pgoff_t end_index;
1644 unsigned long nr, ret;
1645 loff_t i_size = i_size_read(inode);
1646
1647 end_index = i_size >> PAGE_SHIFT;
1648 if (index > end_index)
1649 break;
1650 if (index == end_index) {
1651 nr = i_size & ~PAGE_MASK;
1652 if (nr <= offset)
1653 break;
1654 }
1655
1656 error = shmem_getpage(inode, index, &page, sgp, NULL);
1657 if (error) {
1658 if (error == -EINVAL)
1659 error = 0;
1660 break;
1661 }
1662 if (page)
1663 unlock_page(page);
1664
1665 /*
1666 * We must evaluate after, since reads (unlike writes)
1667 * are called without i_mutex protection against truncate
1668 */
1669 nr = PAGE_SIZE;
1670 i_size = i_size_read(inode);
1671 end_index = i_size >> PAGE_SHIFT;
1672 if (index == end_index) {
1673 nr = i_size & ~PAGE_MASK;
1674 if (nr <= offset) {
1675 if (page)
1676 put_page(page);
1677 break;
1678 }
1679 }
1680 nr -= offset;
1681
1682 if (page) {
1683 /*
1684 * If users can be writing to this page using arbitrary
1685 * virtual addresses, take care about potential aliasing
1686 * before reading the page on the kernel side.
1687 */
1688 if (mapping_writably_mapped(mapping))
1689 flush_dcache_page(page);
1690 /*
1691 * Mark the page accessed if we read the beginning.
1692 */
1693 if (!offset)
1694 mark_page_accessed(page);
1695 } else {
1696 page = ZERO_PAGE(0);
1697 get_page(page);
1698 }
1699
1700 /*
1701 * Ok, we have the page, and it's up-to-date, so
1702 * now we can copy it to user space...
1703 */
1704 ret = copy_page_to_iter(page, offset, nr, to);
1705 retval += ret;
1706 offset += ret;
1707 index += offset >> PAGE_SHIFT;
1708 offset &= ~PAGE_MASK;
1709
1710 put_page(page);
1711 if (!iov_iter_count(to))
1712 break;
1713 if (ret < nr) {
1714 error = -EFAULT;
1715 break;
1716 }
1717 cond_resched();
1718 }
1719
1720 *ppos = ((loff_t) index << PAGE_SHIFT) + offset;
1721 file_accessed(file);
1722 return retval ? retval : error;
1723}
1724
1725static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
1726 struct pipe_inode_info *pipe, size_t len,
1727 unsigned int flags)
1728{
1729 struct address_space *mapping = in->f_mapping;
1730 struct inode *inode = mapping->host;
1731 unsigned int loff, nr_pages, req_pages;
1732 struct page *pages[PIPE_DEF_BUFFERS];
1733 struct partial_page partial[PIPE_DEF_BUFFERS];
1734 struct page *page;
1735 pgoff_t index, end_index;
1736 loff_t isize, left;
1737 int error, page_nr;
1738 struct splice_pipe_desc spd = {
1739 .pages = pages,
1740 .partial = partial,
1741 .nr_pages_max = PIPE_DEF_BUFFERS,
1742 .flags = flags,
1743 .ops = &page_cache_pipe_buf_ops,
1744 .spd_release = spd_release_page,
1745 };
1746
1747 isize = i_size_read(inode);
1748 if (unlikely(*ppos >= isize))
1749 return 0;
1750
1751 left = isize - *ppos;
1752 if (unlikely(left < len))
1753 len = left;
1754
1755 if (splice_grow_spd(pipe, &spd))
1756 return -ENOMEM;
1757
1758 index = *ppos >> PAGE_SHIFT;
1759 loff = *ppos & ~PAGE_MASK;
1760 req_pages = (len + loff + PAGE_SIZE - 1) >> PAGE_SHIFT;
1761 nr_pages = min(req_pages, spd.nr_pages_max);
1762
1763 spd.nr_pages = find_get_pages_contig(mapping, index,
1764 nr_pages, spd.pages);
1765 index += spd.nr_pages;
1766 error = 0;
1767
1768 while (spd.nr_pages < nr_pages) {
1769 error = shmem_getpage(inode, index, &page, SGP_CACHE, NULL);
1770 if (error)
1771 break;
1772 unlock_page(page);
1773 spd.pages[spd.nr_pages++] = page;
1774 index++;
1775 }
1776
1777 index = *ppos >> PAGE_SHIFT;
1778 nr_pages = spd.nr_pages;
1779 spd.nr_pages = 0;
1780
1781 for (page_nr = 0; page_nr < nr_pages; page_nr++) {
1782 unsigned int this_len;
1783
1784 if (!len)
1785 break;
1786
1787 this_len = min_t(unsigned long, len, PAGE_SIZE - loff);
1788 page = spd.pages[page_nr];
1789
1790 if (!PageUptodate(page) || page->mapping != mapping) {
1791 error = shmem_getpage(inode, index, &page,
1792 SGP_CACHE, NULL);
1793 if (error)
1794 break;
1795 unlock_page(page);
1796 put_page(spd.pages[page_nr]);
1797 spd.pages[page_nr] = page;
1798 }
1799
1800 isize = i_size_read(inode);
1801 end_index = (isize - 1) >> PAGE_SHIFT;
1802 if (unlikely(!isize || index > end_index))
1803 break;
1804
1805 if (end_index == index) {
1806 unsigned int plen;
1807
1808 plen = ((isize - 1) & ~PAGE_MASK) + 1;
1809 if (plen <= loff)
1810 break;
1811
1812 this_len = min(this_len, plen - loff);
1813 len = this_len;
1814 }
1815
1816 spd.partial[page_nr].offset = loff;
1817 spd.partial[page_nr].len = this_len;
1818 len -= this_len;
1819 loff = 0;
1820 spd.nr_pages++;
1821 index++;
1822 }
1823
1824 while (page_nr < nr_pages)
1825 put_page(spd.pages[page_nr++]);
1826
1827 if (spd.nr_pages)
1828 error = splice_to_pipe(pipe, &spd);
1829
1830 splice_shrink_spd(&spd);
1831
1832 if (error > 0) {
1833 *ppos += error;
1834 file_accessed(in);
1835 }
1836 return error;
1837}
1838
1839/*
1840 * llseek SEEK_DATA or SEEK_HOLE through the radix_tree.
1841 */
1842static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
1843 pgoff_t index, pgoff_t end, int whence)
1844{
1845 struct page *page;
1846 struct pagevec pvec;
1847 pgoff_t indices[PAGEVEC_SIZE];
1848 bool done = false;
1849 int i;
1850
1851 pagevec_init(&pvec, 0);
1852 pvec.nr = 1; /* start small: we may be there already */
1853 while (!done) {
1854 pvec.nr = find_get_entries(mapping, index,
1855 pvec.nr, pvec.pages, indices);
1856 if (!pvec.nr) {
1857 if (whence == SEEK_DATA)
1858 index = end;
1859 break;
1860 }
1861 for (i = 0; i < pvec.nr; i++, index++) {
1862 if (index < indices[i]) {
1863 if (whence == SEEK_HOLE) {
1864 done = true;
1865 break;
1866 }
1867 index = indices[i];
1868 }
1869 page = pvec.pages[i];
1870 if (page && !radix_tree_exceptional_entry(page)) {
1871 if (!PageUptodate(page))
1872 page = NULL;
1873 }
1874 if (index >= end ||
1875 (page && whence == SEEK_DATA) ||
1876 (!page && whence == SEEK_HOLE)) {
1877 done = true;
1878 break;
1879 }
1880 }
1881 pagevec_remove_exceptionals(&pvec);
1882 pagevec_release(&pvec);
1883 pvec.nr = PAGEVEC_SIZE;
1884 cond_resched();
1885 }
1886 return index;
1887}
1888
1889static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
1890{
1891 struct address_space *mapping = file->f_mapping;
1892 struct inode *inode = mapping->host;
1893 pgoff_t start, end;
1894 loff_t new_offset;
1895
1896 if (whence != SEEK_DATA && whence != SEEK_HOLE)
1897 return generic_file_llseek_size(file, offset, whence,
1898 MAX_LFS_FILESIZE, i_size_read(inode));
1899 inode_lock(inode);
1900 /* We're holding i_mutex so we can access i_size directly */
1901
1902 if (offset < 0)
1903 offset = -EINVAL;
1904 else if (offset >= inode->i_size)
1905 offset = -ENXIO;
1906 else {
1907 start = offset >> PAGE_SHIFT;
1908 end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1909 new_offset = shmem_seek_hole_data(mapping, start, end, whence);
1910 new_offset <<= PAGE_SHIFT;
1911 if (new_offset > offset) {
1912 if (new_offset < inode->i_size)
1913 offset = new_offset;
1914 else if (whence == SEEK_DATA)
1915 offset = -ENXIO;
1916 else
1917 offset = inode->i_size;
1918 }
1919 }
1920
1921 if (offset >= 0)
1922 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
1923 inode_unlock(inode);
1924 return offset;
1925}
1926
1927/*
1928 * We need a tag: a new tag would expand every radix_tree_node by 8 bytes,
1929 * so reuse a tag which we firmly believe is never set or cleared on shmem.
1930 */
1931#define SHMEM_TAG_PINNED PAGECACHE_TAG_TOWRITE
1932#define LAST_SCAN 4 /* about 150ms max */
1933
1934static void shmem_tag_pins(struct address_space *mapping)
1935{
1936 struct radix_tree_iter iter;
1937 void **slot;
1938 pgoff_t start;
1939 struct page *page;
1940
1941 lru_add_drain();
1942 start = 0;
1943 rcu_read_lock();
1944
1945 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
1946 page = radix_tree_deref_slot(slot);
1947 if (!page || radix_tree_exception(page)) {
1948 if (radix_tree_deref_retry(page)) {
1949 slot = radix_tree_iter_retry(&iter);
1950 continue;
1951 }
1952 } else if (page_count(page) - page_mapcount(page) > 1) {
1953 spin_lock_irq(&mapping->tree_lock);
1954 radix_tree_tag_set(&mapping->page_tree, iter.index,
1955 SHMEM_TAG_PINNED);
1956 spin_unlock_irq(&mapping->tree_lock);
1957 }
1958
1959 if (need_resched()) {
1960 cond_resched_rcu();
1961 slot = radix_tree_iter_next(&iter);
1962 }
1963 }
1964 rcu_read_unlock();
1965}
1966
1967/*
1968 * Setting SEAL_WRITE requires us to verify there's no pending writer. However,
1969 * via get_user_pages(), drivers might have some pending I/O without any active
1970 * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages
1971 * and see whether it has an elevated ref-count. If so, we tag them and wait for
1972 * them to be dropped.
1973 * The caller must guarantee that no new user will acquire writable references
1974 * to those pages to avoid races.
1975 */
1976static int shmem_wait_for_pins(struct address_space *mapping)
1977{
1978 struct radix_tree_iter iter;
1979 void **slot;
1980 pgoff_t start;
1981 struct page *page;
1982 int error, scan;
1983
1984 shmem_tag_pins(mapping);
1985
1986 error = 0;
1987 for (scan = 0; scan <= LAST_SCAN; scan++) {
1988 if (!radix_tree_tagged(&mapping->page_tree, SHMEM_TAG_PINNED))
1989 break;
1990
1991 if (!scan)
1992 lru_add_drain_all();
1993 else if (schedule_timeout_killable((HZ << scan) / 200))
1994 scan = LAST_SCAN;
1995
1996 start = 0;
1997 rcu_read_lock();
1998 radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter,
1999 start, SHMEM_TAG_PINNED) {
2000
2001 page = radix_tree_deref_slot(slot);
2002 if (radix_tree_exception(page)) {
2003 if (radix_tree_deref_retry(page)) {
2004 slot = radix_tree_iter_retry(&iter);
2005 continue;
2006 }
2007
2008 page = NULL;
2009 }
2010
2011 if (page &&
2012 page_count(page) - page_mapcount(page) != 1) {
2013 if (scan < LAST_SCAN)
2014 goto continue_resched;
2015
2016 /*
2017 * On the last scan, we clean up all those tags
2018 * we inserted; but make a note that we still
2019 * found pages pinned.
2020 */
2021 error = -EBUSY;
2022 }
2023
2024 spin_lock_irq(&mapping->tree_lock);
2025 radix_tree_tag_clear(&mapping->page_tree,
2026 iter.index, SHMEM_TAG_PINNED);
2027 spin_unlock_irq(&mapping->tree_lock);
2028continue_resched:
2029 if (need_resched()) {
2030 cond_resched_rcu();
2031 slot = radix_tree_iter_next(&iter);
2032 }
2033 }
2034 rcu_read_unlock();
2035 }
2036
2037 return error;
2038}
2039
2040#define F_ALL_SEALS (F_SEAL_SEAL | \
2041 F_SEAL_SHRINK | \
2042 F_SEAL_GROW | \
2043 F_SEAL_WRITE)
2044
2045int shmem_add_seals(struct file *file, unsigned int seals)
2046{
2047 struct inode *inode = file_inode(file);
2048 struct shmem_inode_info *info = SHMEM_I(inode);
2049 int error;
2050
2051 /*
2052 * SEALING
2053 * Sealing allows multiple parties to share a shmem-file but restrict
2054 * access to a specific subset of file operations. Seals can only be
2055 * added, but never removed. This way, mutually untrusted parties can
2056 * share common memory regions with a well-defined policy. A malicious
2057 * peer can thus never perform unwanted operations on a shared object.
2058 *
2059 * Seals are only supported on special shmem-files and always affect
2060 * the whole underlying inode. Once a seal is set, it may prevent some
2061 * kinds of access to the file. Currently, the following seals are
2062 * defined:
2063 * SEAL_SEAL: Prevent further seals from being set on this file
2064 * SEAL_SHRINK: Prevent the file from shrinking
2065 * SEAL_GROW: Prevent the file from growing
2066 * SEAL_WRITE: Prevent write access to the file
2067 *
2068 * As we don't require any trust relationship between two parties, we
2069 * must prevent seals from being removed. Therefore, sealing a file
2070 * only adds a given set of seals to the file, it never touches
2071 * existing seals. Furthermore, the "setting seals"-operation can be
2072 * sealed itself, which basically prevents any further seal from being
2073 * added.
2074 *
2075 * Semantics of sealing are only defined on volatile files. Only
2076 * anonymous shmem files support sealing. More importantly, seals are
2077 * never written to disk. Therefore, there's no plan to support it on
2078 * other file types.
2079 */
2080
2081 if (file->f_op != &shmem_file_operations)
2082 return -EINVAL;
2083 if (!(file->f_mode & FMODE_WRITE))
2084 return -EPERM;
2085 if (seals & ~(unsigned int)F_ALL_SEALS)
2086 return -EINVAL;
2087
2088 inode_lock(inode);
2089
2090 if (info->seals & F_SEAL_SEAL) {
2091 error = -EPERM;
2092 goto unlock;
2093 }
2094
2095 if ((seals & F_SEAL_WRITE) && !(info->seals & F_SEAL_WRITE)) {
2096 error = mapping_deny_writable(file->f_mapping);
2097 if (error)
2098 goto unlock;
2099
2100 error = shmem_wait_for_pins(file->f_mapping);
2101 if (error) {
2102 mapping_allow_writable(file->f_mapping);
2103 goto unlock;
2104 }
2105 }
2106
2107 info->seals |= seals;
2108 error = 0;
2109
2110unlock:
2111 inode_unlock(inode);
2112 return error;
2113}
2114EXPORT_SYMBOL_GPL(shmem_add_seals);
2115
2116int shmem_get_seals(struct file *file)
2117{
2118 if (file->f_op != &shmem_file_operations)
2119 return -EINVAL;
2120
2121 return SHMEM_I(file_inode(file))->seals;
2122}
2123EXPORT_SYMBOL_GPL(shmem_get_seals);
2124
2125long shmem_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
2126{
2127 long error;
2128
2129 switch (cmd) {
2130 case F_ADD_SEALS:
2131 /* disallow upper 32bit */
2132 if (arg > UINT_MAX)
2133 return -EINVAL;
2134
2135 error = shmem_add_seals(file, arg);
2136 break;
2137 case F_GET_SEALS:
2138 error = shmem_get_seals(file);
2139 break;
2140 default:
2141 error = -EINVAL;
2142 break;
2143 }
2144
2145 return error;
2146}
2147
2148static long shmem_fallocate(struct file *file, int mode, loff_t offset,
2149 loff_t len)
2150{
2151 struct inode *inode = file_inode(file);
2152 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2153 struct shmem_inode_info *info = SHMEM_I(inode);
2154 struct shmem_falloc shmem_falloc;
2155 pgoff_t start, index, end;
2156 int error;
2157
2158 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2159 return -EOPNOTSUPP;
2160
2161 inode_lock(inode);
2162
2163 if (mode & FALLOC_FL_PUNCH_HOLE) {
2164 struct address_space *mapping = file->f_mapping;
2165 loff_t unmap_start = round_up(offset, PAGE_SIZE);
2166 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
2167 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
2168
2169 /* protected by i_mutex */
2170 if (info->seals & F_SEAL_WRITE) {
2171 error = -EPERM;
2172 goto out;
2173 }
2174
2175 shmem_falloc.waitq = &shmem_falloc_waitq;
2176 shmem_falloc.start = unmap_start >> PAGE_SHIFT;
2177 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
2178 spin_lock(&inode->i_lock);
2179 inode->i_private = &shmem_falloc;
2180 spin_unlock(&inode->i_lock);
2181
2182 if ((u64)unmap_end > (u64)unmap_start)
2183 unmap_mapping_range(mapping, unmap_start,
2184 1 + unmap_end - unmap_start, 0);
2185 shmem_truncate_range(inode, offset, offset + len - 1);
2186 /* No need to unmap again: hole-punching leaves COWed pages */
2187
2188 spin_lock(&inode->i_lock);
2189 inode->i_private = NULL;
2190 wake_up_all(&shmem_falloc_waitq);
2191 spin_unlock(&inode->i_lock);
2192 error = 0;
2193 goto out;
2194 }
2195
2196 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
2197 error = inode_newsize_ok(inode, offset + len);
2198 if (error)
2199 goto out;
2200
2201 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
2202 error = -EPERM;
2203 goto out;
2204 }
2205
2206 start = offset >> PAGE_SHIFT;
2207 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2208 /* Try to avoid a swapstorm if len is impossible to satisfy */
2209 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
2210 error = -ENOSPC;
2211 goto out;
2212 }
2213
2214 shmem_falloc.waitq = NULL;
2215 shmem_falloc.start = start;
2216 shmem_falloc.next = start;
2217 shmem_falloc.nr_falloced = 0;
2218 shmem_falloc.nr_unswapped = 0;
2219 spin_lock(&inode->i_lock);
2220 inode->i_private = &shmem_falloc;
2221 spin_unlock(&inode->i_lock);
2222
2223 for (index = start; index < end; index++) {
2224 struct page *page;
2225
2226 /*
2227 * Good, the fallocate(2) manpage permits EINTR: we may have
2228 * been interrupted because we are using up too much memory.
2229 */
2230 if (signal_pending(current))
2231 error = -EINTR;
2232 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
2233 error = -ENOMEM;
2234 else
2235 error = shmem_getpage(inode, index, &page, SGP_FALLOC,
2236 NULL);
2237 if (error) {
2238 /* Remove the !PageUptodate pages we added */
2239 shmem_undo_range(inode,
2240 (loff_t)start << PAGE_SHIFT,
2241 (loff_t)index << PAGE_SHIFT, true);
2242 goto undone;
2243 }
2244
2245 /*
2246 * Inform shmem_writepage() how far we have reached.
2247 * No need for lock or barrier: we have the page lock.
2248 */
2249 shmem_falloc.next++;
2250 if (!PageUptodate(page))
2251 shmem_falloc.nr_falloced++;
2252
2253 /*
2254 * If !PageUptodate, leave it that way so that freeable pages
2255 * can be recognized if we need to rollback on error later.
2256 * But set_page_dirty so that memory pressure will swap rather
2257 * than free the pages we are allocating (and SGP_CACHE pages
2258 * might still be clean: we now need to mark those dirty too).
2259 */
2260 set_page_dirty(page);
2261 unlock_page(page);
2262 put_page(page);
2263 cond_resched();
2264 }
2265
2266 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
2267 i_size_write(inode, offset + len);
2268 inode->i_ctime = CURRENT_TIME;
2269undone:
2270 spin_lock(&inode->i_lock);
2271 inode->i_private = NULL;
2272 spin_unlock(&inode->i_lock);
2273out:
2274 inode_unlock(inode);
2275 return error;
2276}
2277
2278static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
2279{
2280 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
2281
2282 buf->f_type = TMPFS_MAGIC;
2283 buf->f_bsize = PAGE_SIZE;
2284 buf->f_namelen = NAME_MAX;
2285 if (sbinfo->max_blocks) {
2286 buf->f_blocks = sbinfo->max_blocks;
2287 buf->f_bavail =
2288 buf->f_bfree = sbinfo->max_blocks -
2289 percpu_counter_sum(&sbinfo->used_blocks);
2290 }
2291 if (sbinfo->max_inodes) {
2292 buf->f_files = sbinfo->max_inodes;
2293 buf->f_ffree = sbinfo->free_inodes;
2294 }
2295 /* else leave those fields 0 like simple_statfs */
2296 return 0;
2297}
2298
2299/*
2300 * File creation. Allocate an inode, and we're done..
2301 */
2302static int
2303shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
2304{
2305 struct inode *inode;
2306 int error = -ENOSPC;
2307
2308 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
2309 if (inode) {
2310 error = simple_acl_create(dir, inode);
2311 if (error)
2312 goto out_iput;
2313 error = security_inode_init_security(inode, dir,
2314 &dentry->d_name,
2315 shmem_initxattrs, NULL);
2316 if (error && error != -EOPNOTSUPP)
2317 goto out_iput;
2318
2319 error = 0;
2320 dir->i_size += BOGO_DIRENT_SIZE;
2321 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
2322 d_instantiate(dentry, inode);
2323 dget(dentry); /* Extra count - pin the dentry in core */
2324 }
2325 return error;
2326out_iput:
2327 iput(inode);
2328 return error;
2329}
2330
2331static int
2332shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
2333{
2334 struct inode *inode;
2335 int error = -ENOSPC;
2336
2337 inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
2338 if (inode) {
2339 error = security_inode_init_security(inode, dir,
2340 NULL,
2341 shmem_initxattrs, NULL);
2342 if (error && error != -EOPNOTSUPP)
2343 goto out_iput;
2344 error = simple_acl_create(dir, inode);
2345 if (error)
2346 goto out_iput;
2347 d_tmpfile(dentry, inode);
2348 }
2349 return error;
2350out_iput:
2351 iput(inode);
2352 return error;
2353}
2354
2355static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
2356{
2357 int error;
2358
2359 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
2360 return error;
2361 inc_nlink(dir);
2362 return 0;
2363}
2364
2365static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode,
2366 bool excl)
2367{
2368 return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
2369}
2370
2371/*
2372 * Link a file..
2373 */
2374static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
2375{
2376 struct inode *inode = d_inode(old_dentry);
2377 int ret;
2378
2379 /*
2380 * No ordinary (disk based) filesystem counts links as inodes;
2381 * but each new link needs a new dentry, pinning lowmem, and
2382 * tmpfs dentries cannot be pruned until they are unlinked.
2383 */
2384 ret = shmem_reserve_inode(inode->i_sb);
2385 if (ret)
2386 goto out;
2387
2388 dir->i_size += BOGO_DIRENT_SIZE;
2389 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
2390 inc_nlink(inode);
2391 ihold(inode); /* New dentry reference */
2392 dget(dentry); /* Extra pinning count for the created dentry */
2393 d_instantiate(dentry, inode);
2394out:
2395 return ret;
2396}
2397
2398static int shmem_unlink(struct inode *dir, struct dentry *dentry)
2399{
2400 struct inode *inode = d_inode(dentry);
2401
2402 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
2403 shmem_free_inode(inode->i_sb);
2404
2405 dir->i_size -= BOGO_DIRENT_SIZE;
2406 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
2407 drop_nlink(inode);
2408 dput(dentry); /* Undo the count from "create" - this does all the work */
2409 return 0;
2410}
2411
2412static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
2413{
2414 if (!simple_empty(dentry))
2415 return -ENOTEMPTY;
2416
2417 drop_nlink(d_inode(dentry));
2418 drop_nlink(dir);
2419 return shmem_unlink(dir, dentry);
2420}
2421
2422static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
2423{
2424 bool old_is_dir = d_is_dir(old_dentry);
2425 bool new_is_dir = d_is_dir(new_dentry);
2426
2427 if (old_dir != new_dir && old_is_dir != new_is_dir) {
2428 if (old_is_dir) {
2429 drop_nlink(old_dir);
2430 inc_nlink(new_dir);
2431 } else {
2432 drop_nlink(new_dir);
2433 inc_nlink(old_dir);
2434 }
2435 }
2436 old_dir->i_ctime = old_dir->i_mtime =
2437 new_dir->i_ctime = new_dir->i_mtime =
2438 d_inode(old_dentry)->i_ctime =
2439 d_inode(new_dentry)->i_ctime = CURRENT_TIME;
2440
2441 return 0;
2442}
2443
2444static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry)
2445{
2446 struct dentry *whiteout;
2447 int error;
2448
2449 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
2450 if (!whiteout)
2451 return -ENOMEM;
2452
2453 error = shmem_mknod(old_dir, whiteout,
2454 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
2455 dput(whiteout);
2456 if (error)
2457 return error;
2458
2459 /*
2460 * Cheat and hash the whiteout while the old dentry is still in
2461 * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
2462 *
2463 * d_lookup() will consistently find one of them at this point,
2464 * not sure which one, but that isn't even important.
2465 */
2466 d_rehash(whiteout);
2467 return 0;
2468}
2469
2470/*
2471 * The VFS layer already does all the dentry stuff for rename,
2472 * we just have to decrement the usage count for the target if
2473 * it exists so that the VFS layer correctly free's it when it
2474 * gets overwritten.
2475 */
2476static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags)
2477{
2478 struct inode *inode = d_inode(old_dentry);
2479 int they_are_dirs = S_ISDIR(inode->i_mode);
2480
2481 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
2482 return -EINVAL;
2483
2484 if (flags & RENAME_EXCHANGE)
2485 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry);
2486
2487 if (!simple_empty(new_dentry))
2488 return -ENOTEMPTY;
2489
2490 if (flags & RENAME_WHITEOUT) {
2491 int error;
2492
2493 error = shmem_whiteout(old_dir, old_dentry);
2494 if (error)
2495 return error;
2496 }
2497
2498 if (d_really_is_positive(new_dentry)) {
2499 (void) shmem_unlink(new_dir, new_dentry);
2500 if (they_are_dirs) {
2501 drop_nlink(d_inode(new_dentry));
2502 drop_nlink(old_dir);
2503 }
2504 } else if (they_are_dirs) {
2505 drop_nlink(old_dir);
2506 inc_nlink(new_dir);
2507 }
2508
2509 old_dir->i_size -= BOGO_DIRENT_SIZE;
2510 new_dir->i_size += BOGO_DIRENT_SIZE;
2511 old_dir->i_ctime = old_dir->i_mtime =
2512 new_dir->i_ctime = new_dir->i_mtime =
2513 inode->i_ctime = CURRENT_TIME;
2514 return 0;
2515}
2516
2517static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
2518{
2519 int error;
2520 int len;
2521 struct inode *inode;
2522 struct page *page;
2523 struct shmem_inode_info *info;
2524
2525 len = strlen(symname) + 1;
2526 if (len > PAGE_SIZE)
2527 return -ENAMETOOLONG;
2528
2529 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
2530 if (!inode)
2531 return -ENOSPC;
2532
2533 error = security_inode_init_security(inode, dir, &dentry->d_name,
2534 shmem_initxattrs, NULL);
2535 if (error) {
2536 if (error != -EOPNOTSUPP) {
2537 iput(inode);
2538 return error;
2539 }
2540 error = 0;
2541 }
2542
2543 info = SHMEM_I(inode);
2544 inode->i_size = len-1;
2545 if (len <= SHORT_SYMLINK_LEN) {
2546 inode->i_link = kmemdup(symname, len, GFP_KERNEL);
2547 if (!inode->i_link) {
2548 iput(inode);
2549 return -ENOMEM;
2550 }
2551 inode->i_op = &shmem_short_symlink_operations;
2552 } else {
2553 inode_nohighmem(inode);
2554 error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
2555 if (error) {
2556 iput(inode);
2557 return error;
2558 }
2559 inode->i_mapping->a_ops = &shmem_aops;
2560 inode->i_op = &shmem_symlink_inode_operations;
2561 memcpy(page_address(page), symname, len);
2562 SetPageUptodate(page);
2563 set_page_dirty(page);
2564 unlock_page(page);
2565 put_page(page);
2566 }
2567 dir->i_size += BOGO_DIRENT_SIZE;
2568 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
2569 d_instantiate(dentry, inode);
2570 dget(dentry);
2571 return 0;
2572}
2573
2574static void shmem_put_link(void *arg)
2575{
2576 mark_page_accessed(arg);
2577 put_page(arg);
2578}
2579
2580static const char *shmem_get_link(struct dentry *dentry,
2581 struct inode *inode,
2582 struct delayed_call *done)
2583{
2584 struct page *page = NULL;
2585 int error;
2586 if (!dentry) {
2587 page = find_get_page(inode->i_mapping, 0);
2588 if (!page)
2589 return ERR_PTR(-ECHILD);
2590 if (!PageUptodate(page)) {
2591 put_page(page);
2592 return ERR_PTR(-ECHILD);
2593 }
2594 } else {
2595 error = shmem_getpage(inode, 0, &page, SGP_READ, NULL);
2596 if (error)
2597 return ERR_PTR(error);
2598 unlock_page(page);
2599 }
2600 set_delayed_call(done, shmem_put_link, page);
2601 return page_address(page);
2602}
2603
2604#ifdef CONFIG_TMPFS_XATTR
2605/*
2606 * Superblocks without xattr inode operations may get some security.* xattr
2607 * support from the LSM "for free". As soon as we have any other xattrs
2608 * like ACLs, we also need to implement the security.* handlers at
2609 * filesystem level, though.
2610 */
2611
2612/*
2613 * Callback for security_inode_init_security() for acquiring xattrs.
2614 */
2615static int shmem_initxattrs(struct inode *inode,
2616 const struct xattr *xattr_array,
2617 void *fs_info)
2618{
2619 struct shmem_inode_info *info = SHMEM_I(inode);
2620 const struct xattr *xattr;
2621 struct simple_xattr *new_xattr;
2622 size_t len;
2623
2624 for (xattr = xattr_array; xattr->name != NULL; xattr++) {
2625 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
2626 if (!new_xattr)
2627 return -ENOMEM;
2628
2629 len = strlen(xattr->name) + 1;
2630 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
2631 GFP_KERNEL);
2632 if (!new_xattr->name) {
2633 kfree(new_xattr);
2634 return -ENOMEM;
2635 }
2636
2637 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
2638 XATTR_SECURITY_PREFIX_LEN);
2639 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
2640 xattr->name, len);
2641
2642 simple_xattr_list_add(&info->xattrs, new_xattr);
2643 }
2644
2645 return 0;
2646}
2647
2648static int shmem_xattr_handler_get(const struct xattr_handler *handler,
2649 struct dentry *dentry, const char *name,
2650 void *buffer, size_t size)
2651{
2652 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
2653
2654 name = xattr_full_name(handler, name);
2655 return simple_xattr_get(&info->xattrs, name, buffer, size);
2656}
2657
2658static int shmem_xattr_handler_set(const struct xattr_handler *handler,
2659 struct dentry *dentry, const char *name,
2660 const void *value, size_t size, int flags)
2661{
2662 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
2663
2664 name = xattr_full_name(handler, name);
2665 return simple_xattr_set(&info->xattrs, name, value, size, flags);
2666}
2667
2668static const struct xattr_handler shmem_security_xattr_handler = {
2669 .prefix = XATTR_SECURITY_PREFIX,
2670 .get = shmem_xattr_handler_get,
2671 .set = shmem_xattr_handler_set,
2672};
2673
2674static const struct xattr_handler shmem_trusted_xattr_handler = {
2675 .prefix = XATTR_TRUSTED_PREFIX,
2676 .get = shmem_xattr_handler_get,
2677 .set = shmem_xattr_handler_set,
2678};
2679
2680static const struct xattr_handler *shmem_xattr_handlers[] = {
2681#ifdef CONFIG_TMPFS_POSIX_ACL
2682 &posix_acl_access_xattr_handler,
2683 &posix_acl_default_xattr_handler,
2684#endif
2685 &shmem_security_xattr_handler,
2686 &shmem_trusted_xattr_handler,
2687 NULL
2688};
2689
2690static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
2691{
2692 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
2693 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
2694}
2695#endif /* CONFIG_TMPFS_XATTR */
2696
2697static const struct inode_operations shmem_short_symlink_operations = {
2698 .readlink = generic_readlink,
2699 .get_link = simple_get_link,
2700#ifdef CONFIG_TMPFS_XATTR
2701 .setxattr = generic_setxattr,
2702 .getxattr = generic_getxattr,
2703 .listxattr = shmem_listxattr,
2704 .removexattr = generic_removexattr,
2705#endif
2706};
2707
2708static const struct inode_operations shmem_symlink_inode_operations = {
2709 .readlink = generic_readlink,
2710 .get_link = shmem_get_link,
2711#ifdef CONFIG_TMPFS_XATTR
2712 .setxattr = generic_setxattr,
2713 .getxattr = generic_getxattr,
2714 .listxattr = shmem_listxattr,
2715 .removexattr = generic_removexattr,
2716#endif
2717};
2718
2719static struct dentry *shmem_get_parent(struct dentry *child)
2720{
2721 return ERR_PTR(-ESTALE);
2722}
2723
2724static int shmem_match(struct inode *ino, void *vfh)
2725{
2726 __u32 *fh = vfh;
2727 __u64 inum = fh[2];
2728 inum = (inum << 32) | fh[1];
2729 return ino->i_ino == inum && fh[0] == ino->i_generation;
2730}
2731
2732static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
2733 struct fid *fid, int fh_len, int fh_type)
2734{
2735 struct inode *inode;
2736 struct dentry *dentry = NULL;
2737 u64 inum;
2738
2739 if (fh_len < 3)
2740 return NULL;
2741
2742 inum = fid->raw[2];
2743 inum = (inum << 32) | fid->raw[1];
2744
2745 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
2746 shmem_match, fid->raw);
2747 if (inode) {
2748 dentry = d_find_alias(inode);
2749 iput(inode);
2750 }
2751
2752 return dentry;
2753}
2754
2755static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
2756 struct inode *parent)
2757{
2758 if (*len < 3) {
2759 *len = 3;
2760 return FILEID_INVALID;
2761 }
2762
2763 if (inode_unhashed(inode)) {
2764 /* Unfortunately insert_inode_hash is not idempotent,
2765 * so as we hash inodes here rather than at creation
2766 * time, we need a lock to ensure we only try
2767 * to do it once
2768 */
2769 static DEFINE_SPINLOCK(lock);
2770 spin_lock(&lock);
2771 if (inode_unhashed(inode))
2772 __insert_inode_hash(inode,
2773 inode->i_ino + inode->i_generation);
2774 spin_unlock(&lock);
2775 }
2776
2777 fh[0] = inode->i_generation;
2778 fh[1] = inode->i_ino;
2779 fh[2] = ((__u64)inode->i_ino) >> 32;
2780
2781 *len = 3;
2782 return 1;
2783}
2784
2785static const struct export_operations shmem_export_ops = {
2786 .get_parent = shmem_get_parent,
2787 .encode_fh = shmem_encode_fh,
2788 .fh_to_dentry = shmem_fh_to_dentry,
2789};
2790
2791static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
2792 bool remount)
2793{
2794 char *this_char, *value, *rest;
2795 struct mempolicy *mpol = NULL;
2796 uid_t uid;
2797 gid_t gid;
2798
2799 while (options != NULL) {
2800 this_char = options;
2801 for (;;) {
2802 /*
2803 * NUL-terminate this option: unfortunately,
2804 * mount options form a comma-separated list,
2805 * but mpol's nodelist may also contain commas.
2806 */
2807 options = strchr(options, ',');
2808 if (options == NULL)
2809 break;
2810 options++;
2811 if (!isdigit(*options)) {
2812 options[-1] = '\0';
2813 break;
2814 }
2815 }
2816 if (!*this_char)
2817 continue;
2818 if ((value = strchr(this_char,'=')) != NULL) {
2819 *value++ = 0;
2820 } else {
2821 pr_err("tmpfs: No value for mount option '%s'\n",
2822 this_char);
2823 goto error;
2824 }
2825
2826 if (!strcmp(this_char,"size")) {
2827 unsigned long long size;
2828 size = memparse(value,&rest);
2829 if (*rest == '%') {
2830 size <<= PAGE_SHIFT;
2831 size *= totalram_pages;
2832 do_div(size, 100);
2833 rest++;
2834 }
2835 if (*rest)
2836 goto bad_val;
2837 sbinfo->max_blocks =
2838 DIV_ROUND_UP(size, PAGE_SIZE);
2839 } else if (!strcmp(this_char,"nr_blocks")) {
2840 sbinfo->max_blocks = memparse(value, &rest);
2841 if (*rest)
2842 goto bad_val;
2843 } else if (!strcmp(this_char,"nr_inodes")) {
2844 sbinfo->max_inodes = memparse(value, &rest);
2845 if (*rest)
2846 goto bad_val;
2847 } else if (!strcmp(this_char,"mode")) {
2848 if (remount)
2849 continue;
2850 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
2851 if (*rest)
2852 goto bad_val;
2853 } else if (!strcmp(this_char,"uid")) {
2854 if (remount)
2855 continue;
2856 uid = simple_strtoul(value, &rest, 0);
2857 if (*rest)
2858 goto bad_val;
2859 sbinfo->uid = make_kuid(current_user_ns(), uid);
2860 if (!uid_valid(sbinfo->uid))
2861 goto bad_val;
2862 } else if (!strcmp(this_char,"gid")) {
2863 if (remount)
2864 continue;
2865 gid = simple_strtoul(value, &rest, 0);
2866 if (*rest)
2867 goto bad_val;
2868 sbinfo->gid = make_kgid(current_user_ns(), gid);
2869 if (!gid_valid(sbinfo->gid))
2870 goto bad_val;
2871 } else if (!strcmp(this_char,"mpol")) {
2872 mpol_put(mpol);
2873 mpol = NULL;
2874 if (mpol_parse_str(value, &mpol))
2875 goto bad_val;
2876 } else {
2877 pr_err("tmpfs: Bad mount option %s\n", this_char);
2878 goto error;
2879 }
2880 }
2881 sbinfo->mpol = mpol;
2882 return 0;
2883
2884bad_val:
2885 pr_err("tmpfs: Bad value '%s' for mount option '%s'\n",
2886 value, this_char);
2887error:
2888 mpol_put(mpol);
2889 return 1;
2890
2891}
2892
2893static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
2894{
2895 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2896 struct shmem_sb_info config = *sbinfo;
2897 unsigned long inodes;
2898 int error = -EINVAL;
2899
2900 config.mpol = NULL;
2901 if (shmem_parse_options(data, &config, true))
2902 return error;
2903
2904 spin_lock(&sbinfo->stat_lock);
2905 inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2906 if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
2907 goto out;
2908 if (config.max_inodes < inodes)
2909 goto out;
2910 /*
2911 * Those tests disallow limited->unlimited while any are in use;
2912 * but we must separately disallow unlimited->limited, because
2913 * in that case we have no record of how much is already in use.
2914 */
2915 if (config.max_blocks && !sbinfo->max_blocks)
2916 goto out;
2917 if (config.max_inodes && !sbinfo->max_inodes)
2918 goto out;
2919
2920 error = 0;
2921 sbinfo->max_blocks = config.max_blocks;
2922 sbinfo->max_inodes = config.max_inodes;
2923 sbinfo->free_inodes = config.max_inodes - inodes;
2924
2925 /*
2926 * Preserve previous mempolicy unless mpol remount option was specified.
2927 */
2928 if (config.mpol) {
2929 mpol_put(sbinfo->mpol);
2930 sbinfo->mpol = config.mpol; /* transfers initial ref */
2931 }
2932out:
2933 spin_unlock(&sbinfo->stat_lock);
2934 return error;
2935}
2936
2937static int shmem_show_options(struct seq_file *seq, struct dentry *root)
2938{
2939 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
2940
2941 if (sbinfo->max_blocks != shmem_default_max_blocks())
2942 seq_printf(seq, ",size=%luk",
2943 sbinfo->max_blocks << (PAGE_SHIFT - 10));
2944 if (sbinfo->max_inodes != shmem_default_max_inodes())
2945 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
2946 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
2947 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
2948 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
2949 seq_printf(seq, ",uid=%u",
2950 from_kuid_munged(&init_user_ns, sbinfo->uid));
2951 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
2952 seq_printf(seq, ",gid=%u",
2953 from_kgid_munged(&init_user_ns, sbinfo->gid));
2954 shmem_show_mpol(seq, sbinfo->mpol);
2955 return 0;
2956}
2957
2958#define MFD_NAME_PREFIX "memfd:"
2959#define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1)
2960#define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN)
2961
2962#define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING)
2963
2964SYSCALL_DEFINE2(memfd_create,
2965 const char __user *, uname,
2966 unsigned int, flags)
2967{
2968 struct shmem_inode_info *info;
2969 struct file *file;
2970 int fd, error;
2971 char *name;
2972 long len;
2973
2974 if (flags & ~(unsigned int)MFD_ALL_FLAGS)
2975 return -EINVAL;
2976
2977 /* length includes terminating zero */
2978 len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1);
2979 if (len <= 0)
2980 return -EFAULT;
2981 if (len > MFD_NAME_MAX_LEN + 1)
2982 return -EINVAL;
2983
2984 name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_TEMPORARY);
2985 if (!name)
2986 return -ENOMEM;
2987
2988 strcpy(name, MFD_NAME_PREFIX);
2989 if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) {
2990 error = -EFAULT;
2991 goto err_name;
2992 }
2993
2994 /* terminating-zero may have changed after strnlen_user() returned */
2995 if (name[len + MFD_NAME_PREFIX_LEN - 1]) {
2996 error = -EFAULT;
2997 goto err_name;
2998 }
2999
3000 fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0);
3001 if (fd < 0) {
3002 error = fd;
3003 goto err_name;
3004 }
3005
3006 file = shmem_file_setup(name, 0, VM_NORESERVE);
3007 if (IS_ERR(file)) {
3008 error = PTR_ERR(file);
3009 goto err_fd;
3010 }
3011 info = SHMEM_I(file_inode(file));
3012 file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE;
3013 file->f_flags |= O_RDWR | O_LARGEFILE;
3014 if (flags & MFD_ALLOW_SEALING)
3015 info->seals &= ~F_SEAL_SEAL;
3016
3017 fd_install(fd, file);
3018 kfree(name);
3019 return fd;
3020
3021err_fd:
3022 put_unused_fd(fd);
3023err_name:
3024 kfree(name);
3025 return error;
3026}
3027
3028#endif /* CONFIG_TMPFS */
3029
3030static void shmem_put_super(struct super_block *sb)
3031{
3032 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3033
3034 percpu_counter_destroy(&sbinfo->used_blocks);
3035 mpol_put(sbinfo->mpol);
3036 kfree(sbinfo);
3037 sb->s_fs_info = NULL;
3038}
3039
3040int shmem_fill_super(struct super_block *sb, void *data, int silent)
3041{
3042 struct inode *inode;
3043 struct shmem_sb_info *sbinfo;
3044 int err = -ENOMEM;
3045
3046 /* Round up to L1_CACHE_BYTES to resist false sharing */
3047 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
3048 L1_CACHE_BYTES), GFP_KERNEL);
3049 if (!sbinfo)
3050 return -ENOMEM;
3051
3052 sbinfo->mode = S_IRWXUGO | S_ISVTX;
3053 sbinfo->uid = current_fsuid();
3054 sbinfo->gid = current_fsgid();
3055 sb->s_fs_info = sbinfo;
3056
3057#ifdef CONFIG_TMPFS
3058 /*
3059 * Per default we only allow half of the physical ram per
3060 * tmpfs instance, limiting inodes to one per page of lowmem;
3061 * but the internal instance is left unlimited.
3062 */
3063 if (!(sb->s_flags & MS_KERNMOUNT)) {
3064 sbinfo->max_blocks = shmem_default_max_blocks();
3065 sbinfo->max_inodes = shmem_default_max_inodes();
3066 if (shmem_parse_options(data, sbinfo, false)) {
3067 err = -EINVAL;
3068 goto failed;
3069 }
3070 } else {
3071 sb->s_flags |= MS_NOUSER;
3072 }
3073 sb->s_export_op = &shmem_export_ops;
3074 sb->s_flags |= MS_NOSEC;
3075#else
3076 sb->s_flags |= MS_NOUSER;
3077#endif
3078
3079 spin_lock_init(&sbinfo->stat_lock);
3080 if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
3081 goto failed;
3082 sbinfo->free_inodes = sbinfo->max_inodes;
3083
3084 sb->s_maxbytes = MAX_LFS_FILESIZE;
3085 sb->s_blocksize = PAGE_SIZE;
3086 sb->s_blocksize_bits = PAGE_SHIFT;
3087 sb->s_magic = TMPFS_MAGIC;
3088 sb->s_op = &shmem_ops;
3089 sb->s_time_gran = 1;
3090#ifdef CONFIG_TMPFS_XATTR
3091 sb->s_xattr = shmem_xattr_handlers;
3092#endif
3093#ifdef CONFIG_TMPFS_POSIX_ACL
3094 sb->s_flags |= MS_POSIXACL;
3095#endif
3096
3097 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
3098 if (!inode)
3099 goto failed;
3100 inode->i_uid = sbinfo->uid;
3101 inode->i_gid = sbinfo->gid;
3102 sb->s_root = d_make_root(inode);
3103 if (!sb->s_root)
3104 goto failed;
3105 return 0;
3106
3107failed:
3108 shmem_put_super(sb);
3109 return err;
3110}
3111
3112static struct kmem_cache *shmem_inode_cachep;
3113
3114static struct inode *shmem_alloc_inode(struct super_block *sb)
3115{
3116 struct shmem_inode_info *info;
3117 info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
3118 if (!info)
3119 return NULL;
3120 return &info->vfs_inode;
3121}
3122
3123static void shmem_destroy_callback(struct rcu_head *head)
3124{
3125 struct inode *inode = container_of(head, struct inode, i_rcu);
3126 kfree(inode->i_link);
3127 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
3128}
3129
3130static void shmem_destroy_inode(struct inode *inode)
3131{
3132 if (S_ISREG(inode->i_mode))
3133 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
3134 call_rcu(&inode->i_rcu, shmem_destroy_callback);
3135}
3136
3137static void shmem_init_inode(void *foo)
3138{
3139 struct shmem_inode_info *info = foo;
3140 inode_init_once(&info->vfs_inode);
3141}
3142
3143static int shmem_init_inodecache(void)
3144{
3145 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
3146 sizeof(struct shmem_inode_info),
3147 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
3148 return 0;
3149}
3150
3151static void shmem_destroy_inodecache(void)
3152{
3153 kmem_cache_destroy(shmem_inode_cachep);
3154}
3155
3156static const struct address_space_operations shmem_aops = {
3157 .writepage = shmem_writepage,
3158 .set_page_dirty = __set_page_dirty_no_writeback,
3159#ifdef CONFIG_TMPFS
3160 .write_begin = shmem_write_begin,
3161 .write_end = shmem_write_end,
3162#endif
3163#ifdef CONFIG_MIGRATION
3164 .migratepage = migrate_page,
3165#endif
3166 .error_remove_page = generic_error_remove_page,
3167};
3168
3169static const struct file_operations shmem_file_operations = {
3170 .mmap = shmem_mmap,
3171#ifdef CONFIG_TMPFS
3172 .llseek = shmem_file_llseek,
3173 .read_iter = shmem_file_read_iter,
3174 .write_iter = generic_file_write_iter,
3175 .fsync = noop_fsync,
3176 .splice_read = shmem_file_splice_read,
3177 .splice_write = iter_file_splice_write,
3178 .fallocate = shmem_fallocate,
3179#endif
3180};
3181
3182static const struct inode_operations shmem_inode_operations = {
3183 .getattr = shmem_getattr,
3184 .setattr = shmem_setattr,
3185#ifdef CONFIG_TMPFS_XATTR
3186 .setxattr = generic_setxattr,
3187 .getxattr = generic_getxattr,
3188 .listxattr = shmem_listxattr,
3189 .removexattr = generic_removexattr,
3190 .set_acl = simple_set_acl,
3191#endif
3192};
3193
3194static const struct inode_operations shmem_dir_inode_operations = {
3195#ifdef CONFIG_TMPFS
3196 .create = shmem_create,
3197 .lookup = simple_lookup,
3198 .link = shmem_link,
3199 .unlink = shmem_unlink,
3200 .symlink = shmem_symlink,
3201 .mkdir = shmem_mkdir,
3202 .rmdir = shmem_rmdir,
3203 .mknod = shmem_mknod,
3204 .rename2 = shmem_rename2,
3205 .tmpfile = shmem_tmpfile,
3206#endif
3207#ifdef CONFIG_TMPFS_XATTR
3208 .setxattr = generic_setxattr,
3209 .getxattr = generic_getxattr,
3210 .listxattr = shmem_listxattr,
3211 .removexattr = generic_removexattr,
3212#endif
3213#ifdef CONFIG_TMPFS_POSIX_ACL
3214 .setattr = shmem_setattr,
3215 .set_acl = simple_set_acl,
3216#endif
3217};
3218
3219static const struct inode_operations shmem_special_inode_operations = {
3220#ifdef CONFIG_TMPFS_XATTR
3221 .setxattr = generic_setxattr,
3222 .getxattr = generic_getxattr,
3223 .listxattr = shmem_listxattr,
3224 .removexattr = generic_removexattr,
3225#endif
3226#ifdef CONFIG_TMPFS_POSIX_ACL
3227 .setattr = shmem_setattr,
3228 .set_acl = simple_set_acl,
3229#endif
3230};
3231
3232static const struct super_operations shmem_ops = {
3233 .alloc_inode = shmem_alloc_inode,
3234 .destroy_inode = shmem_destroy_inode,
3235#ifdef CONFIG_TMPFS
3236 .statfs = shmem_statfs,
3237 .remount_fs = shmem_remount_fs,
3238 .show_options = shmem_show_options,
3239#endif
3240 .evict_inode = shmem_evict_inode,
3241 .drop_inode = generic_delete_inode,
3242 .put_super = shmem_put_super,
3243};
3244
3245static const struct vm_operations_struct shmem_vm_ops = {
3246 .fault = shmem_fault,
3247 .map_pages = filemap_map_pages,
3248#ifdef CONFIG_NUMA
3249 .set_policy = shmem_set_policy,
3250 .get_policy = shmem_get_policy,
3251#endif
3252};
3253
3254static struct dentry *shmem_mount(struct file_system_type *fs_type,
3255 int flags, const char *dev_name, void *data)
3256{
3257 return mount_nodev(fs_type, flags, data, shmem_fill_super);
3258}
3259
3260static struct file_system_type shmem_fs_type = {
3261 .owner = THIS_MODULE,
3262 .name = "tmpfs",
3263 .mount = shmem_mount,
3264 .kill_sb = kill_litter_super,
3265 .fs_flags = FS_USERNS_MOUNT,
3266};
3267
3268int __init shmem_init(void)
3269{
3270 int error;
3271
3272 /* If rootfs called this, don't re-init */
3273 if (shmem_inode_cachep)
3274 return 0;
3275
3276 error = shmem_init_inodecache();
3277 if (error)
3278 goto out3;
3279
3280 error = register_filesystem(&shmem_fs_type);
3281 if (error) {
3282 pr_err("Could not register tmpfs\n");
3283 goto out2;
3284 }
3285
3286 shm_mnt = kern_mount(&shmem_fs_type);
3287 if (IS_ERR(shm_mnt)) {
3288 error = PTR_ERR(shm_mnt);
3289 pr_err("Could not kern_mount tmpfs\n");
3290 goto out1;
3291 }
3292 return 0;
3293
3294out1:
3295 unregister_filesystem(&shmem_fs_type);
3296out2:
3297 shmem_destroy_inodecache();
3298out3:
3299 shm_mnt = ERR_PTR(error);
3300 return error;
3301}
3302
3303#else /* !CONFIG_SHMEM */
3304
3305/*
3306 * tiny-shmem: simple shmemfs and tmpfs using ramfs code
3307 *
3308 * This is intended for small system where the benefits of the full
3309 * shmem code (swap-backed and resource-limited) are outweighed by
3310 * their complexity. On systems without swap this code should be
3311 * effectively equivalent, but much lighter weight.
3312 */
3313
3314static struct file_system_type shmem_fs_type = {
3315 .name = "tmpfs",
3316 .mount = ramfs_mount,
3317 .kill_sb = kill_litter_super,
3318 .fs_flags = FS_USERNS_MOUNT,
3319};
3320
3321int __init shmem_init(void)
3322{
3323 BUG_ON(register_filesystem(&shmem_fs_type) != 0);
3324
3325 shm_mnt = kern_mount(&shmem_fs_type);
3326 BUG_ON(IS_ERR(shm_mnt));
3327
3328 return 0;
3329}
3330
3331int shmem_unuse(swp_entry_t swap, struct page *page)
3332{
3333 return 0;
3334}
3335
3336int shmem_lock(struct file *file, int lock, struct user_struct *user)
3337{
3338 return 0;
3339}
3340
3341void shmem_unlock_mapping(struct address_space *mapping)
3342{
3343}
3344
3345void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
3346{
3347 truncate_inode_pages_range(inode->i_mapping, lstart, lend);
3348}
3349EXPORT_SYMBOL_GPL(shmem_truncate_range);
3350
3351#define shmem_vm_ops generic_file_vm_ops
3352#define shmem_file_operations ramfs_file_operations
3353#define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev)
3354#define shmem_acct_size(flags, size) 0
3355#define shmem_unacct_size(flags, size) do {} while (0)
3356
3357#endif /* CONFIG_SHMEM */
3358
3359/* common code */
3360
3361static struct dentry_operations anon_ops = {
3362 .d_dname = simple_dname
3363};
3364
3365static struct file *__shmem_file_setup(const char *name, loff_t size,
3366 unsigned long flags, unsigned int i_flags)
3367{
3368 struct file *res;
3369 struct inode *inode;
3370 struct path path;
3371 struct super_block *sb;
3372 struct qstr this;
3373
3374 if (IS_ERR(shm_mnt))
3375 return ERR_CAST(shm_mnt);
3376
3377 if (size < 0 || size > MAX_LFS_FILESIZE)
3378 return ERR_PTR(-EINVAL);
3379
3380 if (shmem_acct_size(flags, size))
3381 return ERR_PTR(-ENOMEM);
3382
3383 res = ERR_PTR(-ENOMEM);
3384 this.name = name;
3385 this.len = strlen(name);
3386 this.hash = 0; /* will go */
3387 sb = shm_mnt->mnt_sb;
3388 path.mnt = mntget(shm_mnt);
3389 path.dentry = d_alloc_pseudo(sb, &this);
3390 if (!path.dentry)
3391 goto put_memory;
3392 d_set_d_op(path.dentry, &anon_ops);
3393
3394 res = ERR_PTR(-ENOSPC);
3395 inode = shmem_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
3396 if (!inode)
3397 goto put_memory;
3398
3399 inode->i_flags |= i_flags;
3400 d_instantiate(path.dentry, inode);
3401 inode->i_size = size;
3402 clear_nlink(inode); /* It is unlinked */
3403 res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
3404 if (IS_ERR(res))
3405 goto put_path;
3406
3407 res = alloc_file(&path, FMODE_WRITE | FMODE_READ,
3408 &shmem_file_operations);
3409 if (IS_ERR(res))
3410 goto put_path;
3411
3412 return res;
3413
3414put_memory:
3415 shmem_unacct_size(flags, size);
3416put_path:
3417 path_put(&path);
3418 return res;
3419}
3420
3421/**
3422 * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
3423 * kernel internal. There will be NO LSM permission checks against the
3424 * underlying inode. So users of this interface must do LSM checks at a
3425 * higher layer. The users are the big_key and shm implementations. LSM
3426 * checks are provided at the key or shm level rather than the inode.
3427 * @name: name for dentry (to be seen in /proc/<pid>/maps
3428 * @size: size to be set for the file
3429 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
3430 */
3431struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
3432{
3433 return __shmem_file_setup(name, size, flags, S_PRIVATE);
3434}
3435
3436/**
3437 * shmem_file_setup - get an unlinked file living in tmpfs
3438 * @name: name for dentry (to be seen in /proc/<pid>/maps
3439 * @size: size to be set for the file
3440 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
3441 */
3442struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
3443{
3444 return __shmem_file_setup(name, size, flags, 0);
3445}
3446EXPORT_SYMBOL_GPL(shmem_file_setup);
3447
3448/**
3449 * shmem_zero_setup - setup a shared anonymous mapping
3450 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
3451 */
3452int shmem_zero_setup(struct vm_area_struct *vma)
3453{
3454 struct file *file;
3455 loff_t size = vma->vm_end - vma->vm_start;
3456
3457 /*
3458 * Cloning a new file under mmap_sem leads to a lock ordering conflict
3459 * between XFS directory reading and selinux: since this file is only
3460 * accessible to the user through its mapping, use S_PRIVATE flag to
3461 * bypass file security, in the same way as shmem_kernel_file_setup().
3462 */
3463 file = __shmem_file_setup("dev/zero", size, vma->vm_flags, S_PRIVATE);
3464 if (IS_ERR(file))
3465 return PTR_ERR(file);
3466
3467 if (vma->vm_file)
3468 fput(vma->vm_file);
3469 vma->vm_file = file;
3470 vma->vm_ops = &shmem_vm_ops;
3471 return 0;
3472}
3473
3474/**
3475 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
3476 * @mapping: the page's address_space
3477 * @index: the page index
3478 * @gfp: the page allocator flags to use if allocating
3479 *
3480 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
3481 * with any new page allocations done using the specified allocation flags.
3482 * But read_cache_page_gfp() uses the ->readpage() method: which does not
3483 * suit tmpfs, since it may have pages in swapcache, and needs to find those
3484 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
3485 *
3486 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
3487 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
3488 */
3489struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
3490 pgoff_t index, gfp_t gfp)
3491{
3492#ifdef CONFIG_SHMEM
3493 struct inode *inode = mapping->host;
3494 struct page *page;
3495 int error;
3496
3497 BUG_ON(mapping->a_ops != &shmem_aops);
3498 error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, gfp, NULL);
3499 if (error)
3500 page = ERR_PTR(error);
3501 else
3502 unlock_page(page);
3503 return page;
3504#else
3505 /*
3506 * The tiny !SHMEM case uses ramfs without swap
3507 */
3508 return read_cache_page_gfp(mapping, index, gfp);
3509#endif
3510}
3511EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);
1/*
2 * Resizable virtual memory filesystem for Linux.
3 *
4 * Copyright (C) 2000 Linus Torvalds.
5 * 2000 Transmeta Corp.
6 * 2000-2001 Christoph Rohland
7 * 2000-2001 SAP AG
8 * 2002 Red Hat Inc.
9 * Copyright (C) 2002-2011 Hugh Dickins.
10 * Copyright (C) 2011 Google Inc.
11 * Copyright (C) 2002-2005 VERITAS Software Corporation.
12 * Copyright (C) 2004 Andi Kleen, SuSE Labs
13 *
14 * Extended attribute support for tmpfs:
15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17 *
18 * tiny-shmem:
19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
20 *
21 * This file is released under the GPL.
22 */
23
24#include <linux/fs.h>
25#include <linux/init.h>
26#include <linux/vfs.h>
27#include <linux/mount.h>
28#include <linux/pagemap.h>
29#include <linux/file.h>
30#include <linux/mm.h>
31#include <linux/module.h>
32#include <linux/swap.h>
33
34static struct vfsmount *shm_mnt;
35
36#ifdef CONFIG_SHMEM
37/*
38 * This virtual memory filesystem is heavily based on the ramfs. It
39 * extends ramfs by the ability to use swap and honor resource limits
40 * which makes it a completely usable filesystem.
41 */
42
43#include <linux/xattr.h>
44#include <linux/exportfs.h>
45#include <linux/posix_acl.h>
46#include <linux/generic_acl.h>
47#include <linux/mman.h>
48#include <linux/string.h>
49#include <linux/slab.h>
50#include <linux/backing-dev.h>
51#include <linux/shmem_fs.h>
52#include <linux/writeback.h>
53#include <linux/blkdev.h>
54#include <linux/pagevec.h>
55#include <linux/percpu_counter.h>
56#include <linux/splice.h>
57#include <linux/security.h>
58#include <linux/swapops.h>
59#include <linux/mempolicy.h>
60#include <linux/namei.h>
61#include <linux/ctype.h>
62#include <linux/migrate.h>
63#include <linux/highmem.h>
64#include <linux/seq_file.h>
65#include <linux/magic.h>
66
67#include <asm/uaccess.h>
68#include <asm/pgtable.h>
69
70#define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
71#define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
72
73/* Pretend that each entry is of this size in directory's i_size */
74#define BOGO_DIRENT_SIZE 20
75
76/* Symlink up to this size is kmalloc'ed instead of using a swappable page */
77#define SHORT_SYMLINK_LEN 128
78
79struct shmem_xattr {
80 struct list_head list; /* anchored by shmem_inode_info->xattr_list */
81 char *name; /* xattr name */
82 size_t size;
83 char value[0];
84};
85
86/* Flag allocation requirements to shmem_getpage */
87enum sgp_type {
88 SGP_READ, /* don't exceed i_size, don't allocate page */
89 SGP_CACHE, /* don't exceed i_size, may allocate page */
90 SGP_DIRTY, /* like SGP_CACHE, but set new page dirty */
91 SGP_WRITE, /* may exceed i_size, may allocate page */
92};
93
94#ifdef CONFIG_TMPFS
95static unsigned long shmem_default_max_blocks(void)
96{
97 return totalram_pages / 2;
98}
99
100static unsigned long shmem_default_max_inodes(void)
101{
102 return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
103}
104#endif
105
106static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
107 struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type);
108
109static inline int shmem_getpage(struct inode *inode, pgoff_t index,
110 struct page **pagep, enum sgp_type sgp, int *fault_type)
111{
112 return shmem_getpage_gfp(inode, index, pagep, sgp,
113 mapping_gfp_mask(inode->i_mapping), fault_type);
114}
115
116static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
117{
118 return sb->s_fs_info;
119}
120
121/*
122 * shmem_file_setup pre-accounts the whole fixed size of a VM object,
123 * for shared memory and for shared anonymous (/dev/zero) mappings
124 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
125 * consistent with the pre-accounting of private mappings ...
126 */
127static inline int shmem_acct_size(unsigned long flags, loff_t size)
128{
129 return (flags & VM_NORESERVE) ?
130 0 : security_vm_enough_memory_kern(VM_ACCT(size));
131}
132
133static inline void shmem_unacct_size(unsigned long flags, loff_t size)
134{
135 if (!(flags & VM_NORESERVE))
136 vm_unacct_memory(VM_ACCT(size));
137}
138
139/*
140 * ... whereas tmpfs objects are accounted incrementally as
141 * pages are allocated, in order to allow huge sparse files.
142 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
143 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
144 */
145static inline int shmem_acct_block(unsigned long flags)
146{
147 return (flags & VM_NORESERVE) ?
148 security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE)) : 0;
149}
150
151static inline void shmem_unacct_blocks(unsigned long flags, long pages)
152{
153 if (flags & VM_NORESERVE)
154 vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
155}
156
157static const struct super_operations shmem_ops;
158static const struct address_space_operations shmem_aops;
159static const struct file_operations shmem_file_operations;
160static const struct inode_operations shmem_inode_operations;
161static const struct inode_operations shmem_dir_inode_operations;
162static const struct inode_operations shmem_special_inode_operations;
163static const struct vm_operations_struct shmem_vm_ops;
164
165static struct backing_dev_info shmem_backing_dev_info __read_mostly = {
166 .ra_pages = 0, /* No readahead */
167 .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED,
168};
169
170static LIST_HEAD(shmem_swaplist);
171static DEFINE_MUTEX(shmem_swaplist_mutex);
172
173static int shmem_reserve_inode(struct super_block *sb)
174{
175 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
176 if (sbinfo->max_inodes) {
177 spin_lock(&sbinfo->stat_lock);
178 if (!sbinfo->free_inodes) {
179 spin_unlock(&sbinfo->stat_lock);
180 return -ENOSPC;
181 }
182 sbinfo->free_inodes--;
183 spin_unlock(&sbinfo->stat_lock);
184 }
185 return 0;
186}
187
188static void shmem_free_inode(struct super_block *sb)
189{
190 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
191 if (sbinfo->max_inodes) {
192 spin_lock(&sbinfo->stat_lock);
193 sbinfo->free_inodes++;
194 spin_unlock(&sbinfo->stat_lock);
195 }
196}
197
198/**
199 * shmem_recalc_inode - recalculate the block usage of an inode
200 * @inode: inode to recalc
201 *
202 * We have to calculate the free blocks since the mm can drop
203 * undirtied hole pages behind our back.
204 *
205 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped
206 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
207 *
208 * It has to be called with the spinlock held.
209 */
210static void shmem_recalc_inode(struct inode *inode)
211{
212 struct shmem_inode_info *info = SHMEM_I(inode);
213 long freed;
214
215 freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
216 if (freed > 0) {
217 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
218 if (sbinfo->max_blocks)
219 percpu_counter_add(&sbinfo->used_blocks, -freed);
220 info->alloced -= freed;
221 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
222 shmem_unacct_blocks(info->flags, freed);
223 }
224}
225
226/*
227 * Replace item expected in radix tree by a new item, while holding tree lock.
228 */
229static int shmem_radix_tree_replace(struct address_space *mapping,
230 pgoff_t index, void *expected, void *replacement)
231{
232 void **pslot;
233 void *item = NULL;
234
235 VM_BUG_ON(!expected);
236 pslot = radix_tree_lookup_slot(&mapping->page_tree, index);
237 if (pslot)
238 item = radix_tree_deref_slot_protected(pslot,
239 &mapping->tree_lock);
240 if (item != expected)
241 return -ENOENT;
242 if (replacement)
243 radix_tree_replace_slot(pslot, replacement);
244 else
245 radix_tree_delete(&mapping->page_tree, index);
246 return 0;
247}
248
249/*
250 * Like add_to_page_cache_locked, but error if expected item has gone.
251 */
252static int shmem_add_to_page_cache(struct page *page,
253 struct address_space *mapping,
254 pgoff_t index, gfp_t gfp, void *expected)
255{
256 int error = 0;
257
258 VM_BUG_ON(!PageLocked(page));
259 VM_BUG_ON(!PageSwapBacked(page));
260
261 if (!expected)
262 error = radix_tree_preload(gfp & GFP_RECLAIM_MASK);
263 if (!error) {
264 page_cache_get(page);
265 page->mapping = mapping;
266 page->index = index;
267
268 spin_lock_irq(&mapping->tree_lock);
269 if (!expected)
270 error = radix_tree_insert(&mapping->page_tree,
271 index, page);
272 else
273 error = shmem_radix_tree_replace(mapping, index,
274 expected, page);
275 if (!error) {
276 mapping->nrpages++;
277 __inc_zone_page_state(page, NR_FILE_PAGES);
278 __inc_zone_page_state(page, NR_SHMEM);
279 spin_unlock_irq(&mapping->tree_lock);
280 } else {
281 page->mapping = NULL;
282 spin_unlock_irq(&mapping->tree_lock);
283 page_cache_release(page);
284 }
285 if (!expected)
286 radix_tree_preload_end();
287 }
288 if (error)
289 mem_cgroup_uncharge_cache_page(page);
290 return error;
291}
292
293/*
294 * Like delete_from_page_cache, but substitutes swap for page.
295 */
296static void shmem_delete_from_page_cache(struct page *page, void *radswap)
297{
298 struct address_space *mapping = page->mapping;
299 int error;
300
301 spin_lock_irq(&mapping->tree_lock);
302 error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
303 page->mapping = NULL;
304 mapping->nrpages--;
305 __dec_zone_page_state(page, NR_FILE_PAGES);
306 __dec_zone_page_state(page, NR_SHMEM);
307 spin_unlock_irq(&mapping->tree_lock);
308 page_cache_release(page);
309 BUG_ON(error);
310}
311
312/*
313 * Like find_get_pages, but collecting swap entries as well as pages.
314 */
315static unsigned shmem_find_get_pages_and_swap(struct address_space *mapping,
316 pgoff_t start, unsigned int nr_pages,
317 struct page **pages, pgoff_t *indices)
318{
319 unsigned int i;
320 unsigned int ret;
321 unsigned int nr_found;
322
323 rcu_read_lock();
324restart:
325 nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
326 (void ***)pages, indices, start, nr_pages);
327 ret = 0;
328 for (i = 0; i < nr_found; i++) {
329 struct page *page;
330repeat:
331 page = radix_tree_deref_slot((void **)pages[i]);
332 if (unlikely(!page))
333 continue;
334 if (radix_tree_exception(page)) {
335 if (radix_tree_deref_retry(page))
336 goto restart;
337 /*
338 * Otherwise, we must be storing a swap entry
339 * here as an exceptional entry: so return it
340 * without attempting to raise page count.
341 */
342 goto export;
343 }
344 if (!page_cache_get_speculative(page))
345 goto repeat;
346
347 /* Has the page moved? */
348 if (unlikely(page != *((void **)pages[i]))) {
349 page_cache_release(page);
350 goto repeat;
351 }
352export:
353 indices[ret] = indices[i];
354 pages[ret] = page;
355 ret++;
356 }
357 if (unlikely(!ret && nr_found))
358 goto restart;
359 rcu_read_unlock();
360 return ret;
361}
362
363/*
364 * Remove swap entry from radix tree, free the swap and its page cache.
365 */
366static int shmem_free_swap(struct address_space *mapping,
367 pgoff_t index, void *radswap)
368{
369 int error;
370
371 spin_lock_irq(&mapping->tree_lock);
372 error = shmem_radix_tree_replace(mapping, index, radswap, NULL);
373 spin_unlock_irq(&mapping->tree_lock);
374 if (!error)
375 free_swap_and_cache(radix_to_swp_entry(radswap));
376 return error;
377}
378
379/*
380 * Pagevec may contain swap entries, so shuffle up pages before releasing.
381 */
382static void shmem_pagevec_release(struct pagevec *pvec)
383{
384 int i, j;
385
386 for (i = 0, j = 0; i < pagevec_count(pvec); i++) {
387 struct page *page = pvec->pages[i];
388 if (!radix_tree_exceptional_entry(page))
389 pvec->pages[j++] = page;
390 }
391 pvec->nr = j;
392 pagevec_release(pvec);
393}
394
395/*
396 * Remove range of pages and swap entries from radix tree, and free them.
397 */
398void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
399{
400 struct address_space *mapping = inode->i_mapping;
401 struct shmem_inode_info *info = SHMEM_I(inode);
402 pgoff_t start = (lstart + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
403 unsigned partial = lstart & (PAGE_CACHE_SIZE - 1);
404 pgoff_t end = (lend >> PAGE_CACHE_SHIFT);
405 struct pagevec pvec;
406 pgoff_t indices[PAGEVEC_SIZE];
407 long nr_swaps_freed = 0;
408 pgoff_t index;
409 int i;
410
411 BUG_ON((lend & (PAGE_CACHE_SIZE - 1)) != (PAGE_CACHE_SIZE - 1));
412
413 pagevec_init(&pvec, 0);
414 index = start;
415 while (index <= end) {
416 pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
417 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
418 pvec.pages, indices);
419 if (!pvec.nr)
420 break;
421 mem_cgroup_uncharge_start();
422 for (i = 0; i < pagevec_count(&pvec); i++) {
423 struct page *page = pvec.pages[i];
424
425 index = indices[i];
426 if (index > end)
427 break;
428
429 if (radix_tree_exceptional_entry(page)) {
430 nr_swaps_freed += !shmem_free_swap(mapping,
431 index, page);
432 continue;
433 }
434
435 if (!trylock_page(page))
436 continue;
437 if (page->mapping == mapping) {
438 VM_BUG_ON(PageWriteback(page));
439 truncate_inode_page(mapping, page);
440 }
441 unlock_page(page);
442 }
443 shmem_pagevec_release(&pvec);
444 mem_cgroup_uncharge_end();
445 cond_resched();
446 index++;
447 }
448
449 if (partial) {
450 struct page *page = NULL;
451 shmem_getpage(inode, start - 1, &page, SGP_READ, NULL);
452 if (page) {
453 zero_user_segment(page, partial, PAGE_CACHE_SIZE);
454 set_page_dirty(page);
455 unlock_page(page);
456 page_cache_release(page);
457 }
458 }
459
460 index = start;
461 for ( ; ; ) {
462 cond_resched();
463 pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
464 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
465 pvec.pages, indices);
466 if (!pvec.nr) {
467 if (index == start)
468 break;
469 index = start;
470 continue;
471 }
472 if (index == start && indices[0] > end) {
473 shmem_pagevec_release(&pvec);
474 break;
475 }
476 mem_cgroup_uncharge_start();
477 for (i = 0; i < pagevec_count(&pvec); i++) {
478 struct page *page = pvec.pages[i];
479
480 index = indices[i];
481 if (index > end)
482 break;
483
484 if (radix_tree_exceptional_entry(page)) {
485 nr_swaps_freed += !shmem_free_swap(mapping,
486 index, page);
487 continue;
488 }
489
490 lock_page(page);
491 if (page->mapping == mapping) {
492 VM_BUG_ON(PageWriteback(page));
493 truncate_inode_page(mapping, page);
494 }
495 unlock_page(page);
496 }
497 shmem_pagevec_release(&pvec);
498 mem_cgroup_uncharge_end();
499 index++;
500 }
501
502 spin_lock(&info->lock);
503 info->swapped -= nr_swaps_freed;
504 shmem_recalc_inode(inode);
505 spin_unlock(&info->lock);
506
507 inode->i_ctime = inode->i_mtime = CURRENT_TIME;
508}
509EXPORT_SYMBOL_GPL(shmem_truncate_range);
510
511static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
512{
513 struct inode *inode = dentry->d_inode;
514 int error;
515
516 error = inode_change_ok(inode, attr);
517 if (error)
518 return error;
519
520 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
521 loff_t oldsize = inode->i_size;
522 loff_t newsize = attr->ia_size;
523
524 if (newsize != oldsize) {
525 i_size_write(inode, newsize);
526 inode->i_ctime = inode->i_mtime = CURRENT_TIME;
527 }
528 if (newsize < oldsize) {
529 loff_t holebegin = round_up(newsize, PAGE_SIZE);
530 unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
531 shmem_truncate_range(inode, newsize, (loff_t)-1);
532 /* unmap again to remove racily COWed private pages */
533 unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
534 }
535 }
536
537 setattr_copy(inode, attr);
538#ifdef CONFIG_TMPFS_POSIX_ACL
539 if (attr->ia_valid & ATTR_MODE)
540 error = generic_acl_chmod(inode);
541#endif
542 return error;
543}
544
545static void shmem_evict_inode(struct inode *inode)
546{
547 struct shmem_inode_info *info = SHMEM_I(inode);
548 struct shmem_xattr *xattr, *nxattr;
549
550 if (inode->i_mapping->a_ops == &shmem_aops) {
551 shmem_unacct_size(info->flags, inode->i_size);
552 inode->i_size = 0;
553 shmem_truncate_range(inode, 0, (loff_t)-1);
554 if (!list_empty(&info->swaplist)) {
555 mutex_lock(&shmem_swaplist_mutex);
556 list_del_init(&info->swaplist);
557 mutex_unlock(&shmem_swaplist_mutex);
558 }
559 } else
560 kfree(info->symlink);
561
562 list_for_each_entry_safe(xattr, nxattr, &info->xattr_list, list) {
563 kfree(xattr->name);
564 kfree(xattr);
565 }
566 BUG_ON(inode->i_blocks);
567 shmem_free_inode(inode->i_sb);
568 end_writeback(inode);
569}
570
571/*
572 * If swap found in inode, free it and move page from swapcache to filecache.
573 */
574static int shmem_unuse_inode(struct shmem_inode_info *info,
575 swp_entry_t swap, struct page *page)
576{
577 struct address_space *mapping = info->vfs_inode.i_mapping;
578 void *radswap;
579 pgoff_t index;
580 int error;
581
582 radswap = swp_to_radix_entry(swap);
583 index = radix_tree_locate_item(&mapping->page_tree, radswap);
584 if (index == -1)
585 return 0;
586
587 /*
588 * Move _head_ to start search for next from here.
589 * But be careful: shmem_evict_inode checks list_empty without taking
590 * mutex, and there's an instant in list_move_tail when info->swaplist
591 * would appear empty, if it were the only one on shmem_swaplist.
592 */
593 if (shmem_swaplist.next != &info->swaplist)
594 list_move_tail(&shmem_swaplist, &info->swaplist);
595
596 /*
597 * We rely on shmem_swaplist_mutex, not only to protect the swaplist,
598 * but also to hold up shmem_evict_inode(): so inode cannot be freed
599 * beneath us (pagelock doesn't help until the page is in pagecache).
600 */
601 error = shmem_add_to_page_cache(page, mapping, index,
602 GFP_NOWAIT, radswap);
603 /* which does mem_cgroup_uncharge_cache_page on error */
604
605 if (error != -ENOMEM) {
606 /*
607 * Truncation and eviction use free_swap_and_cache(), which
608 * only does trylock page: if we raced, best clean up here.
609 */
610 delete_from_swap_cache(page);
611 set_page_dirty(page);
612 if (!error) {
613 spin_lock(&info->lock);
614 info->swapped--;
615 spin_unlock(&info->lock);
616 swap_free(swap);
617 }
618 error = 1; /* not an error, but entry was found */
619 }
620 return error;
621}
622
623/*
624 * Search through swapped inodes to find and replace swap by page.
625 */
626int shmem_unuse(swp_entry_t swap, struct page *page)
627{
628 struct list_head *this, *next;
629 struct shmem_inode_info *info;
630 int found = 0;
631 int error;
632
633 /*
634 * Charge page using GFP_KERNEL while we can wait, before taking
635 * the shmem_swaplist_mutex which might hold up shmem_writepage().
636 * Charged back to the user (not to caller) when swap account is used.
637 */
638 error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL);
639 if (error)
640 goto out;
641 /* No radix_tree_preload: swap entry keeps a place for page in tree */
642
643 mutex_lock(&shmem_swaplist_mutex);
644 list_for_each_safe(this, next, &shmem_swaplist) {
645 info = list_entry(this, struct shmem_inode_info, swaplist);
646 if (info->swapped)
647 found = shmem_unuse_inode(info, swap, page);
648 else
649 list_del_init(&info->swaplist);
650 cond_resched();
651 if (found)
652 break;
653 }
654 mutex_unlock(&shmem_swaplist_mutex);
655
656 if (!found)
657 mem_cgroup_uncharge_cache_page(page);
658 if (found < 0)
659 error = found;
660out:
661 unlock_page(page);
662 page_cache_release(page);
663 return error;
664}
665
666/*
667 * Move the page from the page cache to the swap cache.
668 */
669static int shmem_writepage(struct page *page, struct writeback_control *wbc)
670{
671 struct shmem_inode_info *info;
672 struct address_space *mapping;
673 struct inode *inode;
674 swp_entry_t swap;
675 pgoff_t index;
676
677 BUG_ON(!PageLocked(page));
678 mapping = page->mapping;
679 index = page->index;
680 inode = mapping->host;
681 info = SHMEM_I(inode);
682 if (info->flags & VM_LOCKED)
683 goto redirty;
684 if (!total_swap_pages)
685 goto redirty;
686
687 /*
688 * shmem_backing_dev_info's capabilities prevent regular writeback or
689 * sync from ever calling shmem_writepage; but a stacking filesystem
690 * might use ->writepage of its underlying filesystem, in which case
691 * tmpfs should write out to swap only in response to memory pressure,
692 * and not for the writeback threads or sync.
693 */
694 if (!wbc->for_reclaim) {
695 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */
696 goto redirty;
697 }
698 swap = get_swap_page();
699 if (!swap.val)
700 goto redirty;
701
702 /*
703 * Add inode to shmem_unuse()'s list of swapped-out inodes,
704 * if it's not already there. Do it now before the page is
705 * moved to swap cache, when its pagelock no longer protects
706 * the inode from eviction. But don't unlock the mutex until
707 * we've incremented swapped, because shmem_unuse_inode() will
708 * prune a !swapped inode from the swaplist under this mutex.
709 */
710 mutex_lock(&shmem_swaplist_mutex);
711 if (list_empty(&info->swaplist))
712 list_add_tail(&info->swaplist, &shmem_swaplist);
713
714 if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
715 swap_shmem_alloc(swap);
716 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
717
718 spin_lock(&info->lock);
719 info->swapped++;
720 shmem_recalc_inode(inode);
721 spin_unlock(&info->lock);
722
723 mutex_unlock(&shmem_swaplist_mutex);
724 BUG_ON(page_mapped(page));
725 swap_writepage(page, wbc);
726 return 0;
727 }
728
729 mutex_unlock(&shmem_swaplist_mutex);
730 swapcache_free(swap, NULL);
731redirty:
732 set_page_dirty(page);
733 if (wbc->for_reclaim)
734 return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */
735 unlock_page(page);
736 return 0;
737}
738
739#ifdef CONFIG_NUMA
740#ifdef CONFIG_TMPFS
741static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
742{
743 char buffer[64];
744
745 if (!mpol || mpol->mode == MPOL_DEFAULT)
746 return; /* show nothing */
747
748 mpol_to_str(buffer, sizeof(buffer), mpol, 1);
749
750 seq_printf(seq, ",mpol=%s", buffer);
751}
752
753static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
754{
755 struct mempolicy *mpol = NULL;
756 if (sbinfo->mpol) {
757 spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */
758 mpol = sbinfo->mpol;
759 mpol_get(mpol);
760 spin_unlock(&sbinfo->stat_lock);
761 }
762 return mpol;
763}
764#endif /* CONFIG_TMPFS */
765
766static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
767 struct shmem_inode_info *info, pgoff_t index)
768{
769 struct mempolicy mpol, *spol;
770 struct vm_area_struct pvma;
771
772 spol = mpol_cond_copy(&mpol,
773 mpol_shared_policy_lookup(&info->policy, index));
774
775 /* Create a pseudo vma that just contains the policy */
776 pvma.vm_start = 0;
777 pvma.vm_pgoff = index;
778 pvma.vm_ops = NULL;
779 pvma.vm_policy = spol;
780 return swapin_readahead(swap, gfp, &pvma, 0);
781}
782
783static struct page *shmem_alloc_page(gfp_t gfp,
784 struct shmem_inode_info *info, pgoff_t index)
785{
786 struct vm_area_struct pvma;
787
788 /* Create a pseudo vma that just contains the policy */
789 pvma.vm_start = 0;
790 pvma.vm_pgoff = index;
791 pvma.vm_ops = NULL;
792 pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index);
793
794 /*
795 * alloc_page_vma() will drop the shared policy reference
796 */
797 return alloc_page_vma(gfp, &pvma, 0);
798}
799#else /* !CONFIG_NUMA */
800#ifdef CONFIG_TMPFS
801static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
802{
803}
804#endif /* CONFIG_TMPFS */
805
806static inline struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
807 struct shmem_inode_info *info, pgoff_t index)
808{
809 return swapin_readahead(swap, gfp, NULL, 0);
810}
811
812static inline struct page *shmem_alloc_page(gfp_t gfp,
813 struct shmem_inode_info *info, pgoff_t index)
814{
815 return alloc_page(gfp);
816}
817#endif /* CONFIG_NUMA */
818
819#if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
820static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
821{
822 return NULL;
823}
824#endif
825
826/*
827 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
828 *
829 * If we allocate a new one we do not mark it dirty. That's up to the
830 * vm. If we swap it in we mark it dirty since we also free the swap
831 * entry since a page cannot live in both the swap and page cache
832 */
833static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
834 struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type)
835{
836 struct address_space *mapping = inode->i_mapping;
837 struct shmem_inode_info *info;
838 struct shmem_sb_info *sbinfo;
839 struct page *page;
840 swp_entry_t swap;
841 int error;
842 int once = 0;
843
844 if (index > (MAX_LFS_FILESIZE >> PAGE_CACHE_SHIFT))
845 return -EFBIG;
846repeat:
847 swap.val = 0;
848 page = find_lock_page(mapping, index);
849 if (radix_tree_exceptional_entry(page)) {
850 swap = radix_to_swp_entry(page);
851 page = NULL;
852 }
853
854 if (sgp != SGP_WRITE &&
855 ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
856 error = -EINVAL;
857 goto failed;
858 }
859
860 if (page || (sgp == SGP_READ && !swap.val)) {
861 /*
862 * Once we can get the page lock, it must be uptodate:
863 * if there were an error in reading back from swap,
864 * the page would not be inserted into the filecache.
865 */
866 BUG_ON(page && !PageUptodate(page));
867 *pagep = page;
868 return 0;
869 }
870
871 /*
872 * Fast cache lookup did not find it:
873 * bring it back from swap or allocate.
874 */
875 info = SHMEM_I(inode);
876 sbinfo = SHMEM_SB(inode->i_sb);
877
878 if (swap.val) {
879 /* Look it up and read it in.. */
880 page = lookup_swap_cache(swap);
881 if (!page) {
882 /* here we actually do the io */
883 if (fault_type)
884 *fault_type |= VM_FAULT_MAJOR;
885 page = shmem_swapin(swap, gfp, info, index);
886 if (!page) {
887 error = -ENOMEM;
888 goto failed;
889 }
890 }
891
892 /* We have to do this with page locked to prevent races */
893 lock_page(page);
894 if (!PageUptodate(page)) {
895 error = -EIO;
896 goto failed;
897 }
898 wait_on_page_writeback(page);
899
900 /* Someone may have already done it for us */
901 if (page->mapping) {
902 if (page->mapping == mapping &&
903 page->index == index)
904 goto done;
905 error = -EEXIST;
906 goto failed;
907 }
908
909 error = mem_cgroup_cache_charge(page, current->mm,
910 gfp & GFP_RECLAIM_MASK);
911 if (!error)
912 error = shmem_add_to_page_cache(page, mapping, index,
913 gfp, swp_to_radix_entry(swap));
914 if (error)
915 goto failed;
916
917 spin_lock(&info->lock);
918 info->swapped--;
919 shmem_recalc_inode(inode);
920 spin_unlock(&info->lock);
921
922 delete_from_swap_cache(page);
923 set_page_dirty(page);
924 swap_free(swap);
925
926 } else {
927 if (shmem_acct_block(info->flags)) {
928 error = -ENOSPC;
929 goto failed;
930 }
931 if (sbinfo->max_blocks) {
932 if (percpu_counter_compare(&sbinfo->used_blocks,
933 sbinfo->max_blocks) >= 0) {
934 error = -ENOSPC;
935 goto unacct;
936 }
937 percpu_counter_inc(&sbinfo->used_blocks);
938 }
939
940 page = shmem_alloc_page(gfp, info, index);
941 if (!page) {
942 error = -ENOMEM;
943 goto decused;
944 }
945
946 SetPageSwapBacked(page);
947 __set_page_locked(page);
948 error = mem_cgroup_cache_charge(page, current->mm,
949 gfp & GFP_RECLAIM_MASK);
950 if (!error)
951 error = shmem_add_to_page_cache(page, mapping, index,
952 gfp, NULL);
953 if (error)
954 goto decused;
955 lru_cache_add_anon(page);
956
957 spin_lock(&info->lock);
958 info->alloced++;
959 inode->i_blocks += BLOCKS_PER_PAGE;
960 shmem_recalc_inode(inode);
961 spin_unlock(&info->lock);
962
963 clear_highpage(page);
964 flush_dcache_page(page);
965 SetPageUptodate(page);
966 if (sgp == SGP_DIRTY)
967 set_page_dirty(page);
968 }
969done:
970 /* Perhaps the file has been truncated since we checked */
971 if (sgp != SGP_WRITE &&
972 ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
973 error = -EINVAL;
974 goto trunc;
975 }
976 *pagep = page;
977 return 0;
978
979 /*
980 * Error recovery.
981 */
982trunc:
983 ClearPageDirty(page);
984 delete_from_page_cache(page);
985 spin_lock(&info->lock);
986 info->alloced--;
987 inode->i_blocks -= BLOCKS_PER_PAGE;
988 spin_unlock(&info->lock);
989decused:
990 if (sbinfo->max_blocks)
991 percpu_counter_add(&sbinfo->used_blocks, -1);
992unacct:
993 shmem_unacct_blocks(info->flags, 1);
994failed:
995 if (swap.val && error != -EINVAL) {
996 struct page *test = find_get_page(mapping, index);
997 if (test && !radix_tree_exceptional_entry(test))
998 page_cache_release(test);
999 /* Have another try if the entry has changed */
1000 if (test != swp_to_radix_entry(swap))
1001 error = -EEXIST;
1002 }
1003 if (page) {
1004 unlock_page(page);
1005 page_cache_release(page);
1006 }
1007 if (error == -ENOSPC && !once++) {
1008 info = SHMEM_I(inode);
1009 spin_lock(&info->lock);
1010 shmem_recalc_inode(inode);
1011 spin_unlock(&info->lock);
1012 goto repeat;
1013 }
1014 if (error == -EEXIST)
1015 goto repeat;
1016 return error;
1017}
1018
1019static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1020{
1021 struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1022 int error;
1023 int ret = VM_FAULT_LOCKED;
1024
1025 error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
1026 if (error)
1027 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1028
1029 if (ret & VM_FAULT_MAJOR) {
1030 count_vm_event(PGMAJFAULT);
1031 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
1032 }
1033 return ret;
1034}
1035
1036#ifdef CONFIG_NUMA
1037static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
1038{
1039 struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1040 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
1041}
1042
1043static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
1044 unsigned long addr)
1045{
1046 struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1047 pgoff_t index;
1048
1049 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1050 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
1051}
1052#endif
1053
1054int shmem_lock(struct file *file, int lock, struct user_struct *user)
1055{
1056 struct inode *inode = file->f_path.dentry->d_inode;
1057 struct shmem_inode_info *info = SHMEM_I(inode);
1058 int retval = -ENOMEM;
1059
1060 spin_lock(&info->lock);
1061 if (lock && !(info->flags & VM_LOCKED)) {
1062 if (!user_shm_lock(inode->i_size, user))
1063 goto out_nomem;
1064 info->flags |= VM_LOCKED;
1065 mapping_set_unevictable(file->f_mapping);
1066 }
1067 if (!lock && (info->flags & VM_LOCKED) && user) {
1068 user_shm_unlock(inode->i_size, user);
1069 info->flags &= ~VM_LOCKED;
1070 mapping_clear_unevictable(file->f_mapping);
1071 scan_mapping_unevictable_pages(file->f_mapping);
1072 }
1073 retval = 0;
1074
1075out_nomem:
1076 spin_unlock(&info->lock);
1077 return retval;
1078}
1079
1080static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1081{
1082 file_accessed(file);
1083 vma->vm_ops = &shmem_vm_ops;
1084 vma->vm_flags |= VM_CAN_NONLINEAR;
1085 return 0;
1086}
1087
1088static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
1089 int mode, dev_t dev, unsigned long flags)
1090{
1091 struct inode *inode;
1092 struct shmem_inode_info *info;
1093 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1094
1095 if (shmem_reserve_inode(sb))
1096 return NULL;
1097
1098 inode = new_inode(sb);
1099 if (inode) {
1100 inode->i_ino = get_next_ino();
1101 inode_init_owner(inode, dir, mode);
1102 inode->i_blocks = 0;
1103 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1104 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1105 inode->i_generation = get_seconds();
1106 info = SHMEM_I(inode);
1107 memset(info, 0, (char *)inode - (char *)info);
1108 spin_lock_init(&info->lock);
1109 info->flags = flags & VM_NORESERVE;
1110 INIT_LIST_HEAD(&info->swaplist);
1111 INIT_LIST_HEAD(&info->xattr_list);
1112 cache_no_acl(inode);
1113
1114 switch (mode & S_IFMT) {
1115 default:
1116 inode->i_op = &shmem_special_inode_operations;
1117 init_special_inode(inode, mode, dev);
1118 break;
1119 case S_IFREG:
1120 inode->i_mapping->a_ops = &shmem_aops;
1121 inode->i_op = &shmem_inode_operations;
1122 inode->i_fop = &shmem_file_operations;
1123 mpol_shared_policy_init(&info->policy,
1124 shmem_get_sbmpol(sbinfo));
1125 break;
1126 case S_IFDIR:
1127 inc_nlink(inode);
1128 /* Some things misbehave if size == 0 on a directory */
1129 inode->i_size = 2 * BOGO_DIRENT_SIZE;
1130 inode->i_op = &shmem_dir_inode_operations;
1131 inode->i_fop = &simple_dir_operations;
1132 break;
1133 case S_IFLNK:
1134 /*
1135 * Must not load anything in the rbtree,
1136 * mpol_free_shared_policy will not be called.
1137 */
1138 mpol_shared_policy_init(&info->policy, NULL);
1139 break;
1140 }
1141 } else
1142 shmem_free_inode(sb);
1143 return inode;
1144}
1145
1146#ifdef CONFIG_TMPFS
1147static const struct inode_operations shmem_symlink_inode_operations;
1148static const struct inode_operations shmem_short_symlink_operations;
1149
1150static int
1151shmem_write_begin(struct file *file, struct address_space *mapping,
1152 loff_t pos, unsigned len, unsigned flags,
1153 struct page **pagep, void **fsdata)
1154{
1155 struct inode *inode = mapping->host;
1156 pgoff_t index = pos >> PAGE_CACHE_SHIFT;
1157 return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
1158}
1159
1160static int
1161shmem_write_end(struct file *file, struct address_space *mapping,
1162 loff_t pos, unsigned len, unsigned copied,
1163 struct page *page, void *fsdata)
1164{
1165 struct inode *inode = mapping->host;
1166
1167 if (pos + copied > inode->i_size)
1168 i_size_write(inode, pos + copied);
1169
1170 set_page_dirty(page);
1171 unlock_page(page);
1172 page_cache_release(page);
1173
1174 return copied;
1175}
1176
1177static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
1178{
1179 struct inode *inode = filp->f_path.dentry->d_inode;
1180 struct address_space *mapping = inode->i_mapping;
1181 pgoff_t index;
1182 unsigned long offset;
1183 enum sgp_type sgp = SGP_READ;
1184
1185 /*
1186 * Might this read be for a stacking filesystem? Then when reading
1187 * holes of a sparse file, we actually need to allocate those pages,
1188 * and even mark them dirty, so it cannot exceed the max_blocks limit.
1189 */
1190 if (segment_eq(get_fs(), KERNEL_DS))
1191 sgp = SGP_DIRTY;
1192
1193 index = *ppos >> PAGE_CACHE_SHIFT;
1194 offset = *ppos & ~PAGE_CACHE_MASK;
1195
1196 for (;;) {
1197 struct page *page = NULL;
1198 pgoff_t end_index;
1199 unsigned long nr, ret;
1200 loff_t i_size = i_size_read(inode);
1201
1202 end_index = i_size >> PAGE_CACHE_SHIFT;
1203 if (index > end_index)
1204 break;
1205 if (index == end_index) {
1206 nr = i_size & ~PAGE_CACHE_MASK;
1207 if (nr <= offset)
1208 break;
1209 }
1210
1211 desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
1212 if (desc->error) {
1213 if (desc->error == -EINVAL)
1214 desc->error = 0;
1215 break;
1216 }
1217 if (page)
1218 unlock_page(page);
1219
1220 /*
1221 * We must evaluate after, since reads (unlike writes)
1222 * are called without i_mutex protection against truncate
1223 */
1224 nr = PAGE_CACHE_SIZE;
1225 i_size = i_size_read(inode);
1226 end_index = i_size >> PAGE_CACHE_SHIFT;
1227 if (index == end_index) {
1228 nr = i_size & ~PAGE_CACHE_MASK;
1229 if (nr <= offset) {
1230 if (page)
1231 page_cache_release(page);
1232 break;
1233 }
1234 }
1235 nr -= offset;
1236
1237 if (page) {
1238 /*
1239 * If users can be writing to this page using arbitrary
1240 * virtual addresses, take care about potential aliasing
1241 * before reading the page on the kernel side.
1242 */
1243 if (mapping_writably_mapped(mapping))
1244 flush_dcache_page(page);
1245 /*
1246 * Mark the page accessed if we read the beginning.
1247 */
1248 if (!offset)
1249 mark_page_accessed(page);
1250 } else {
1251 page = ZERO_PAGE(0);
1252 page_cache_get(page);
1253 }
1254
1255 /*
1256 * Ok, we have the page, and it's up-to-date, so
1257 * now we can copy it to user space...
1258 *
1259 * The actor routine returns how many bytes were actually used..
1260 * NOTE! This may not be the same as how much of a user buffer
1261 * we filled up (we may be padding etc), so we can only update
1262 * "pos" here (the actor routine has to update the user buffer
1263 * pointers and the remaining count).
1264 */
1265 ret = actor(desc, page, offset, nr);
1266 offset += ret;
1267 index += offset >> PAGE_CACHE_SHIFT;
1268 offset &= ~PAGE_CACHE_MASK;
1269
1270 page_cache_release(page);
1271 if (ret != nr || !desc->count)
1272 break;
1273
1274 cond_resched();
1275 }
1276
1277 *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
1278 file_accessed(filp);
1279}
1280
1281static ssize_t shmem_file_aio_read(struct kiocb *iocb,
1282 const struct iovec *iov, unsigned long nr_segs, loff_t pos)
1283{
1284 struct file *filp = iocb->ki_filp;
1285 ssize_t retval;
1286 unsigned long seg;
1287 size_t count;
1288 loff_t *ppos = &iocb->ki_pos;
1289
1290 retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
1291 if (retval)
1292 return retval;
1293
1294 for (seg = 0; seg < nr_segs; seg++) {
1295 read_descriptor_t desc;
1296
1297 desc.written = 0;
1298 desc.arg.buf = iov[seg].iov_base;
1299 desc.count = iov[seg].iov_len;
1300 if (desc.count == 0)
1301 continue;
1302 desc.error = 0;
1303 do_shmem_file_read(filp, ppos, &desc, file_read_actor);
1304 retval += desc.written;
1305 if (desc.error) {
1306 retval = retval ?: desc.error;
1307 break;
1308 }
1309 if (desc.count > 0)
1310 break;
1311 }
1312 return retval;
1313}
1314
1315static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
1316 struct pipe_inode_info *pipe, size_t len,
1317 unsigned int flags)
1318{
1319 struct address_space *mapping = in->f_mapping;
1320 struct inode *inode = mapping->host;
1321 unsigned int loff, nr_pages, req_pages;
1322 struct page *pages[PIPE_DEF_BUFFERS];
1323 struct partial_page partial[PIPE_DEF_BUFFERS];
1324 struct page *page;
1325 pgoff_t index, end_index;
1326 loff_t isize, left;
1327 int error, page_nr;
1328 struct splice_pipe_desc spd = {
1329 .pages = pages,
1330 .partial = partial,
1331 .flags = flags,
1332 .ops = &page_cache_pipe_buf_ops,
1333 .spd_release = spd_release_page,
1334 };
1335
1336 isize = i_size_read(inode);
1337 if (unlikely(*ppos >= isize))
1338 return 0;
1339
1340 left = isize - *ppos;
1341 if (unlikely(left < len))
1342 len = left;
1343
1344 if (splice_grow_spd(pipe, &spd))
1345 return -ENOMEM;
1346
1347 index = *ppos >> PAGE_CACHE_SHIFT;
1348 loff = *ppos & ~PAGE_CACHE_MASK;
1349 req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1350 nr_pages = min(req_pages, pipe->buffers);
1351
1352 spd.nr_pages = find_get_pages_contig(mapping, index,
1353 nr_pages, spd.pages);
1354 index += spd.nr_pages;
1355 error = 0;
1356
1357 while (spd.nr_pages < nr_pages) {
1358 error = shmem_getpage(inode, index, &page, SGP_CACHE, NULL);
1359 if (error)
1360 break;
1361 unlock_page(page);
1362 spd.pages[spd.nr_pages++] = page;
1363 index++;
1364 }
1365
1366 index = *ppos >> PAGE_CACHE_SHIFT;
1367 nr_pages = spd.nr_pages;
1368 spd.nr_pages = 0;
1369
1370 for (page_nr = 0; page_nr < nr_pages; page_nr++) {
1371 unsigned int this_len;
1372
1373 if (!len)
1374 break;
1375
1376 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
1377 page = spd.pages[page_nr];
1378
1379 if (!PageUptodate(page) || page->mapping != mapping) {
1380 error = shmem_getpage(inode, index, &page,
1381 SGP_CACHE, NULL);
1382 if (error)
1383 break;
1384 unlock_page(page);
1385 page_cache_release(spd.pages[page_nr]);
1386 spd.pages[page_nr] = page;
1387 }
1388
1389 isize = i_size_read(inode);
1390 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
1391 if (unlikely(!isize || index > end_index))
1392 break;
1393
1394 if (end_index == index) {
1395 unsigned int plen;
1396
1397 plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
1398 if (plen <= loff)
1399 break;
1400
1401 this_len = min(this_len, plen - loff);
1402 len = this_len;
1403 }
1404
1405 spd.partial[page_nr].offset = loff;
1406 spd.partial[page_nr].len = this_len;
1407 len -= this_len;
1408 loff = 0;
1409 spd.nr_pages++;
1410 index++;
1411 }
1412
1413 while (page_nr < nr_pages)
1414 page_cache_release(spd.pages[page_nr++]);
1415
1416 if (spd.nr_pages)
1417 error = splice_to_pipe(pipe, &spd);
1418
1419 splice_shrink_spd(pipe, &spd);
1420
1421 if (error > 0) {
1422 *ppos += error;
1423 file_accessed(in);
1424 }
1425 return error;
1426}
1427
1428static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
1429{
1430 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
1431
1432 buf->f_type = TMPFS_MAGIC;
1433 buf->f_bsize = PAGE_CACHE_SIZE;
1434 buf->f_namelen = NAME_MAX;
1435 if (sbinfo->max_blocks) {
1436 buf->f_blocks = sbinfo->max_blocks;
1437 buf->f_bavail =
1438 buf->f_bfree = sbinfo->max_blocks -
1439 percpu_counter_sum(&sbinfo->used_blocks);
1440 }
1441 if (sbinfo->max_inodes) {
1442 buf->f_files = sbinfo->max_inodes;
1443 buf->f_ffree = sbinfo->free_inodes;
1444 }
1445 /* else leave those fields 0 like simple_statfs */
1446 return 0;
1447}
1448
1449/*
1450 * File creation. Allocate an inode, and we're done..
1451 */
1452static int
1453shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
1454{
1455 struct inode *inode;
1456 int error = -ENOSPC;
1457
1458 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
1459 if (inode) {
1460 error = security_inode_init_security(inode, dir,
1461 &dentry->d_name, NULL,
1462 NULL, NULL);
1463 if (error) {
1464 if (error != -EOPNOTSUPP) {
1465 iput(inode);
1466 return error;
1467 }
1468 }
1469#ifdef CONFIG_TMPFS_POSIX_ACL
1470 error = generic_acl_init(inode, dir);
1471 if (error) {
1472 iput(inode);
1473 return error;
1474 }
1475#else
1476 error = 0;
1477#endif
1478 dir->i_size += BOGO_DIRENT_SIZE;
1479 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1480 d_instantiate(dentry, inode);
1481 dget(dentry); /* Extra count - pin the dentry in core */
1482 }
1483 return error;
1484}
1485
1486static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
1487{
1488 int error;
1489
1490 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
1491 return error;
1492 inc_nlink(dir);
1493 return 0;
1494}
1495
1496static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
1497 struct nameidata *nd)
1498{
1499 return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
1500}
1501
1502/*
1503 * Link a file..
1504 */
1505static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
1506{
1507 struct inode *inode = old_dentry->d_inode;
1508 int ret;
1509
1510 /*
1511 * No ordinary (disk based) filesystem counts links as inodes;
1512 * but each new link needs a new dentry, pinning lowmem, and
1513 * tmpfs dentries cannot be pruned until they are unlinked.
1514 */
1515 ret = shmem_reserve_inode(inode->i_sb);
1516 if (ret)
1517 goto out;
1518
1519 dir->i_size += BOGO_DIRENT_SIZE;
1520 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1521 inc_nlink(inode);
1522 ihold(inode); /* New dentry reference */
1523 dget(dentry); /* Extra pinning count for the created dentry */
1524 d_instantiate(dentry, inode);
1525out:
1526 return ret;
1527}
1528
1529static int shmem_unlink(struct inode *dir, struct dentry *dentry)
1530{
1531 struct inode *inode = dentry->d_inode;
1532
1533 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
1534 shmem_free_inode(inode->i_sb);
1535
1536 dir->i_size -= BOGO_DIRENT_SIZE;
1537 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1538 drop_nlink(inode);
1539 dput(dentry); /* Undo the count from "create" - this does all the work */
1540 return 0;
1541}
1542
1543static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
1544{
1545 if (!simple_empty(dentry))
1546 return -ENOTEMPTY;
1547
1548 drop_nlink(dentry->d_inode);
1549 drop_nlink(dir);
1550 return shmem_unlink(dir, dentry);
1551}
1552
1553/*
1554 * The VFS layer already does all the dentry stuff for rename,
1555 * we just have to decrement the usage count for the target if
1556 * it exists so that the VFS layer correctly free's it when it
1557 * gets overwritten.
1558 */
1559static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
1560{
1561 struct inode *inode = old_dentry->d_inode;
1562 int they_are_dirs = S_ISDIR(inode->i_mode);
1563
1564 if (!simple_empty(new_dentry))
1565 return -ENOTEMPTY;
1566
1567 if (new_dentry->d_inode) {
1568 (void) shmem_unlink(new_dir, new_dentry);
1569 if (they_are_dirs)
1570 drop_nlink(old_dir);
1571 } else if (they_are_dirs) {
1572 drop_nlink(old_dir);
1573 inc_nlink(new_dir);
1574 }
1575
1576 old_dir->i_size -= BOGO_DIRENT_SIZE;
1577 new_dir->i_size += BOGO_DIRENT_SIZE;
1578 old_dir->i_ctime = old_dir->i_mtime =
1579 new_dir->i_ctime = new_dir->i_mtime =
1580 inode->i_ctime = CURRENT_TIME;
1581 return 0;
1582}
1583
1584static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
1585{
1586 int error;
1587 int len;
1588 struct inode *inode;
1589 struct page *page;
1590 char *kaddr;
1591 struct shmem_inode_info *info;
1592
1593 len = strlen(symname) + 1;
1594 if (len > PAGE_CACHE_SIZE)
1595 return -ENAMETOOLONG;
1596
1597 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
1598 if (!inode)
1599 return -ENOSPC;
1600
1601 error = security_inode_init_security(inode, dir, &dentry->d_name, NULL,
1602 NULL, NULL);
1603 if (error) {
1604 if (error != -EOPNOTSUPP) {
1605 iput(inode);
1606 return error;
1607 }
1608 error = 0;
1609 }
1610
1611 info = SHMEM_I(inode);
1612 inode->i_size = len-1;
1613 if (len <= SHORT_SYMLINK_LEN) {
1614 info->symlink = kmemdup(symname, len, GFP_KERNEL);
1615 if (!info->symlink) {
1616 iput(inode);
1617 return -ENOMEM;
1618 }
1619 inode->i_op = &shmem_short_symlink_operations;
1620 } else {
1621 error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
1622 if (error) {
1623 iput(inode);
1624 return error;
1625 }
1626 inode->i_mapping->a_ops = &shmem_aops;
1627 inode->i_op = &shmem_symlink_inode_operations;
1628 kaddr = kmap_atomic(page, KM_USER0);
1629 memcpy(kaddr, symname, len);
1630 kunmap_atomic(kaddr, KM_USER0);
1631 set_page_dirty(page);
1632 unlock_page(page);
1633 page_cache_release(page);
1634 }
1635 dir->i_size += BOGO_DIRENT_SIZE;
1636 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1637 d_instantiate(dentry, inode);
1638 dget(dentry);
1639 return 0;
1640}
1641
1642static void *shmem_follow_short_symlink(struct dentry *dentry, struct nameidata *nd)
1643{
1644 nd_set_link(nd, SHMEM_I(dentry->d_inode)->symlink);
1645 return NULL;
1646}
1647
1648static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
1649{
1650 struct page *page = NULL;
1651 int error = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
1652 nd_set_link(nd, error ? ERR_PTR(error) : kmap(page));
1653 if (page)
1654 unlock_page(page);
1655 return page;
1656}
1657
1658static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
1659{
1660 if (!IS_ERR(nd_get_link(nd))) {
1661 struct page *page = cookie;
1662 kunmap(page);
1663 mark_page_accessed(page);
1664 page_cache_release(page);
1665 }
1666}
1667
1668#ifdef CONFIG_TMPFS_XATTR
1669/*
1670 * Superblocks without xattr inode operations may get some security.* xattr
1671 * support from the LSM "for free". As soon as we have any other xattrs
1672 * like ACLs, we also need to implement the security.* handlers at
1673 * filesystem level, though.
1674 */
1675
1676static int shmem_xattr_get(struct dentry *dentry, const char *name,
1677 void *buffer, size_t size)
1678{
1679 struct shmem_inode_info *info;
1680 struct shmem_xattr *xattr;
1681 int ret = -ENODATA;
1682
1683 info = SHMEM_I(dentry->d_inode);
1684
1685 spin_lock(&info->lock);
1686 list_for_each_entry(xattr, &info->xattr_list, list) {
1687 if (strcmp(name, xattr->name))
1688 continue;
1689
1690 ret = xattr->size;
1691 if (buffer) {
1692 if (size < xattr->size)
1693 ret = -ERANGE;
1694 else
1695 memcpy(buffer, xattr->value, xattr->size);
1696 }
1697 break;
1698 }
1699 spin_unlock(&info->lock);
1700 return ret;
1701}
1702
1703static int shmem_xattr_set(struct dentry *dentry, const char *name,
1704 const void *value, size_t size, int flags)
1705{
1706 struct inode *inode = dentry->d_inode;
1707 struct shmem_inode_info *info = SHMEM_I(inode);
1708 struct shmem_xattr *xattr;
1709 struct shmem_xattr *new_xattr = NULL;
1710 size_t len;
1711 int err = 0;
1712
1713 /* value == NULL means remove */
1714 if (value) {
1715 /* wrap around? */
1716 len = sizeof(*new_xattr) + size;
1717 if (len <= sizeof(*new_xattr))
1718 return -ENOMEM;
1719
1720 new_xattr = kmalloc(len, GFP_KERNEL);
1721 if (!new_xattr)
1722 return -ENOMEM;
1723
1724 new_xattr->name = kstrdup(name, GFP_KERNEL);
1725 if (!new_xattr->name) {
1726 kfree(new_xattr);
1727 return -ENOMEM;
1728 }
1729
1730 new_xattr->size = size;
1731 memcpy(new_xattr->value, value, size);
1732 }
1733
1734 spin_lock(&info->lock);
1735 list_for_each_entry(xattr, &info->xattr_list, list) {
1736 if (!strcmp(name, xattr->name)) {
1737 if (flags & XATTR_CREATE) {
1738 xattr = new_xattr;
1739 err = -EEXIST;
1740 } else if (new_xattr) {
1741 list_replace(&xattr->list, &new_xattr->list);
1742 } else {
1743 list_del(&xattr->list);
1744 }
1745 goto out;
1746 }
1747 }
1748 if (flags & XATTR_REPLACE) {
1749 xattr = new_xattr;
1750 err = -ENODATA;
1751 } else {
1752 list_add(&new_xattr->list, &info->xattr_list);
1753 xattr = NULL;
1754 }
1755out:
1756 spin_unlock(&info->lock);
1757 if (xattr)
1758 kfree(xattr->name);
1759 kfree(xattr);
1760 return err;
1761}
1762
1763static const struct xattr_handler *shmem_xattr_handlers[] = {
1764#ifdef CONFIG_TMPFS_POSIX_ACL
1765 &generic_acl_access_handler,
1766 &generic_acl_default_handler,
1767#endif
1768 NULL
1769};
1770
1771static int shmem_xattr_validate(const char *name)
1772{
1773 struct { const char *prefix; size_t len; } arr[] = {
1774 { XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN },
1775 { XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN }
1776 };
1777 int i;
1778
1779 for (i = 0; i < ARRAY_SIZE(arr); i++) {
1780 size_t preflen = arr[i].len;
1781 if (strncmp(name, arr[i].prefix, preflen) == 0) {
1782 if (!name[preflen])
1783 return -EINVAL;
1784 return 0;
1785 }
1786 }
1787 return -EOPNOTSUPP;
1788}
1789
1790static ssize_t shmem_getxattr(struct dentry *dentry, const char *name,
1791 void *buffer, size_t size)
1792{
1793 int err;
1794
1795 /*
1796 * If this is a request for a synthetic attribute in the system.*
1797 * namespace use the generic infrastructure to resolve a handler
1798 * for it via sb->s_xattr.
1799 */
1800 if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
1801 return generic_getxattr(dentry, name, buffer, size);
1802
1803 err = shmem_xattr_validate(name);
1804 if (err)
1805 return err;
1806
1807 return shmem_xattr_get(dentry, name, buffer, size);
1808}
1809
1810static int shmem_setxattr(struct dentry *dentry, const char *name,
1811 const void *value, size_t size, int flags)
1812{
1813 int err;
1814
1815 /*
1816 * If this is a request for a synthetic attribute in the system.*
1817 * namespace use the generic infrastructure to resolve a handler
1818 * for it via sb->s_xattr.
1819 */
1820 if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
1821 return generic_setxattr(dentry, name, value, size, flags);
1822
1823 err = shmem_xattr_validate(name);
1824 if (err)
1825 return err;
1826
1827 if (size == 0)
1828 value = ""; /* empty EA, do not remove */
1829
1830 return shmem_xattr_set(dentry, name, value, size, flags);
1831
1832}
1833
1834static int shmem_removexattr(struct dentry *dentry, const char *name)
1835{
1836 int err;
1837
1838 /*
1839 * If this is a request for a synthetic attribute in the system.*
1840 * namespace use the generic infrastructure to resolve a handler
1841 * for it via sb->s_xattr.
1842 */
1843 if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
1844 return generic_removexattr(dentry, name);
1845
1846 err = shmem_xattr_validate(name);
1847 if (err)
1848 return err;
1849
1850 return shmem_xattr_set(dentry, name, NULL, 0, XATTR_REPLACE);
1851}
1852
1853static bool xattr_is_trusted(const char *name)
1854{
1855 return !strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN);
1856}
1857
1858static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
1859{
1860 bool trusted = capable(CAP_SYS_ADMIN);
1861 struct shmem_xattr *xattr;
1862 struct shmem_inode_info *info;
1863 size_t used = 0;
1864
1865 info = SHMEM_I(dentry->d_inode);
1866
1867 spin_lock(&info->lock);
1868 list_for_each_entry(xattr, &info->xattr_list, list) {
1869 size_t len;
1870
1871 /* skip "trusted." attributes for unprivileged callers */
1872 if (!trusted && xattr_is_trusted(xattr->name))
1873 continue;
1874
1875 len = strlen(xattr->name) + 1;
1876 used += len;
1877 if (buffer) {
1878 if (size < used) {
1879 used = -ERANGE;
1880 break;
1881 }
1882 memcpy(buffer, xattr->name, len);
1883 buffer += len;
1884 }
1885 }
1886 spin_unlock(&info->lock);
1887
1888 return used;
1889}
1890#endif /* CONFIG_TMPFS_XATTR */
1891
1892static const struct inode_operations shmem_short_symlink_operations = {
1893 .readlink = generic_readlink,
1894 .follow_link = shmem_follow_short_symlink,
1895#ifdef CONFIG_TMPFS_XATTR
1896 .setxattr = shmem_setxattr,
1897 .getxattr = shmem_getxattr,
1898 .listxattr = shmem_listxattr,
1899 .removexattr = shmem_removexattr,
1900#endif
1901};
1902
1903static const struct inode_operations shmem_symlink_inode_operations = {
1904 .readlink = generic_readlink,
1905 .follow_link = shmem_follow_link,
1906 .put_link = shmem_put_link,
1907#ifdef CONFIG_TMPFS_XATTR
1908 .setxattr = shmem_setxattr,
1909 .getxattr = shmem_getxattr,
1910 .listxattr = shmem_listxattr,
1911 .removexattr = shmem_removexattr,
1912#endif
1913};
1914
1915static struct dentry *shmem_get_parent(struct dentry *child)
1916{
1917 return ERR_PTR(-ESTALE);
1918}
1919
1920static int shmem_match(struct inode *ino, void *vfh)
1921{
1922 __u32 *fh = vfh;
1923 __u64 inum = fh[2];
1924 inum = (inum << 32) | fh[1];
1925 return ino->i_ino == inum && fh[0] == ino->i_generation;
1926}
1927
1928static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
1929 struct fid *fid, int fh_len, int fh_type)
1930{
1931 struct inode *inode;
1932 struct dentry *dentry = NULL;
1933 u64 inum = fid->raw[2];
1934 inum = (inum << 32) | fid->raw[1];
1935
1936 if (fh_len < 3)
1937 return NULL;
1938
1939 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
1940 shmem_match, fid->raw);
1941 if (inode) {
1942 dentry = d_find_alias(inode);
1943 iput(inode);
1944 }
1945
1946 return dentry;
1947}
1948
1949static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len,
1950 int connectable)
1951{
1952 struct inode *inode = dentry->d_inode;
1953
1954 if (*len < 3) {
1955 *len = 3;
1956 return 255;
1957 }
1958
1959 if (inode_unhashed(inode)) {
1960 /* Unfortunately insert_inode_hash is not idempotent,
1961 * so as we hash inodes here rather than at creation
1962 * time, we need a lock to ensure we only try
1963 * to do it once
1964 */
1965 static DEFINE_SPINLOCK(lock);
1966 spin_lock(&lock);
1967 if (inode_unhashed(inode))
1968 __insert_inode_hash(inode,
1969 inode->i_ino + inode->i_generation);
1970 spin_unlock(&lock);
1971 }
1972
1973 fh[0] = inode->i_generation;
1974 fh[1] = inode->i_ino;
1975 fh[2] = ((__u64)inode->i_ino) >> 32;
1976
1977 *len = 3;
1978 return 1;
1979}
1980
1981static const struct export_operations shmem_export_ops = {
1982 .get_parent = shmem_get_parent,
1983 .encode_fh = shmem_encode_fh,
1984 .fh_to_dentry = shmem_fh_to_dentry,
1985};
1986
1987static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
1988 bool remount)
1989{
1990 char *this_char, *value, *rest;
1991
1992 while (options != NULL) {
1993 this_char = options;
1994 for (;;) {
1995 /*
1996 * NUL-terminate this option: unfortunately,
1997 * mount options form a comma-separated list,
1998 * but mpol's nodelist may also contain commas.
1999 */
2000 options = strchr(options, ',');
2001 if (options == NULL)
2002 break;
2003 options++;
2004 if (!isdigit(*options)) {
2005 options[-1] = '\0';
2006 break;
2007 }
2008 }
2009 if (!*this_char)
2010 continue;
2011 if ((value = strchr(this_char,'=')) != NULL) {
2012 *value++ = 0;
2013 } else {
2014 printk(KERN_ERR
2015 "tmpfs: No value for mount option '%s'\n",
2016 this_char);
2017 return 1;
2018 }
2019
2020 if (!strcmp(this_char,"size")) {
2021 unsigned long long size;
2022 size = memparse(value,&rest);
2023 if (*rest == '%') {
2024 size <<= PAGE_SHIFT;
2025 size *= totalram_pages;
2026 do_div(size, 100);
2027 rest++;
2028 }
2029 if (*rest)
2030 goto bad_val;
2031 sbinfo->max_blocks =
2032 DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
2033 } else if (!strcmp(this_char,"nr_blocks")) {
2034 sbinfo->max_blocks = memparse(value, &rest);
2035 if (*rest)
2036 goto bad_val;
2037 } else if (!strcmp(this_char,"nr_inodes")) {
2038 sbinfo->max_inodes = memparse(value, &rest);
2039 if (*rest)
2040 goto bad_val;
2041 } else if (!strcmp(this_char,"mode")) {
2042 if (remount)
2043 continue;
2044 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
2045 if (*rest)
2046 goto bad_val;
2047 } else if (!strcmp(this_char,"uid")) {
2048 if (remount)
2049 continue;
2050 sbinfo->uid = simple_strtoul(value, &rest, 0);
2051 if (*rest)
2052 goto bad_val;
2053 } else if (!strcmp(this_char,"gid")) {
2054 if (remount)
2055 continue;
2056 sbinfo->gid = simple_strtoul(value, &rest, 0);
2057 if (*rest)
2058 goto bad_val;
2059 } else if (!strcmp(this_char,"mpol")) {
2060 if (mpol_parse_str(value, &sbinfo->mpol, 1))
2061 goto bad_val;
2062 } else {
2063 printk(KERN_ERR "tmpfs: Bad mount option %s\n",
2064 this_char);
2065 return 1;
2066 }
2067 }
2068 return 0;
2069
2070bad_val:
2071 printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
2072 value, this_char);
2073 return 1;
2074
2075}
2076
2077static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
2078{
2079 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2080 struct shmem_sb_info config = *sbinfo;
2081 unsigned long inodes;
2082 int error = -EINVAL;
2083
2084 if (shmem_parse_options(data, &config, true))
2085 return error;
2086
2087 spin_lock(&sbinfo->stat_lock);
2088 inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2089 if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
2090 goto out;
2091 if (config.max_inodes < inodes)
2092 goto out;
2093 /*
2094 * Those tests disallow limited->unlimited while any are in use;
2095 * but we must separately disallow unlimited->limited, because
2096 * in that case we have no record of how much is already in use.
2097 */
2098 if (config.max_blocks && !sbinfo->max_blocks)
2099 goto out;
2100 if (config.max_inodes && !sbinfo->max_inodes)
2101 goto out;
2102
2103 error = 0;
2104 sbinfo->max_blocks = config.max_blocks;
2105 sbinfo->max_inodes = config.max_inodes;
2106 sbinfo->free_inodes = config.max_inodes - inodes;
2107
2108 mpol_put(sbinfo->mpol);
2109 sbinfo->mpol = config.mpol; /* transfers initial ref */
2110out:
2111 spin_unlock(&sbinfo->stat_lock);
2112 return error;
2113}
2114
2115static int shmem_show_options(struct seq_file *seq, struct vfsmount *vfs)
2116{
2117 struct shmem_sb_info *sbinfo = SHMEM_SB(vfs->mnt_sb);
2118
2119 if (sbinfo->max_blocks != shmem_default_max_blocks())
2120 seq_printf(seq, ",size=%luk",
2121 sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
2122 if (sbinfo->max_inodes != shmem_default_max_inodes())
2123 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
2124 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
2125 seq_printf(seq, ",mode=%03o", sbinfo->mode);
2126 if (sbinfo->uid != 0)
2127 seq_printf(seq, ",uid=%u", sbinfo->uid);
2128 if (sbinfo->gid != 0)
2129 seq_printf(seq, ",gid=%u", sbinfo->gid);
2130 shmem_show_mpol(seq, sbinfo->mpol);
2131 return 0;
2132}
2133#endif /* CONFIG_TMPFS */
2134
2135static void shmem_put_super(struct super_block *sb)
2136{
2137 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2138
2139 percpu_counter_destroy(&sbinfo->used_blocks);
2140 kfree(sbinfo);
2141 sb->s_fs_info = NULL;
2142}
2143
2144int shmem_fill_super(struct super_block *sb, void *data, int silent)
2145{
2146 struct inode *inode;
2147 struct dentry *root;
2148 struct shmem_sb_info *sbinfo;
2149 int err = -ENOMEM;
2150
2151 /* Round up to L1_CACHE_BYTES to resist false sharing */
2152 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
2153 L1_CACHE_BYTES), GFP_KERNEL);
2154 if (!sbinfo)
2155 return -ENOMEM;
2156
2157 sbinfo->mode = S_IRWXUGO | S_ISVTX;
2158 sbinfo->uid = current_fsuid();
2159 sbinfo->gid = current_fsgid();
2160 sb->s_fs_info = sbinfo;
2161
2162#ifdef CONFIG_TMPFS
2163 /*
2164 * Per default we only allow half of the physical ram per
2165 * tmpfs instance, limiting inodes to one per page of lowmem;
2166 * but the internal instance is left unlimited.
2167 */
2168 if (!(sb->s_flags & MS_NOUSER)) {
2169 sbinfo->max_blocks = shmem_default_max_blocks();
2170 sbinfo->max_inodes = shmem_default_max_inodes();
2171 if (shmem_parse_options(data, sbinfo, false)) {
2172 err = -EINVAL;
2173 goto failed;
2174 }
2175 }
2176 sb->s_export_op = &shmem_export_ops;
2177#else
2178 sb->s_flags |= MS_NOUSER;
2179#endif
2180
2181 spin_lock_init(&sbinfo->stat_lock);
2182 if (percpu_counter_init(&sbinfo->used_blocks, 0))
2183 goto failed;
2184 sbinfo->free_inodes = sbinfo->max_inodes;
2185
2186 sb->s_maxbytes = MAX_LFS_FILESIZE;
2187 sb->s_blocksize = PAGE_CACHE_SIZE;
2188 sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
2189 sb->s_magic = TMPFS_MAGIC;
2190 sb->s_op = &shmem_ops;
2191 sb->s_time_gran = 1;
2192#ifdef CONFIG_TMPFS_XATTR
2193 sb->s_xattr = shmem_xattr_handlers;
2194#endif
2195#ifdef CONFIG_TMPFS_POSIX_ACL
2196 sb->s_flags |= MS_POSIXACL;
2197#endif
2198
2199 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
2200 if (!inode)
2201 goto failed;
2202 inode->i_uid = sbinfo->uid;
2203 inode->i_gid = sbinfo->gid;
2204 root = d_alloc_root(inode);
2205 if (!root)
2206 goto failed_iput;
2207 sb->s_root = root;
2208 return 0;
2209
2210failed_iput:
2211 iput(inode);
2212failed:
2213 shmem_put_super(sb);
2214 return err;
2215}
2216
2217static struct kmem_cache *shmem_inode_cachep;
2218
2219static struct inode *shmem_alloc_inode(struct super_block *sb)
2220{
2221 struct shmem_inode_info *info;
2222 info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
2223 if (!info)
2224 return NULL;
2225 return &info->vfs_inode;
2226}
2227
2228static void shmem_destroy_callback(struct rcu_head *head)
2229{
2230 struct inode *inode = container_of(head, struct inode, i_rcu);
2231 INIT_LIST_HEAD(&inode->i_dentry);
2232 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
2233}
2234
2235static void shmem_destroy_inode(struct inode *inode)
2236{
2237 if ((inode->i_mode & S_IFMT) == S_IFREG)
2238 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
2239 call_rcu(&inode->i_rcu, shmem_destroy_callback);
2240}
2241
2242static void shmem_init_inode(void *foo)
2243{
2244 struct shmem_inode_info *info = foo;
2245 inode_init_once(&info->vfs_inode);
2246}
2247
2248static int shmem_init_inodecache(void)
2249{
2250 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
2251 sizeof(struct shmem_inode_info),
2252 0, SLAB_PANIC, shmem_init_inode);
2253 return 0;
2254}
2255
2256static void shmem_destroy_inodecache(void)
2257{
2258 kmem_cache_destroy(shmem_inode_cachep);
2259}
2260
2261static const struct address_space_operations shmem_aops = {
2262 .writepage = shmem_writepage,
2263 .set_page_dirty = __set_page_dirty_no_writeback,
2264#ifdef CONFIG_TMPFS
2265 .write_begin = shmem_write_begin,
2266 .write_end = shmem_write_end,
2267#endif
2268 .migratepage = migrate_page,
2269 .error_remove_page = generic_error_remove_page,
2270};
2271
2272static const struct file_operations shmem_file_operations = {
2273 .mmap = shmem_mmap,
2274#ifdef CONFIG_TMPFS
2275 .llseek = generic_file_llseek,
2276 .read = do_sync_read,
2277 .write = do_sync_write,
2278 .aio_read = shmem_file_aio_read,
2279 .aio_write = generic_file_aio_write,
2280 .fsync = noop_fsync,
2281 .splice_read = shmem_file_splice_read,
2282 .splice_write = generic_file_splice_write,
2283#endif
2284};
2285
2286static const struct inode_operations shmem_inode_operations = {
2287 .setattr = shmem_setattr,
2288 .truncate_range = shmem_truncate_range,
2289#ifdef CONFIG_TMPFS_XATTR
2290 .setxattr = shmem_setxattr,
2291 .getxattr = shmem_getxattr,
2292 .listxattr = shmem_listxattr,
2293 .removexattr = shmem_removexattr,
2294#endif
2295};
2296
2297static const struct inode_operations shmem_dir_inode_operations = {
2298#ifdef CONFIG_TMPFS
2299 .create = shmem_create,
2300 .lookup = simple_lookup,
2301 .link = shmem_link,
2302 .unlink = shmem_unlink,
2303 .symlink = shmem_symlink,
2304 .mkdir = shmem_mkdir,
2305 .rmdir = shmem_rmdir,
2306 .mknod = shmem_mknod,
2307 .rename = shmem_rename,
2308#endif
2309#ifdef CONFIG_TMPFS_XATTR
2310 .setxattr = shmem_setxattr,
2311 .getxattr = shmem_getxattr,
2312 .listxattr = shmem_listxattr,
2313 .removexattr = shmem_removexattr,
2314#endif
2315#ifdef CONFIG_TMPFS_POSIX_ACL
2316 .setattr = shmem_setattr,
2317#endif
2318};
2319
2320static const struct inode_operations shmem_special_inode_operations = {
2321#ifdef CONFIG_TMPFS_XATTR
2322 .setxattr = shmem_setxattr,
2323 .getxattr = shmem_getxattr,
2324 .listxattr = shmem_listxattr,
2325 .removexattr = shmem_removexattr,
2326#endif
2327#ifdef CONFIG_TMPFS_POSIX_ACL
2328 .setattr = shmem_setattr,
2329#endif
2330};
2331
2332static const struct super_operations shmem_ops = {
2333 .alloc_inode = shmem_alloc_inode,
2334 .destroy_inode = shmem_destroy_inode,
2335#ifdef CONFIG_TMPFS
2336 .statfs = shmem_statfs,
2337 .remount_fs = shmem_remount_fs,
2338 .show_options = shmem_show_options,
2339#endif
2340 .evict_inode = shmem_evict_inode,
2341 .drop_inode = generic_delete_inode,
2342 .put_super = shmem_put_super,
2343};
2344
2345static const struct vm_operations_struct shmem_vm_ops = {
2346 .fault = shmem_fault,
2347#ifdef CONFIG_NUMA
2348 .set_policy = shmem_set_policy,
2349 .get_policy = shmem_get_policy,
2350#endif
2351};
2352
2353static struct dentry *shmem_mount(struct file_system_type *fs_type,
2354 int flags, const char *dev_name, void *data)
2355{
2356 return mount_nodev(fs_type, flags, data, shmem_fill_super);
2357}
2358
2359static struct file_system_type shmem_fs_type = {
2360 .owner = THIS_MODULE,
2361 .name = "tmpfs",
2362 .mount = shmem_mount,
2363 .kill_sb = kill_litter_super,
2364};
2365
2366int __init shmem_init(void)
2367{
2368 int error;
2369
2370 error = bdi_init(&shmem_backing_dev_info);
2371 if (error)
2372 goto out4;
2373
2374 error = shmem_init_inodecache();
2375 if (error)
2376 goto out3;
2377
2378 error = register_filesystem(&shmem_fs_type);
2379 if (error) {
2380 printk(KERN_ERR "Could not register tmpfs\n");
2381 goto out2;
2382 }
2383
2384 shm_mnt = vfs_kern_mount(&shmem_fs_type, MS_NOUSER,
2385 shmem_fs_type.name, NULL);
2386 if (IS_ERR(shm_mnt)) {
2387 error = PTR_ERR(shm_mnt);
2388 printk(KERN_ERR "Could not kern_mount tmpfs\n");
2389 goto out1;
2390 }
2391 return 0;
2392
2393out1:
2394 unregister_filesystem(&shmem_fs_type);
2395out2:
2396 shmem_destroy_inodecache();
2397out3:
2398 bdi_destroy(&shmem_backing_dev_info);
2399out4:
2400 shm_mnt = ERR_PTR(error);
2401 return error;
2402}
2403
2404#else /* !CONFIG_SHMEM */
2405
2406/*
2407 * tiny-shmem: simple shmemfs and tmpfs using ramfs code
2408 *
2409 * This is intended for small system where the benefits of the full
2410 * shmem code (swap-backed and resource-limited) are outweighed by
2411 * their complexity. On systems without swap this code should be
2412 * effectively equivalent, but much lighter weight.
2413 */
2414
2415#include <linux/ramfs.h>
2416
2417static struct file_system_type shmem_fs_type = {
2418 .name = "tmpfs",
2419 .mount = ramfs_mount,
2420 .kill_sb = kill_litter_super,
2421};
2422
2423int __init shmem_init(void)
2424{
2425 BUG_ON(register_filesystem(&shmem_fs_type) != 0);
2426
2427 shm_mnt = kern_mount(&shmem_fs_type);
2428 BUG_ON(IS_ERR(shm_mnt));
2429
2430 return 0;
2431}
2432
2433int shmem_unuse(swp_entry_t swap, struct page *page)
2434{
2435 return 0;
2436}
2437
2438int shmem_lock(struct file *file, int lock, struct user_struct *user)
2439{
2440 return 0;
2441}
2442
2443void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
2444{
2445 truncate_inode_pages_range(inode->i_mapping, lstart, lend);
2446}
2447EXPORT_SYMBOL_GPL(shmem_truncate_range);
2448
2449#define shmem_vm_ops generic_file_vm_ops
2450#define shmem_file_operations ramfs_file_operations
2451#define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev)
2452#define shmem_acct_size(flags, size) 0
2453#define shmem_unacct_size(flags, size) do {} while (0)
2454
2455#endif /* CONFIG_SHMEM */
2456
2457/* common code */
2458
2459/**
2460 * shmem_file_setup - get an unlinked file living in tmpfs
2461 * @name: name for dentry (to be seen in /proc/<pid>/maps
2462 * @size: size to be set for the file
2463 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
2464 */
2465struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
2466{
2467 int error;
2468 struct file *file;
2469 struct inode *inode;
2470 struct path path;
2471 struct dentry *root;
2472 struct qstr this;
2473
2474 if (IS_ERR(shm_mnt))
2475 return (void *)shm_mnt;
2476
2477 if (size < 0 || size > MAX_LFS_FILESIZE)
2478 return ERR_PTR(-EINVAL);
2479
2480 if (shmem_acct_size(flags, size))
2481 return ERR_PTR(-ENOMEM);
2482
2483 error = -ENOMEM;
2484 this.name = name;
2485 this.len = strlen(name);
2486 this.hash = 0; /* will go */
2487 root = shm_mnt->mnt_root;
2488 path.dentry = d_alloc(root, &this);
2489 if (!path.dentry)
2490 goto put_memory;
2491 path.mnt = mntget(shm_mnt);
2492
2493 error = -ENOSPC;
2494 inode = shmem_get_inode(root->d_sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
2495 if (!inode)
2496 goto put_dentry;
2497
2498 d_instantiate(path.dentry, inode);
2499 inode->i_size = size;
2500 inode->i_nlink = 0; /* It is unlinked */
2501#ifndef CONFIG_MMU
2502 error = ramfs_nommu_expand_for_mapping(inode, size);
2503 if (error)
2504 goto put_dentry;
2505#endif
2506
2507 error = -ENFILE;
2508 file = alloc_file(&path, FMODE_WRITE | FMODE_READ,
2509 &shmem_file_operations);
2510 if (!file)
2511 goto put_dentry;
2512
2513 return file;
2514
2515put_dentry:
2516 path_put(&path);
2517put_memory:
2518 shmem_unacct_size(flags, size);
2519 return ERR_PTR(error);
2520}
2521EXPORT_SYMBOL_GPL(shmem_file_setup);
2522
2523/**
2524 * shmem_zero_setup - setup a shared anonymous mapping
2525 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
2526 */
2527int shmem_zero_setup(struct vm_area_struct *vma)
2528{
2529 struct file *file;
2530 loff_t size = vma->vm_end - vma->vm_start;
2531
2532 file = shmem_file_setup("dev/zero", size, vma->vm_flags);
2533 if (IS_ERR(file))
2534 return PTR_ERR(file);
2535
2536 if (vma->vm_file)
2537 fput(vma->vm_file);
2538 vma->vm_file = file;
2539 vma->vm_ops = &shmem_vm_ops;
2540 vma->vm_flags |= VM_CAN_NONLINEAR;
2541 return 0;
2542}
2543
2544/**
2545 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
2546 * @mapping: the page's address_space
2547 * @index: the page index
2548 * @gfp: the page allocator flags to use if allocating
2549 *
2550 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
2551 * with any new page allocations done using the specified allocation flags.
2552 * But read_cache_page_gfp() uses the ->readpage() method: which does not
2553 * suit tmpfs, since it may have pages in swapcache, and needs to find those
2554 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
2555 *
2556 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
2557 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
2558 */
2559struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
2560 pgoff_t index, gfp_t gfp)
2561{
2562#ifdef CONFIG_SHMEM
2563 struct inode *inode = mapping->host;
2564 struct page *page;
2565 int error;
2566
2567 BUG_ON(mapping->a_ops != &shmem_aops);
2568 error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, gfp, NULL);
2569 if (error)
2570 page = ERR_PTR(error);
2571 else
2572 unlock_page(page);
2573 return page;
2574#else
2575 /*
2576 * The tiny !SHMEM case uses ramfs without swap
2577 */
2578 return read_cache_page_gfp(mapping, index, gfp);
2579#endif
2580}
2581EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);