Loading...
1#ifndef _LINUX_MM_H
2#define _LINUX_MM_H
3
4#include <linux/errno.h>
5
6#ifdef __KERNEL__
7
8#include <linux/mmdebug.h>
9#include <linux/gfp.h>
10#include <linux/bug.h>
11#include <linux/list.h>
12#include <linux/mmzone.h>
13#include <linux/rbtree.h>
14#include <linux/atomic.h>
15#include <linux/debug_locks.h>
16#include <linux/mm_types.h>
17#include <linux/range.h>
18#include <linux/pfn.h>
19#include <linux/percpu-refcount.h>
20#include <linux/bit_spinlock.h>
21#include <linux/shrinker.h>
22#include <linux/resource.h>
23#include <linux/page_ext.h>
24#include <linux/err.h>
25#include <linux/page_ref.h>
26
27struct mempolicy;
28struct anon_vma;
29struct anon_vma_chain;
30struct file_ra_state;
31struct user_struct;
32struct writeback_control;
33struct bdi_writeback;
34
35#ifndef CONFIG_NEED_MULTIPLE_NODES /* Don't use mapnrs, do it properly */
36extern unsigned long max_mapnr;
37
38static inline void set_max_mapnr(unsigned long limit)
39{
40 max_mapnr = limit;
41}
42#else
43static inline void set_max_mapnr(unsigned long limit) { }
44#endif
45
46extern unsigned long totalram_pages;
47extern void * high_memory;
48extern int page_cluster;
49
50#ifdef CONFIG_SYSCTL
51extern int sysctl_legacy_va_layout;
52#else
53#define sysctl_legacy_va_layout 0
54#endif
55
56#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
57extern const int mmap_rnd_bits_min;
58extern const int mmap_rnd_bits_max;
59extern int mmap_rnd_bits __read_mostly;
60#endif
61#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
62extern const int mmap_rnd_compat_bits_min;
63extern const int mmap_rnd_compat_bits_max;
64extern int mmap_rnd_compat_bits __read_mostly;
65#endif
66
67#include <asm/page.h>
68#include <asm/pgtable.h>
69#include <asm/processor.h>
70
71#ifndef __pa_symbol
72#define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0))
73#endif
74
75#ifndef page_to_virt
76#define page_to_virt(x) __va(PFN_PHYS(page_to_pfn(x)))
77#endif
78
79/*
80 * To prevent common memory management code establishing
81 * a zero page mapping on a read fault.
82 * This macro should be defined within <asm/pgtable.h>.
83 * s390 does this to prevent multiplexing of hardware bits
84 * related to the physical page in case of virtualization.
85 */
86#ifndef mm_forbids_zeropage
87#define mm_forbids_zeropage(X) (0)
88#endif
89
90/*
91 * Default maximum number of active map areas, this limits the number of vmas
92 * per mm struct. Users can overwrite this number by sysctl but there is a
93 * problem.
94 *
95 * When a program's coredump is generated as ELF format, a section is created
96 * per a vma. In ELF, the number of sections is represented in unsigned short.
97 * This means the number of sections should be smaller than 65535 at coredump.
98 * Because the kernel adds some informative sections to a image of program at
99 * generating coredump, we need some margin. The number of extra sections is
100 * 1-3 now and depends on arch. We use "5" as safe margin, here.
101 *
102 * ELF extended numbering allows more than 65535 sections, so 16-bit bound is
103 * not a hard limit any more. Although some userspace tools can be surprised by
104 * that.
105 */
106#define MAPCOUNT_ELF_CORE_MARGIN (5)
107#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
108
109extern int sysctl_max_map_count;
110
111extern unsigned long sysctl_user_reserve_kbytes;
112extern unsigned long sysctl_admin_reserve_kbytes;
113
114extern int sysctl_overcommit_memory;
115extern int sysctl_overcommit_ratio;
116extern unsigned long sysctl_overcommit_kbytes;
117
118extern int overcommit_ratio_handler(struct ctl_table *, int, void __user *,
119 size_t *, loff_t *);
120extern int overcommit_kbytes_handler(struct ctl_table *, int, void __user *,
121 size_t *, loff_t *);
122
123#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
124
125/* to align the pointer to the (next) page boundary */
126#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
127
128/* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */
129#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE)
130
131/*
132 * Linux kernel virtual memory manager primitives.
133 * The idea being to have a "virtual" mm in the same way
134 * we have a virtual fs - giving a cleaner interface to the
135 * mm details, and allowing different kinds of memory mappings
136 * (from shared memory to executable loading to arbitrary
137 * mmap() functions).
138 */
139
140extern struct kmem_cache *vm_area_cachep;
141
142#ifndef CONFIG_MMU
143extern struct rb_root nommu_region_tree;
144extern struct rw_semaphore nommu_region_sem;
145
146extern unsigned int kobjsize(const void *objp);
147#endif
148
149/*
150 * vm_flags in vm_area_struct, see mm_types.h.
151 * When changing, update also include/trace/events/mmflags.h
152 */
153#define VM_NONE 0x00000000
154
155#define VM_READ 0x00000001 /* currently active flags */
156#define VM_WRITE 0x00000002
157#define VM_EXEC 0x00000004
158#define VM_SHARED 0x00000008
159
160/* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */
161#define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */
162#define VM_MAYWRITE 0x00000020
163#define VM_MAYEXEC 0x00000040
164#define VM_MAYSHARE 0x00000080
165
166#define VM_GROWSDOWN 0x00000100 /* general info on the segment */
167#define VM_UFFD_MISSING 0x00000200 /* missing pages tracking */
168#define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
169#define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */
170#define VM_UFFD_WP 0x00001000 /* wrprotect pages tracking */
171
172#define VM_LOCKED 0x00002000
173#define VM_IO 0x00004000 /* Memory mapped I/O or similar */
174
175 /* Used by sys_madvise() */
176#define VM_SEQ_READ 0x00008000 /* App will access data sequentially */
177#define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */
178
179#define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */
180#define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */
181#define VM_LOCKONFAULT 0x00080000 /* Lock the pages covered when they are faulted in */
182#define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */
183#define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */
184#define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
185#define VM_ARCH_1 0x01000000 /* Architecture-specific flag */
186#define VM_ARCH_2 0x02000000
187#define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */
188
189#ifdef CONFIG_MEM_SOFT_DIRTY
190# define VM_SOFTDIRTY 0x08000000 /* Not soft dirty clean area */
191#else
192# define VM_SOFTDIRTY 0
193#endif
194
195#define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
196#define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */
197#define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */
198#define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */
199
200#ifdef CONFIG_ARCH_USES_HIGH_VMA_FLAGS
201#define VM_HIGH_ARCH_BIT_0 32 /* bit only usable on 64-bit architectures */
202#define VM_HIGH_ARCH_BIT_1 33 /* bit only usable on 64-bit architectures */
203#define VM_HIGH_ARCH_BIT_2 34 /* bit only usable on 64-bit architectures */
204#define VM_HIGH_ARCH_BIT_3 35 /* bit only usable on 64-bit architectures */
205#define VM_HIGH_ARCH_0 BIT(VM_HIGH_ARCH_BIT_0)
206#define VM_HIGH_ARCH_1 BIT(VM_HIGH_ARCH_BIT_1)
207#define VM_HIGH_ARCH_2 BIT(VM_HIGH_ARCH_BIT_2)
208#define VM_HIGH_ARCH_3 BIT(VM_HIGH_ARCH_BIT_3)
209#endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */
210
211#if defined(CONFIG_X86)
212# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
213#if defined (CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS)
214# define VM_PKEY_SHIFT VM_HIGH_ARCH_BIT_0
215# define VM_PKEY_BIT0 VM_HIGH_ARCH_0 /* A protection key is a 4-bit value */
216# define VM_PKEY_BIT1 VM_HIGH_ARCH_1
217# define VM_PKEY_BIT2 VM_HIGH_ARCH_2
218# define VM_PKEY_BIT3 VM_HIGH_ARCH_3
219#endif
220#elif defined(CONFIG_PPC)
221# define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */
222#elif defined(CONFIG_PARISC)
223# define VM_GROWSUP VM_ARCH_1
224#elif defined(CONFIG_METAG)
225# define VM_GROWSUP VM_ARCH_1
226#elif defined(CONFIG_IA64)
227# define VM_GROWSUP VM_ARCH_1
228#elif !defined(CONFIG_MMU)
229# define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */
230#endif
231
232#if defined(CONFIG_X86)
233/* MPX specific bounds table or bounds directory */
234# define VM_MPX VM_ARCH_2
235#endif
236
237#ifndef VM_GROWSUP
238# define VM_GROWSUP VM_NONE
239#endif
240
241/* Bits set in the VMA until the stack is in its final location */
242#define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ)
243
244#ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */
245#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
246#endif
247
248#ifdef CONFIG_STACK_GROWSUP
249#define VM_STACK VM_GROWSUP
250#else
251#define VM_STACK VM_GROWSDOWN
252#endif
253
254#define VM_STACK_FLAGS (VM_STACK | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
255
256/*
257 * Special vmas that are non-mergable, non-mlock()able.
258 * Note: mm/huge_memory.c VM_NO_THP depends on this definition.
259 */
260#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP)
261
262/* This mask defines which mm->def_flags a process can inherit its parent */
263#define VM_INIT_DEF_MASK VM_NOHUGEPAGE
264
265/* This mask is used to clear all the VMA flags used by mlock */
266#define VM_LOCKED_CLEAR_MASK (~(VM_LOCKED | VM_LOCKONFAULT))
267
268/*
269 * mapping from the currently active vm_flags protection bits (the
270 * low four bits) to a page protection mask..
271 */
272extern pgprot_t protection_map[16];
273
274#define FAULT_FLAG_WRITE 0x01 /* Fault was a write access */
275#define FAULT_FLAG_MKWRITE 0x02 /* Fault was mkwrite of existing pte */
276#define FAULT_FLAG_ALLOW_RETRY 0x04 /* Retry fault if blocking */
277#define FAULT_FLAG_RETRY_NOWAIT 0x08 /* Don't drop mmap_sem and wait when retrying */
278#define FAULT_FLAG_KILLABLE 0x10 /* The fault task is in SIGKILL killable region */
279#define FAULT_FLAG_TRIED 0x20 /* Second try */
280#define FAULT_FLAG_USER 0x40 /* The fault originated in userspace */
281#define FAULT_FLAG_REMOTE 0x80 /* faulting for non current tsk/mm */
282#define FAULT_FLAG_INSTRUCTION 0x100 /* The fault was during an instruction fetch */
283
284/*
285 * vm_fault is filled by the the pagefault handler and passed to the vma's
286 * ->fault function. The vma's ->fault is responsible for returning a bitmask
287 * of VM_FAULT_xxx flags that give details about how the fault was handled.
288 *
289 * MM layer fills up gfp_mask for page allocations but fault handler might
290 * alter it if its implementation requires a different allocation context.
291 *
292 * pgoff should be used in favour of virtual_address, if possible.
293 */
294struct vm_fault {
295 struct vm_area_struct *vma; /* Target VMA */
296 unsigned int flags; /* FAULT_FLAG_xxx flags */
297 gfp_t gfp_mask; /* gfp mask to be used for allocations */
298 pgoff_t pgoff; /* Logical page offset based on vma */
299 unsigned long address; /* Faulting virtual address */
300 pmd_t *pmd; /* Pointer to pmd entry matching
301 * the 'address' */
302 pte_t orig_pte; /* Value of PTE at the time of fault */
303
304 struct page *cow_page; /* Page handler may use for COW fault */
305 struct mem_cgroup *memcg; /* Cgroup cow_page belongs to */
306 struct page *page; /* ->fault handlers should return a
307 * page here, unless VM_FAULT_NOPAGE
308 * is set (which is also implied by
309 * VM_FAULT_ERROR).
310 */
311 /* These three entries are valid only while holding ptl lock */
312 pte_t *pte; /* Pointer to pte entry matching
313 * the 'address'. NULL if the page
314 * table hasn't been allocated.
315 */
316 spinlock_t *ptl; /* Page table lock.
317 * Protects pte page table if 'pte'
318 * is not NULL, otherwise pmd.
319 */
320 pgtable_t prealloc_pte; /* Pre-allocated pte page table.
321 * vm_ops->map_pages() calls
322 * alloc_set_pte() from atomic context.
323 * do_fault_around() pre-allocates
324 * page table to avoid allocation from
325 * atomic context.
326 */
327};
328
329/*
330 * These are the virtual MM functions - opening of an area, closing and
331 * unmapping it (needed to keep files on disk up-to-date etc), pointer
332 * to the functions called when a no-page or a wp-page exception occurs.
333 */
334struct vm_operations_struct {
335 void (*open)(struct vm_area_struct * area);
336 void (*close)(struct vm_area_struct * area);
337 int (*mremap)(struct vm_area_struct * area);
338 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
339 int (*pmd_fault)(struct vm_area_struct *, unsigned long address,
340 pmd_t *, unsigned int flags);
341 void (*map_pages)(struct vm_fault *vmf,
342 pgoff_t start_pgoff, pgoff_t end_pgoff);
343
344 /* notification that a previously read-only page is about to become
345 * writable, if an error is returned it will cause a SIGBUS */
346 int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
347
348 /* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */
349 int (*pfn_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
350
351 /* called by access_process_vm when get_user_pages() fails, typically
352 * for use by special VMAs that can switch between memory and hardware
353 */
354 int (*access)(struct vm_area_struct *vma, unsigned long addr,
355 void *buf, int len, int write);
356
357 /* Called by the /proc/PID/maps code to ask the vma whether it
358 * has a special name. Returning non-NULL will also cause this
359 * vma to be dumped unconditionally. */
360 const char *(*name)(struct vm_area_struct *vma);
361
362#ifdef CONFIG_NUMA
363 /*
364 * set_policy() op must add a reference to any non-NULL @new mempolicy
365 * to hold the policy upon return. Caller should pass NULL @new to
366 * remove a policy and fall back to surrounding context--i.e. do not
367 * install a MPOL_DEFAULT policy, nor the task or system default
368 * mempolicy.
369 */
370 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
371
372 /*
373 * get_policy() op must add reference [mpol_get()] to any policy at
374 * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure
375 * in mm/mempolicy.c will do this automatically.
376 * get_policy() must NOT add a ref if the policy at (vma,addr) is not
377 * marked as MPOL_SHARED. vma policies are protected by the mmap_sem.
378 * If no [shared/vma] mempolicy exists at the addr, get_policy() op
379 * must return NULL--i.e., do not "fallback" to task or system default
380 * policy.
381 */
382 struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
383 unsigned long addr);
384#endif
385 /*
386 * Called by vm_normal_page() for special PTEs to find the
387 * page for @addr. This is useful if the default behavior
388 * (using pte_page()) would not find the correct page.
389 */
390 struct page *(*find_special_page)(struct vm_area_struct *vma,
391 unsigned long addr);
392};
393
394struct mmu_gather;
395struct inode;
396
397#define page_private(page) ((page)->private)
398#define set_page_private(page, v) ((page)->private = (v))
399
400#if !defined(__HAVE_ARCH_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE)
401static inline int pmd_devmap(pmd_t pmd)
402{
403 return 0;
404}
405#endif
406
407/*
408 * FIXME: take this include out, include page-flags.h in
409 * files which need it (119 of them)
410 */
411#include <linux/page-flags.h>
412#include <linux/huge_mm.h>
413
414/*
415 * Methods to modify the page usage count.
416 *
417 * What counts for a page usage:
418 * - cache mapping (page->mapping)
419 * - private data (page->private)
420 * - page mapped in a task's page tables, each mapping
421 * is counted separately
422 *
423 * Also, many kernel routines increase the page count before a critical
424 * routine so they can be sure the page doesn't go away from under them.
425 */
426
427/*
428 * Drop a ref, return true if the refcount fell to zero (the page has no users)
429 */
430static inline int put_page_testzero(struct page *page)
431{
432 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
433 return page_ref_dec_and_test(page);
434}
435
436/*
437 * Try to grab a ref unless the page has a refcount of zero, return false if
438 * that is the case.
439 * This can be called when MMU is off so it must not access
440 * any of the virtual mappings.
441 */
442static inline int get_page_unless_zero(struct page *page)
443{
444 return page_ref_add_unless(page, 1, 0);
445}
446
447extern int page_is_ram(unsigned long pfn);
448
449enum {
450 REGION_INTERSECTS,
451 REGION_DISJOINT,
452 REGION_MIXED,
453};
454
455int region_intersects(resource_size_t offset, size_t size, unsigned long flags,
456 unsigned long desc);
457
458/* Support for virtually mapped pages */
459struct page *vmalloc_to_page(const void *addr);
460unsigned long vmalloc_to_pfn(const void *addr);
461
462/*
463 * Determine if an address is within the vmalloc range
464 *
465 * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there
466 * is no special casing required.
467 */
468static inline bool is_vmalloc_addr(const void *x)
469{
470#ifdef CONFIG_MMU
471 unsigned long addr = (unsigned long)x;
472
473 return addr >= VMALLOC_START && addr < VMALLOC_END;
474#else
475 return false;
476#endif
477}
478#ifdef CONFIG_MMU
479extern int is_vmalloc_or_module_addr(const void *x);
480#else
481static inline int is_vmalloc_or_module_addr(const void *x)
482{
483 return 0;
484}
485#endif
486
487extern void kvfree(const void *addr);
488
489static inline atomic_t *compound_mapcount_ptr(struct page *page)
490{
491 return &page[1].compound_mapcount;
492}
493
494static inline int compound_mapcount(struct page *page)
495{
496 VM_BUG_ON_PAGE(!PageCompound(page), page);
497 page = compound_head(page);
498 return atomic_read(compound_mapcount_ptr(page)) + 1;
499}
500
501/*
502 * The atomic page->_mapcount, starts from -1: so that transitions
503 * both from it and to it can be tracked, using atomic_inc_and_test
504 * and atomic_add_negative(-1).
505 */
506static inline void page_mapcount_reset(struct page *page)
507{
508 atomic_set(&(page)->_mapcount, -1);
509}
510
511int __page_mapcount(struct page *page);
512
513static inline int page_mapcount(struct page *page)
514{
515 VM_BUG_ON_PAGE(PageSlab(page), page);
516
517 if (unlikely(PageCompound(page)))
518 return __page_mapcount(page);
519 return atomic_read(&page->_mapcount) + 1;
520}
521
522#ifdef CONFIG_TRANSPARENT_HUGEPAGE
523int total_mapcount(struct page *page);
524int page_trans_huge_mapcount(struct page *page, int *total_mapcount);
525#else
526static inline int total_mapcount(struct page *page)
527{
528 return page_mapcount(page);
529}
530static inline int page_trans_huge_mapcount(struct page *page,
531 int *total_mapcount)
532{
533 int mapcount = page_mapcount(page);
534 if (total_mapcount)
535 *total_mapcount = mapcount;
536 return mapcount;
537}
538#endif
539
540static inline struct page *virt_to_head_page(const void *x)
541{
542 struct page *page = virt_to_page(x);
543
544 return compound_head(page);
545}
546
547void __put_page(struct page *page);
548
549void put_pages_list(struct list_head *pages);
550
551void split_page(struct page *page, unsigned int order);
552
553/*
554 * Compound pages have a destructor function. Provide a
555 * prototype for that function and accessor functions.
556 * These are _only_ valid on the head of a compound page.
557 */
558typedef void compound_page_dtor(struct page *);
559
560/* Keep the enum in sync with compound_page_dtors array in mm/page_alloc.c */
561enum compound_dtor_id {
562 NULL_COMPOUND_DTOR,
563 COMPOUND_PAGE_DTOR,
564#ifdef CONFIG_HUGETLB_PAGE
565 HUGETLB_PAGE_DTOR,
566#endif
567#ifdef CONFIG_TRANSPARENT_HUGEPAGE
568 TRANSHUGE_PAGE_DTOR,
569#endif
570 NR_COMPOUND_DTORS,
571};
572extern compound_page_dtor * const compound_page_dtors[];
573
574static inline void set_compound_page_dtor(struct page *page,
575 enum compound_dtor_id compound_dtor)
576{
577 VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page);
578 page[1].compound_dtor = compound_dtor;
579}
580
581static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
582{
583 VM_BUG_ON_PAGE(page[1].compound_dtor >= NR_COMPOUND_DTORS, page);
584 return compound_page_dtors[page[1].compound_dtor];
585}
586
587static inline unsigned int compound_order(struct page *page)
588{
589 if (!PageHead(page))
590 return 0;
591 return page[1].compound_order;
592}
593
594static inline void set_compound_order(struct page *page, unsigned int order)
595{
596 page[1].compound_order = order;
597}
598
599void free_compound_page(struct page *page);
600
601#ifdef CONFIG_MMU
602/*
603 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
604 * servicing faults for write access. In the normal case, do always want
605 * pte_mkwrite. But get_user_pages can cause write faults for mappings
606 * that do not have writing enabled, when used by access_process_vm.
607 */
608static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
609{
610 if (likely(vma->vm_flags & VM_WRITE))
611 pte = pte_mkwrite(pte);
612 return pte;
613}
614
615int alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg,
616 struct page *page);
617int finish_fault(struct vm_fault *vmf);
618int finish_mkwrite_fault(struct vm_fault *vmf);
619#endif
620
621/*
622 * Multiple processes may "see" the same page. E.g. for untouched
623 * mappings of /dev/null, all processes see the same page full of
624 * zeroes, and text pages of executables and shared libraries have
625 * only one copy in memory, at most, normally.
626 *
627 * For the non-reserved pages, page_count(page) denotes a reference count.
628 * page_count() == 0 means the page is free. page->lru is then used for
629 * freelist management in the buddy allocator.
630 * page_count() > 0 means the page has been allocated.
631 *
632 * Pages are allocated by the slab allocator in order to provide memory
633 * to kmalloc and kmem_cache_alloc. In this case, the management of the
634 * page, and the fields in 'struct page' are the responsibility of mm/slab.c
635 * unless a particular usage is carefully commented. (the responsibility of
636 * freeing the kmalloc memory is the caller's, of course).
637 *
638 * A page may be used by anyone else who does a __get_free_page().
639 * In this case, page_count still tracks the references, and should only
640 * be used through the normal accessor functions. The top bits of page->flags
641 * and page->virtual store page management information, but all other fields
642 * are unused and could be used privately, carefully. The management of this
643 * page is the responsibility of the one who allocated it, and those who have
644 * subsequently been given references to it.
645 *
646 * The other pages (we may call them "pagecache pages") are completely
647 * managed by the Linux memory manager: I/O, buffers, swapping etc.
648 * The following discussion applies only to them.
649 *
650 * A pagecache page contains an opaque `private' member, which belongs to the
651 * page's address_space. Usually, this is the address of a circular list of
652 * the page's disk buffers. PG_private must be set to tell the VM to call
653 * into the filesystem to release these pages.
654 *
655 * A page may belong to an inode's memory mapping. In this case, page->mapping
656 * is the pointer to the inode, and page->index is the file offset of the page,
657 * in units of PAGE_SIZE.
658 *
659 * If pagecache pages are not associated with an inode, they are said to be
660 * anonymous pages. These may become associated with the swapcache, and in that
661 * case PG_swapcache is set, and page->private is an offset into the swapcache.
662 *
663 * In either case (swapcache or inode backed), the pagecache itself holds one
664 * reference to the page. Setting PG_private should also increment the
665 * refcount. The each user mapping also has a reference to the page.
666 *
667 * The pagecache pages are stored in a per-mapping radix tree, which is
668 * rooted at mapping->page_tree, and indexed by offset.
669 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space
670 * lists, we instead now tag pages as dirty/writeback in the radix tree.
671 *
672 * All pagecache pages may be subject to I/O:
673 * - inode pages may need to be read from disk,
674 * - inode pages which have been modified and are MAP_SHARED may need
675 * to be written back to the inode on disk,
676 * - anonymous pages (including MAP_PRIVATE file mappings) which have been
677 * modified may need to be swapped out to swap space and (later) to be read
678 * back into memory.
679 */
680
681/*
682 * The zone field is never updated after free_area_init_core()
683 * sets it, so none of the operations on it need to be atomic.
684 */
685
686/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */
687#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
688#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
689#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
690#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)
691
692/*
693 * Define the bit shifts to access each section. For non-existent
694 * sections we define the shift as 0; that plus a 0 mask ensures
695 * the compiler will optimise away reference to them.
696 */
697#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
698#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
699#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
700#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))
701
702/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
703#ifdef NODE_NOT_IN_PAGE_FLAGS
704#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
705#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
706 SECTIONS_PGOFF : ZONES_PGOFF)
707#else
708#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
709#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
710 NODES_PGOFF : ZONES_PGOFF)
711#endif
712
713#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
714
715#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
716#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
717#endif
718
719#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
720#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
721#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
722#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_SHIFT) - 1)
723#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
724
725static inline enum zone_type page_zonenum(const struct page *page)
726{
727 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
728}
729
730#ifdef CONFIG_ZONE_DEVICE
731void get_zone_device_page(struct page *page);
732void put_zone_device_page(struct page *page);
733static inline bool is_zone_device_page(const struct page *page)
734{
735 return page_zonenum(page) == ZONE_DEVICE;
736}
737#else
738static inline void get_zone_device_page(struct page *page)
739{
740}
741static inline void put_zone_device_page(struct page *page)
742{
743}
744static inline bool is_zone_device_page(const struct page *page)
745{
746 return false;
747}
748#endif
749
750static inline void get_page(struct page *page)
751{
752 page = compound_head(page);
753 /*
754 * Getting a normal page or the head of a compound page
755 * requires to already have an elevated page->_refcount.
756 */
757 VM_BUG_ON_PAGE(page_ref_count(page) <= 0, page);
758 page_ref_inc(page);
759
760 if (unlikely(is_zone_device_page(page)))
761 get_zone_device_page(page);
762}
763
764static inline void put_page(struct page *page)
765{
766 page = compound_head(page);
767
768 if (put_page_testzero(page))
769 __put_page(page);
770
771 if (unlikely(is_zone_device_page(page)))
772 put_zone_device_page(page);
773}
774
775#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
776#define SECTION_IN_PAGE_FLAGS
777#endif
778
779/*
780 * The identification function is mainly used by the buddy allocator for
781 * determining if two pages could be buddies. We are not really identifying
782 * the zone since we could be using the section number id if we do not have
783 * node id available in page flags.
784 * We only guarantee that it will return the same value for two combinable
785 * pages in a zone.
786 */
787static inline int page_zone_id(struct page *page)
788{
789 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
790}
791
792static inline int zone_to_nid(struct zone *zone)
793{
794#ifdef CONFIG_NUMA
795 return zone->node;
796#else
797 return 0;
798#endif
799}
800
801#ifdef NODE_NOT_IN_PAGE_FLAGS
802extern int page_to_nid(const struct page *page);
803#else
804static inline int page_to_nid(const struct page *page)
805{
806 return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
807}
808#endif
809
810#ifdef CONFIG_NUMA_BALANCING
811static inline int cpu_pid_to_cpupid(int cpu, int pid)
812{
813 return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
814}
815
816static inline int cpupid_to_pid(int cpupid)
817{
818 return cpupid & LAST__PID_MASK;
819}
820
821static inline int cpupid_to_cpu(int cpupid)
822{
823 return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
824}
825
826static inline int cpupid_to_nid(int cpupid)
827{
828 return cpu_to_node(cpupid_to_cpu(cpupid));
829}
830
831static inline bool cpupid_pid_unset(int cpupid)
832{
833 return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
834}
835
836static inline bool cpupid_cpu_unset(int cpupid)
837{
838 return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
839}
840
841static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
842{
843 return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);
844}
845
846#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)
847#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
848static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
849{
850 return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);
851}
852
853static inline int page_cpupid_last(struct page *page)
854{
855 return page->_last_cpupid;
856}
857static inline void page_cpupid_reset_last(struct page *page)
858{
859 page->_last_cpupid = -1 & LAST_CPUPID_MASK;
860}
861#else
862static inline int page_cpupid_last(struct page *page)
863{
864 return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
865}
866
867extern int page_cpupid_xchg_last(struct page *page, int cpupid);
868
869static inline void page_cpupid_reset_last(struct page *page)
870{
871 page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
872}
873#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
874#else /* !CONFIG_NUMA_BALANCING */
875static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
876{
877 return page_to_nid(page); /* XXX */
878}
879
880static inline int page_cpupid_last(struct page *page)
881{
882 return page_to_nid(page); /* XXX */
883}
884
885static inline int cpupid_to_nid(int cpupid)
886{
887 return -1;
888}
889
890static inline int cpupid_to_pid(int cpupid)
891{
892 return -1;
893}
894
895static inline int cpupid_to_cpu(int cpupid)
896{
897 return -1;
898}
899
900static inline int cpu_pid_to_cpupid(int nid, int pid)
901{
902 return -1;
903}
904
905static inline bool cpupid_pid_unset(int cpupid)
906{
907 return 1;
908}
909
910static inline void page_cpupid_reset_last(struct page *page)
911{
912}
913
914static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
915{
916 return false;
917}
918#endif /* CONFIG_NUMA_BALANCING */
919
920static inline struct zone *page_zone(const struct page *page)
921{
922 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
923}
924
925static inline pg_data_t *page_pgdat(const struct page *page)
926{
927 return NODE_DATA(page_to_nid(page));
928}
929
930#ifdef SECTION_IN_PAGE_FLAGS
931static inline void set_page_section(struct page *page, unsigned long section)
932{
933 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
934 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
935}
936
937static inline unsigned long page_to_section(const struct page *page)
938{
939 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
940}
941#endif
942
943static inline void set_page_zone(struct page *page, enum zone_type zone)
944{
945 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
946 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
947}
948
949static inline void set_page_node(struct page *page, unsigned long node)
950{
951 page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
952 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
953}
954
955static inline void set_page_links(struct page *page, enum zone_type zone,
956 unsigned long node, unsigned long pfn)
957{
958 set_page_zone(page, zone);
959 set_page_node(page, node);
960#ifdef SECTION_IN_PAGE_FLAGS
961 set_page_section(page, pfn_to_section_nr(pfn));
962#endif
963}
964
965#ifdef CONFIG_MEMCG
966static inline struct mem_cgroup *page_memcg(struct page *page)
967{
968 return page->mem_cgroup;
969}
970static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
971{
972 WARN_ON_ONCE(!rcu_read_lock_held());
973 return READ_ONCE(page->mem_cgroup);
974}
975#else
976static inline struct mem_cgroup *page_memcg(struct page *page)
977{
978 return NULL;
979}
980static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
981{
982 WARN_ON_ONCE(!rcu_read_lock_held());
983 return NULL;
984}
985#endif
986
987/*
988 * Some inline functions in vmstat.h depend on page_zone()
989 */
990#include <linux/vmstat.h>
991
992static __always_inline void *lowmem_page_address(const struct page *page)
993{
994 return page_to_virt(page);
995}
996
997#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
998#define HASHED_PAGE_VIRTUAL
999#endif
1000
1001#if defined(WANT_PAGE_VIRTUAL)
1002static inline void *page_address(const struct page *page)
1003{
1004 return page->virtual;
1005}
1006static inline void set_page_address(struct page *page, void *address)
1007{
1008 page->virtual = address;
1009}
1010#define page_address_init() do { } while(0)
1011#endif
1012
1013#if defined(HASHED_PAGE_VIRTUAL)
1014void *page_address(const struct page *page);
1015void set_page_address(struct page *page, void *virtual);
1016void page_address_init(void);
1017#endif
1018
1019#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
1020#define page_address(page) lowmem_page_address(page)
1021#define set_page_address(page, address) do { } while(0)
1022#define page_address_init() do { } while(0)
1023#endif
1024
1025extern void *page_rmapping(struct page *page);
1026extern struct anon_vma *page_anon_vma(struct page *page);
1027extern struct address_space *page_mapping(struct page *page);
1028
1029extern struct address_space *__page_file_mapping(struct page *);
1030
1031static inline
1032struct address_space *page_file_mapping(struct page *page)
1033{
1034 if (unlikely(PageSwapCache(page)))
1035 return __page_file_mapping(page);
1036
1037 return page->mapping;
1038}
1039
1040extern pgoff_t __page_file_index(struct page *page);
1041
1042/*
1043 * Return the pagecache index of the passed page. Regular pagecache pages
1044 * use ->index whereas swapcache pages use swp_offset(->private)
1045 */
1046static inline pgoff_t page_index(struct page *page)
1047{
1048 if (unlikely(PageSwapCache(page)))
1049 return __page_file_index(page);
1050 return page->index;
1051}
1052
1053bool page_mapped(struct page *page);
1054struct address_space *page_mapping(struct page *page);
1055
1056/*
1057 * Return true only if the page has been allocated with
1058 * ALLOC_NO_WATERMARKS and the low watermark was not
1059 * met implying that the system is under some pressure.
1060 */
1061static inline bool page_is_pfmemalloc(struct page *page)
1062{
1063 /*
1064 * Page index cannot be this large so this must be
1065 * a pfmemalloc page.
1066 */
1067 return page->index == -1UL;
1068}
1069
1070/*
1071 * Only to be called by the page allocator on a freshly allocated
1072 * page.
1073 */
1074static inline void set_page_pfmemalloc(struct page *page)
1075{
1076 page->index = -1UL;
1077}
1078
1079static inline void clear_page_pfmemalloc(struct page *page)
1080{
1081 page->index = 0;
1082}
1083
1084/*
1085 * Different kinds of faults, as returned by handle_mm_fault().
1086 * Used to decide whether a process gets delivered SIGBUS or
1087 * just gets major/minor fault counters bumped up.
1088 */
1089
1090#define VM_FAULT_OOM 0x0001
1091#define VM_FAULT_SIGBUS 0x0002
1092#define VM_FAULT_MAJOR 0x0004
1093#define VM_FAULT_WRITE 0x0008 /* Special case for get_user_pages */
1094#define VM_FAULT_HWPOISON 0x0010 /* Hit poisoned small page */
1095#define VM_FAULT_HWPOISON_LARGE 0x0020 /* Hit poisoned large page. Index encoded in upper bits */
1096#define VM_FAULT_SIGSEGV 0x0040
1097
1098#define VM_FAULT_NOPAGE 0x0100 /* ->fault installed the pte, not return page */
1099#define VM_FAULT_LOCKED 0x0200 /* ->fault locked the returned page */
1100#define VM_FAULT_RETRY 0x0400 /* ->fault blocked, must retry */
1101#define VM_FAULT_FALLBACK 0x0800 /* huge page fault failed, fall back to small */
1102#define VM_FAULT_DONE_COW 0x1000 /* ->fault has fully handled COW */
1103
1104#define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */
1105
1106#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | \
1107 VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE | \
1108 VM_FAULT_FALLBACK)
1109
1110/* Encode hstate index for a hwpoisoned large page */
1111#define VM_FAULT_SET_HINDEX(x) ((x) << 12)
1112#define VM_FAULT_GET_HINDEX(x) (((x) >> 12) & 0xf)
1113
1114/*
1115 * Can be called by the pagefault handler when it gets a VM_FAULT_OOM.
1116 */
1117extern void pagefault_out_of_memory(void);
1118
1119#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
1120
1121/*
1122 * Flags passed to show_mem() and show_free_areas() to suppress output in
1123 * various contexts.
1124 */
1125#define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */
1126
1127extern void show_free_areas(unsigned int flags);
1128extern bool skip_free_areas_node(unsigned int flags, int nid);
1129
1130int shmem_zero_setup(struct vm_area_struct *);
1131#ifdef CONFIG_SHMEM
1132bool shmem_mapping(struct address_space *mapping);
1133#else
1134static inline bool shmem_mapping(struct address_space *mapping)
1135{
1136 return false;
1137}
1138#endif
1139
1140extern bool can_do_mlock(void);
1141extern int user_shm_lock(size_t, struct user_struct *);
1142extern void user_shm_unlock(size_t, struct user_struct *);
1143
1144/*
1145 * Parameter block passed down to zap_pte_range in exceptional cases.
1146 */
1147struct zap_details {
1148 struct address_space *check_mapping; /* Check page->mapping if set */
1149 pgoff_t first_index; /* Lowest page->index to unmap */
1150 pgoff_t last_index; /* Highest page->index to unmap */
1151 bool ignore_dirty; /* Ignore dirty pages */
1152 bool check_swap_entries; /* Check also swap entries */
1153};
1154
1155struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1156 pte_t pte);
1157struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
1158 pmd_t pmd);
1159
1160int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
1161 unsigned long size);
1162void zap_page_range(struct vm_area_struct *vma, unsigned long address,
1163 unsigned long size, struct zap_details *);
1164void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
1165 unsigned long start, unsigned long end);
1166
1167/**
1168 * mm_walk - callbacks for walk_page_range
1169 * @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry
1170 * this handler is required to be able to handle
1171 * pmd_trans_huge() pmds. They may simply choose to
1172 * split_huge_page() instead of handling it explicitly.
1173 * @pte_entry: if set, called for each non-empty PTE (4th-level) entry
1174 * @pte_hole: if set, called for each hole at all levels
1175 * @hugetlb_entry: if set, called for each hugetlb entry
1176 * @test_walk: caller specific callback function to determine whether
1177 * we walk over the current vma or not. Returning 0
1178 * value means "do page table walk over the current vma,"
1179 * and a negative one means "abort current page table walk
1180 * right now." 1 means "skip the current vma."
1181 * @mm: mm_struct representing the target process of page table walk
1182 * @vma: vma currently walked (NULL if walking outside vmas)
1183 * @private: private data for callbacks' usage
1184 *
1185 * (see the comment on walk_page_range() for more details)
1186 */
1187struct mm_walk {
1188 int (*pmd_entry)(pmd_t *pmd, unsigned long addr,
1189 unsigned long next, struct mm_walk *walk);
1190 int (*pte_entry)(pte_t *pte, unsigned long addr,
1191 unsigned long next, struct mm_walk *walk);
1192 int (*pte_hole)(unsigned long addr, unsigned long next,
1193 struct mm_walk *walk);
1194 int (*hugetlb_entry)(pte_t *pte, unsigned long hmask,
1195 unsigned long addr, unsigned long next,
1196 struct mm_walk *walk);
1197 int (*test_walk)(unsigned long addr, unsigned long next,
1198 struct mm_walk *walk);
1199 struct mm_struct *mm;
1200 struct vm_area_struct *vma;
1201 void *private;
1202};
1203
1204int walk_page_range(unsigned long addr, unsigned long end,
1205 struct mm_walk *walk);
1206int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk);
1207void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
1208 unsigned long end, unsigned long floor, unsigned long ceiling);
1209int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
1210 struct vm_area_struct *vma);
1211void unmap_mapping_range(struct address_space *mapping,
1212 loff_t const holebegin, loff_t const holelen, int even_cows);
1213int follow_pte_pmd(struct mm_struct *mm, unsigned long address,
1214 pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp);
1215int follow_pfn(struct vm_area_struct *vma, unsigned long address,
1216 unsigned long *pfn);
1217int follow_phys(struct vm_area_struct *vma, unsigned long address,
1218 unsigned int flags, unsigned long *prot, resource_size_t *phys);
1219int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
1220 void *buf, int len, int write);
1221
1222static inline void unmap_shared_mapping_range(struct address_space *mapping,
1223 loff_t const holebegin, loff_t const holelen)
1224{
1225 unmap_mapping_range(mapping, holebegin, holelen, 0);
1226}
1227
1228extern void truncate_pagecache(struct inode *inode, loff_t new);
1229extern void truncate_setsize(struct inode *inode, loff_t newsize);
1230void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
1231void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
1232int truncate_inode_page(struct address_space *mapping, struct page *page);
1233int generic_error_remove_page(struct address_space *mapping, struct page *page);
1234int invalidate_inode_page(struct page *page);
1235
1236#ifdef CONFIG_MMU
1237extern int handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
1238 unsigned int flags);
1239extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
1240 unsigned long address, unsigned int fault_flags,
1241 bool *unlocked);
1242#else
1243static inline int handle_mm_fault(struct vm_area_struct *vma,
1244 unsigned long address, unsigned int flags)
1245{
1246 /* should never happen if there's no MMU */
1247 BUG();
1248 return VM_FAULT_SIGBUS;
1249}
1250static inline int fixup_user_fault(struct task_struct *tsk,
1251 struct mm_struct *mm, unsigned long address,
1252 unsigned int fault_flags, bool *unlocked)
1253{
1254 /* should never happen if there's no MMU */
1255 BUG();
1256 return -EFAULT;
1257}
1258#endif
1259
1260extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len,
1261 unsigned int gup_flags);
1262extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
1263 void *buf, int len, unsigned int gup_flags);
1264extern int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
1265 unsigned long addr, void *buf, int len, unsigned int gup_flags);
1266
1267long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
1268 unsigned long start, unsigned long nr_pages,
1269 unsigned int gup_flags, struct page **pages,
1270 struct vm_area_struct **vmas, int *locked);
1271long get_user_pages(unsigned long start, unsigned long nr_pages,
1272 unsigned int gup_flags, struct page **pages,
1273 struct vm_area_struct **vmas);
1274long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
1275 unsigned int gup_flags, struct page **pages, int *locked);
1276long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
1277 struct page **pages, unsigned int gup_flags);
1278int get_user_pages_fast(unsigned long start, int nr_pages, int write,
1279 struct page **pages);
1280
1281/* Container for pinned pfns / pages */
1282struct frame_vector {
1283 unsigned int nr_allocated; /* Number of frames we have space for */
1284 unsigned int nr_frames; /* Number of frames stored in ptrs array */
1285 bool got_ref; /* Did we pin pages by getting page ref? */
1286 bool is_pfns; /* Does array contain pages or pfns? */
1287 void *ptrs[0]; /* Array of pinned pfns / pages. Use
1288 * pfns_vector_pages() or pfns_vector_pfns()
1289 * for access */
1290};
1291
1292struct frame_vector *frame_vector_create(unsigned int nr_frames);
1293void frame_vector_destroy(struct frame_vector *vec);
1294int get_vaddr_frames(unsigned long start, unsigned int nr_pfns,
1295 unsigned int gup_flags, struct frame_vector *vec);
1296void put_vaddr_frames(struct frame_vector *vec);
1297int frame_vector_to_pages(struct frame_vector *vec);
1298void frame_vector_to_pfns(struct frame_vector *vec);
1299
1300static inline unsigned int frame_vector_count(struct frame_vector *vec)
1301{
1302 return vec->nr_frames;
1303}
1304
1305static inline struct page **frame_vector_pages(struct frame_vector *vec)
1306{
1307 if (vec->is_pfns) {
1308 int err = frame_vector_to_pages(vec);
1309
1310 if (err)
1311 return ERR_PTR(err);
1312 }
1313 return (struct page **)(vec->ptrs);
1314}
1315
1316static inline unsigned long *frame_vector_pfns(struct frame_vector *vec)
1317{
1318 if (!vec->is_pfns)
1319 frame_vector_to_pfns(vec);
1320 return (unsigned long *)(vec->ptrs);
1321}
1322
1323struct kvec;
1324int get_kernel_pages(const struct kvec *iov, int nr_pages, int write,
1325 struct page **pages);
1326int get_kernel_page(unsigned long start, int write, struct page **pages);
1327struct page *get_dump_page(unsigned long addr);
1328
1329extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
1330extern void do_invalidatepage(struct page *page, unsigned int offset,
1331 unsigned int length);
1332
1333int __set_page_dirty_nobuffers(struct page *page);
1334int __set_page_dirty_no_writeback(struct page *page);
1335int redirty_page_for_writepage(struct writeback_control *wbc,
1336 struct page *page);
1337void account_page_dirtied(struct page *page, struct address_space *mapping);
1338void account_page_cleaned(struct page *page, struct address_space *mapping,
1339 struct bdi_writeback *wb);
1340int set_page_dirty(struct page *page);
1341int set_page_dirty_lock(struct page *page);
1342void cancel_dirty_page(struct page *page);
1343int clear_page_dirty_for_io(struct page *page);
1344
1345int get_cmdline(struct task_struct *task, char *buffer, int buflen);
1346
1347/* Is the vma a continuation of the stack vma above it? */
1348static inline int vma_growsdown(struct vm_area_struct *vma, unsigned long addr)
1349{
1350 return vma && (vma->vm_end == addr) && (vma->vm_flags & VM_GROWSDOWN);
1351}
1352
1353static inline bool vma_is_anonymous(struct vm_area_struct *vma)
1354{
1355 return !vma->vm_ops;
1356}
1357
1358static inline int stack_guard_page_start(struct vm_area_struct *vma,
1359 unsigned long addr)
1360{
1361 return (vma->vm_flags & VM_GROWSDOWN) &&
1362 (vma->vm_start == addr) &&
1363 !vma_growsdown(vma->vm_prev, addr);
1364}
1365
1366/* Is the vma a continuation of the stack vma below it? */
1367static inline int vma_growsup(struct vm_area_struct *vma, unsigned long addr)
1368{
1369 return vma && (vma->vm_start == addr) && (vma->vm_flags & VM_GROWSUP);
1370}
1371
1372static inline int stack_guard_page_end(struct vm_area_struct *vma,
1373 unsigned long addr)
1374{
1375 return (vma->vm_flags & VM_GROWSUP) &&
1376 (vma->vm_end == addr) &&
1377 !vma_growsup(vma->vm_next, addr);
1378}
1379
1380int vma_is_stack_for_current(struct vm_area_struct *vma);
1381
1382extern unsigned long move_page_tables(struct vm_area_struct *vma,
1383 unsigned long old_addr, struct vm_area_struct *new_vma,
1384 unsigned long new_addr, unsigned long len,
1385 bool need_rmap_locks);
1386extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
1387 unsigned long end, pgprot_t newprot,
1388 int dirty_accountable, int prot_numa);
1389extern int mprotect_fixup(struct vm_area_struct *vma,
1390 struct vm_area_struct **pprev, unsigned long start,
1391 unsigned long end, unsigned long newflags);
1392
1393/*
1394 * doesn't attempt to fault and will return short.
1395 */
1396int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
1397 struct page **pages);
1398/*
1399 * per-process(per-mm_struct) statistics.
1400 */
1401static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
1402{
1403 long val = atomic_long_read(&mm->rss_stat.count[member]);
1404
1405#ifdef SPLIT_RSS_COUNTING
1406 /*
1407 * counter is updated in asynchronous manner and may go to minus.
1408 * But it's never be expected number for users.
1409 */
1410 if (val < 0)
1411 val = 0;
1412#endif
1413 return (unsigned long)val;
1414}
1415
1416static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
1417{
1418 atomic_long_add(value, &mm->rss_stat.count[member]);
1419}
1420
1421static inline void inc_mm_counter(struct mm_struct *mm, int member)
1422{
1423 atomic_long_inc(&mm->rss_stat.count[member]);
1424}
1425
1426static inline void dec_mm_counter(struct mm_struct *mm, int member)
1427{
1428 atomic_long_dec(&mm->rss_stat.count[member]);
1429}
1430
1431/* Optimized variant when page is already known not to be PageAnon */
1432static inline int mm_counter_file(struct page *page)
1433{
1434 if (PageSwapBacked(page))
1435 return MM_SHMEMPAGES;
1436 return MM_FILEPAGES;
1437}
1438
1439static inline int mm_counter(struct page *page)
1440{
1441 if (PageAnon(page))
1442 return MM_ANONPAGES;
1443 return mm_counter_file(page);
1444}
1445
1446static inline unsigned long get_mm_rss(struct mm_struct *mm)
1447{
1448 return get_mm_counter(mm, MM_FILEPAGES) +
1449 get_mm_counter(mm, MM_ANONPAGES) +
1450 get_mm_counter(mm, MM_SHMEMPAGES);
1451}
1452
1453static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
1454{
1455 return max(mm->hiwater_rss, get_mm_rss(mm));
1456}
1457
1458static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
1459{
1460 return max(mm->hiwater_vm, mm->total_vm);
1461}
1462
1463static inline void update_hiwater_rss(struct mm_struct *mm)
1464{
1465 unsigned long _rss = get_mm_rss(mm);
1466
1467 if ((mm)->hiwater_rss < _rss)
1468 (mm)->hiwater_rss = _rss;
1469}
1470
1471static inline void update_hiwater_vm(struct mm_struct *mm)
1472{
1473 if (mm->hiwater_vm < mm->total_vm)
1474 mm->hiwater_vm = mm->total_vm;
1475}
1476
1477static inline void reset_mm_hiwater_rss(struct mm_struct *mm)
1478{
1479 mm->hiwater_rss = get_mm_rss(mm);
1480}
1481
1482static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
1483 struct mm_struct *mm)
1484{
1485 unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
1486
1487 if (*maxrss < hiwater_rss)
1488 *maxrss = hiwater_rss;
1489}
1490
1491#if defined(SPLIT_RSS_COUNTING)
1492void sync_mm_rss(struct mm_struct *mm);
1493#else
1494static inline void sync_mm_rss(struct mm_struct *mm)
1495{
1496}
1497#endif
1498
1499#ifndef __HAVE_ARCH_PTE_DEVMAP
1500static inline int pte_devmap(pte_t pte)
1501{
1502 return 0;
1503}
1504#endif
1505
1506int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot);
1507
1508extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
1509 spinlock_t **ptl);
1510static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
1511 spinlock_t **ptl)
1512{
1513 pte_t *ptep;
1514 __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
1515 return ptep;
1516}
1517
1518#ifdef __PAGETABLE_PUD_FOLDED
1519static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
1520 unsigned long address)
1521{
1522 return 0;
1523}
1524#else
1525int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
1526#endif
1527
1528#if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)
1529static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
1530 unsigned long address)
1531{
1532 return 0;
1533}
1534
1535static inline void mm_nr_pmds_init(struct mm_struct *mm) {}
1536
1537static inline unsigned long mm_nr_pmds(struct mm_struct *mm)
1538{
1539 return 0;
1540}
1541
1542static inline void mm_inc_nr_pmds(struct mm_struct *mm) {}
1543static inline void mm_dec_nr_pmds(struct mm_struct *mm) {}
1544
1545#else
1546int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
1547
1548static inline void mm_nr_pmds_init(struct mm_struct *mm)
1549{
1550 atomic_long_set(&mm->nr_pmds, 0);
1551}
1552
1553static inline unsigned long mm_nr_pmds(struct mm_struct *mm)
1554{
1555 return atomic_long_read(&mm->nr_pmds);
1556}
1557
1558static inline void mm_inc_nr_pmds(struct mm_struct *mm)
1559{
1560 atomic_long_inc(&mm->nr_pmds);
1561}
1562
1563static inline void mm_dec_nr_pmds(struct mm_struct *mm)
1564{
1565 atomic_long_dec(&mm->nr_pmds);
1566}
1567#endif
1568
1569int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
1570int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
1571
1572/*
1573 * The following ifdef needed to get the 4level-fixup.h header to work.
1574 * Remove it when 4level-fixup.h has been removed.
1575 */
1576#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
1577static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
1578{
1579 return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
1580 NULL: pud_offset(pgd, address);
1581}
1582
1583static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
1584{
1585 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
1586 NULL: pmd_offset(pud, address);
1587}
1588#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
1589
1590#if USE_SPLIT_PTE_PTLOCKS
1591#if ALLOC_SPLIT_PTLOCKS
1592void __init ptlock_cache_init(void);
1593extern bool ptlock_alloc(struct page *page);
1594extern void ptlock_free(struct page *page);
1595
1596static inline spinlock_t *ptlock_ptr(struct page *page)
1597{
1598 return page->ptl;
1599}
1600#else /* ALLOC_SPLIT_PTLOCKS */
1601static inline void ptlock_cache_init(void)
1602{
1603}
1604
1605static inline bool ptlock_alloc(struct page *page)
1606{
1607 return true;
1608}
1609
1610static inline void ptlock_free(struct page *page)
1611{
1612}
1613
1614static inline spinlock_t *ptlock_ptr(struct page *page)
1615{
1616 return &page->ptl;
1617}
1618#endif /* ALLOC_SPLIT_PTLOCKS */
1619
1620static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1621{
1622 return ptlock_ptr(pmd_page(*pmd));
1623}
1624
1625static inline bool ptlock_init(struct page *page)
1626{
1627 /*
1628 * prep_new_page() initialize page->private (and therefore page->ptl)
1629 * with 0. Make sure nobody took it in use in between.
1630 *
1631 * It can happen if arch try to use slab for page table allocation:
1632 * slab code uses page->slab_cache, which share storage with page->ptl.
1633 */
1634 VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page);
1635 if (!ptlock_alloc(page))
1636 return false;
1637 spin_lock_init(ptlock_ptr(page));
1638 return true;
1639}
1640
1641/* Reset page->mapping so free_pages_check won't complain. */
1642static inline void pte_lock_deinit(struct page *page)
1643{
1644 page->mapping = NULL;
1645 ptlock_free(page);
1646}
1647
1648#else /* !USE_SPLIT_PTE_PTLOCKS */
1649/*
1650 * We use mm->page_table_lock to guard all pagetable pages of the mm.
1651 */
1652static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1653{
1654 return &mm->page_table_lock;
1655}
1656static inline void ptlock_cache_init(void) {}
1657static inline bool ptlock_init(struct page *page) { return true; }
1658static inline void pte_lock_deinit(struct page *page) {}
1659#endif /* USE_SPLIT_PTE_PTLOCKS */
1660
1661static inline void pgtable_init(void)
1662{
1663 ptlock_cache_init();
1664 pgtable_cache_init();
1665}
1666
1667static inline bool pgtable_page_ctor(struct page *page)
1668{
1669 if (!ptlock_init(page))
1670 return false;
1671 inc_zone_page_state(page, NR_PAGETABLE);
1672 return true;
1673}
1674
1675static inline void pgtable_page_dtor(struct page *page)
1676{
1677 pte_lock_deinit(page);
1678 dec_zone_page_state(page, NR_PAGETABLE);
1679}
1680
1681#define pte_offset_map_lock(mm, pmd, address, ptlp) \
1682({ \
1683 spinlock_t *__ptl = pte_lockptr(mm, pmd); \
1684 pte_t *__pte = pte_offset_map(pmd, address); \
1685 *(ptlp) = __ptl; \
1686 spin_lock(__ptl); \
1687 __pte; \
1688})
1689
1690#define pte_unmap_unlock(pte, ptl) do { \
1691 spin_unlock(ptl); \
1692 pte_unmap(pte); \
1693} while (0)
1694
1695#define pte_alloc(mm, pmd, address) \
1696 (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd, address))
1697
1698#define pte_alloc_map(mm, pmd, address) \
1699 (pte_alloc(mm, pmd, address) ? NULL : pte_offset_map(pmd, address))
1700
1701#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
1702 (pte_alloc(mm, pmd, address) ? \
1703 NULL : pte_offset_map_lock(mm, pmd, address, ptlp))
1704
1705#define pte_alloc_kernel(pmd, address) \
1706 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
1707 NULL: pte_offset_kernel(pmd, address))
1708
1709#if USE_SPLIT_PMD_PTLOCKS
1710
1711static struct page *pmd_to_page(pmd_t *pmd)
1712{
1713 unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
1714 return virt_to_page((void *)((unsigned long) pmd & mask));
1715}
1716
1717static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1718{
1719 return ptlock_ptr(pmd_to_page(pmd));
1720}
1721
1722static inline bool pgtable_pmd_page_ctor(struct page *page)
1723{
1724#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1725 page->pmd_huge_pte = NULL;
1726#endif
1727 return ptlock_init(page);
1728}
1729
1730static inline void pgtable_pmd_page_dtor(struct page *page)
1731{
1732#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1733 VM_BUG_ON_PAGE(page->pmd_huge_pte, page);
1734#endif
1735 ptlock_free(page);
1736}
1737
1738#define pmd_huge_pte(mm, pmd) (pmd_to_page(pmd)->pmd_huge_pte)
1739
1740#else
1741
1742static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1743{
1744 return &mm->page_table_lock;
1745}
1746
1747static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }
1748static inline void pgtable_pmd_page_dtor(struct page *page) {}
1749
1750#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
1751
1752#endif
1753
1754static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
1755{
1756 spinlock_t *ptl = pmd_lockptr(mm, pmd);
1757 spin_lock(ptl);
1758 return ptl;
1759}
1760
1761extern void __init pagecache_init(void);
1762
1763extern void free_area_init(unsigned long * zones_size);
1764extern void free_area_init_node(int nid, unsigned long * zones_size,
1765 unsigned long zone_start_pfn, unsigned long *zholes_size);
1766extern void free_initmem(void);
1767
1768/*
1769 * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK)
1770 * into the buddy system. The freed pages will be poisoned with pattern
1771 * "poison" if it's within range [0, UCHAR_MAX].
1772 * Return pages freed into the buddy system.
1773 */
1774extern unsigned long free_reserved_area(void *start, void *end,
1775 int poison, char *s);
1776
1777#ifdef CONFIG_HIGHMEM
1778/*
1779 * Free a highmem page into the buddy system, adjusting totalhigh_pages
1780 * and totalram_pages.
1781 */
1782extern void free_highmem_page(struct page *page);
1783#endif
1784
1785extern void adjust_managed_page_count(struct page *page, long count);
1786extern void mem_init_print_info(const char *str);
1787
1788extern void reserve_bootmem_region(phys_addr_t start, phys_addr_t end);
1789
1790/* Free the reserved page into the buddy system, so it gets managed. */
1791static inline void __free_reserved_page(struct page *page)
1792{
1793 ClearPageReserved(page);
1794 init_page_count(page);
1795 __free_page(page);
1796}
1797
1798static inline void free_reserved_page(struct page *page)
1799{
1800 __free_reserved_page(page);
1801 adjust_managed_page_count(page, 1);
1802}
1803
1804static inline void mark_page_reserved(struct page *page)
1805{
1806 SetPageReserved(page);
1807 adjust_managed_page_count(page, -1);
1808}
1809
1810/*
1811 * Default method to free all the __init memory into the buddy system.
1812 * The freed pages will be poisoned with pattern "poison" if it's within
1813 * range [0, UCHAR_MAX].
1814 * Return pages freed into the buddy system.
1815 */
1816static inline unsigned long free_initmem_default(int poison)
1817{
1818 extern char __init_begin[], __init_end[];
1819
1820 return free_reserved_area(&__init_begin, &__init_end,
1821 poison, "unused kernel");
1822}
1823
1824static inline unsigned long get_num_physpages(void)
1825{
1826 int nid;
1827 unsigned long phys_pages = 0;
1828
1829 for_each_online_node(nid)
1830 phys_pages += node_present_pages(nid);
1831
1832 return phys_pages;
1833}
1834
1835#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1836/*
1837 * With CONFIG_HAVE_MEMBLOCK_NODE_MAP set, an architecture may initialise its
1838 * zones, allocate the backing mem_map and account for memory holes in a more
1839 * architecture independent manner. This is a substitute for creating the
1840 * zone_sizes[] and zholes_size[] arrays and passing them to
1841 * free_area_init_node()
1842 *
1843 * An architecture is expected to register range of page frames backed by
1844 * physical memory with memblock_add[_node]() before calling
1845 * free_area_init_nodes() passing in the PFN each zone ends at. At a basic
1846 * usage, an architecture is expected to do something like
1847 *
1848 * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn,
1849 * max_highmem_pfn};
1850 * for_each_valid_physical_page_range()
1851 * memblock_add_node(base, size, nid)
1852 * free_area_init_nodes(max_zone_pfns);
1853 *
1854 * free_bootmem_with_active_regions() calls free_bootmem_node() for each
1855 * registered physical page range. Similarly
1856 * sparse_memory_present_with_active_regions() calls memory_present() for
1857 * each range when SPARSEMEM is enabled.
1858 *
1859 * See mm/page_alloc.c for more information on each function exposed by
1860 * CONFIG_HAVE_MEMBLOCK_NODE_MAP.
1861 */
1862extern void free_area_init_nodes(unsigned long *max_zone_pfn);
1863unsigned long node_map_pfn_alignment(void);
1864unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn,
1865 unsigned long end_pfn);
1866extern unsigned long absent_pages_in_range(unsigned long start_pfn,
1867 unsigned long end_pfn);
1868extern void get_pfn_range_for_nid(unsigned int nid,
1869 unsigned long *start_pfn, unsigned long *end_pfn);
1870extern unsigned long find_min_pfn_with_active_regions(void);
1871extern void free_bootmem_with_active_regions(int nid,
1872 unsigned long max_low_pfn);
1873extern void sparse_memory_present_with_active_regions(int nid);
1874
1875#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
1876
1877#if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \
1878 !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
1879static inline int __early_pfn_to_nid(unsigned long pfn,
1880 struct mminit_pfnnid_cache *state)
1881{
1882 return 0;
1883}
1884#else
1885/* please see mm/page_alloc.c */
1886extern int __meminit early_pfn_to_nid(unsigned long pfn);
1887/* there is a per-arch backend function. */
1888extern int __meminit __early_pfn_to_nid(unsigned long pfn,
1889 struct mminit_pfnnid_cache *state);
1890#endif
1891
1892extern void set_dma_reserve(unsigned long new_dma_reserve);
1893extern void memmap_init_zone(unsigned long, int, unsigned long,
1894 unsigned long, enum memmap_context);
1895extern void setup_per_zone_wmarks(void);
1896extern int __meminit init_per_zone_wmark_min(void);
1897extern void mem_init(void);
1898extern void __init mmap_init(void);
1899extern void show_mem(unsigned int flags);
1900extern long si_mem_available(void);
1901extern void si_meminfo(struct sysinfo * val);
1902extern void si_meminfo_node(struct sysinfo *val, int nid);
1903#ifdef __HAVE_ARCH_RESERVED_KERNEL_PAGES
1904extern unsigned long arch_reserved_kernel_pages(void);
1905#endif
1906
1907extern __printf(2, 3)
1908void warn_alloc(gfp_t gfp_mask, const char *fmt, ...);
1909
1910extern void setup_per_cpu_pageset(void);
1911
1912extern void zone_pcp_update(struct zone *zone);
1913extern void zone_pcp_reset(struct zone *zone);
1914
1915/* page_alloc.c */
1916extern int min_free_kbytes;
1917extern int watermark_scale_factor;
1918
1919/* nommu.c */
1920extern atomic_long_t mmap_pages_allocated;
1921extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
1922
1923/* interval_tree.c */
1924void vma_interval_tree_insert(struct vm_area_struct *node,
1925 struct rb_root *root);
1926void vma_interval_tree_insert_after(struct vm_area_struct *node,
1927 struct vm_area_struct *prev,
1928 struct rb_root *root);
1929void vma_interval_tree_remove(struct vm_area_struct *node,
1930 struct rb_root *root);
1931struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root *root,
1932 unsigned long start, unsigned long last);
1933struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
1934 unsigned long start, unsigned long last);
1935
1936#define vma_interval_tree_foreach(vma, root, start, last) \
1937 for (vma = vma_interval_tree_iter_first(root, start, last); \
1938 vma; vma = vma_interval_tree_iter_next(vma, start, last))
1939
1940void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
1941 struct rb_root *root);
1942void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
1943 struct rb_root *root);
1944struct anon_vma_chain *anon_vma_interval_tree_iter_first(
1945 struct rb_root *root, unsigned long start, unsigned long last);
1946struct anon_vma_chain *anon_vma_interval_tree_iter_next(
1947 struct anon_vma_chain *node, unsigned long start, unsigned long last);
1948#ifdef CONFIG_DEBUG_VM_RB
1949void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
1950#endif
1951
1952#define anon_vma_interval_tree_foreach(avc, root, start, last) \
1953 for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
1954 avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
1955
1956/* mmap.c */
1957extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
1958extern int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
1959 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert,
1960 struct vm_area_struct *expand);
1961static inline int vma_adjust(struct vm_area_struct *vma, unsigned long start,
1962 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
1963{
1964 return __vma_adjust(vma, start, end, pgoff, insert, NULL);
1965}
1966extern struct vm_area_struct *vma_merge(struct mm_struct *,
1967 struct vm_area_struct *prev, unsigned long addr, unsigned long end,
1968 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
1969 struct mempolicy *, struct vm_userfaultfd_ctx);
1970extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
1971extern int split_vma(struct mm_struct *,
1972 struct vm_area_struct *, unsigned long addr, int new_below);
1973extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
1974extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
1975 struct rb_node **, struct rb_node *);
1976extern void unlink_file_vma(struct vm_area_struct *);
1977extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
1978 unsigned long addr, unsigned long len, pgoff_t pgoff,
1979 bool *need_rmap_locks);
1980extern void exit_mmap(struct mm_struct *);
1981
1982static inline int check_data_rlimit(unsigned long rlim,
1983 unsigned long new,
1984 unsigned long start,
1985 unsigned long end_data,
1986 unsigned long start_data)
1987{
1988 if (rlim < RLIM_INFINITY) {
1989 if (((new - start) + (end_data - start_data)) > rlim)
1990 return -ENOSPC;
1991 }
1992
1993 return 0;
1994}
1995
1996extern int mm_take_all_locks(struct mm_struct *mm);
1997extern void mm_drop_all_locks(struct mm_struct *mm);
1998
1999extern void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
2000extern struct file *get_mm_exe_file(struct mm_struct *mm);
2001extern struct file *get_task_exe_file(struct task_struct *task);
2002
2003extern bool may_expand_vm(struct mm_struct *, vm_flags_t, unsigned long npages);
2004extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages);
2005
2006extern bool vma_is_special_mapping(const struct vm_area_struct *vma,
2007 const struct vm_special_mapping *sm);
2008extern struct vm_area_struct *_install_special_mapping(struct mm_struct *mm,
2009 unsigned long addr, unsigned long len,
2010 unsigned long flags,
2011 const struct vm_special_mapping *spec);
2012/* This is an obsolete alternative to _install_special_mapping. */
2013extern int install_special_mapping(struct mm_struct *mm,
2014 unsigned long addr, unsigned long len,
2015 unsigned long flags, struct page **pages);
2016
2017extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
2018
2019extern unsigned long mmap_region(struct file *file, unsigned long addr,
2020 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff);
2021extern unsigned long do_mmap(struct file *file, unsigned long addr,
2022 unsigned long len, unsigned long prot, unsigned long flags,
2023 vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate);
2024extern int do_munmap(struct mm_struct *, unsigned long, size_t);
2025
2026static inline unsigned long
2027do_mmap_pgoff(struct file *file, unsigned long addr,
2028 unsigned long len, unsigned long prot, unsigned long flags,
2029 unsigned long pgoff, unsigned long *populate)
2030{
2031 return do_mmap(file, addr, len, prot, flags, 0, pgoff, populate);
2032}
2033
2034#ifdef CONFIG_MMU
2035extern int __mm_populate(unsigned long addr, unsigned long len,
2036 int ignore_errors);
2037static inline void mm_populate(unsigned long addr, unsigned long len)
2038{
2039 /* Ignore errors */
2040 (void) __mm_populate(addr, len, 1);
2041}
2042#else
2043static inline void mm_populate(unsigned long addr, unsigned long len) {}
2044#endif
2045
2046/* These take the mm semaphore themselves */
2047extern int __must_check vm_brk(unsigned long, unsigned long);
2048extern int vm_munmap(unsigned long, size_t);
2049extern unsigned long __must_check vm_mmap(struct file *, unsigned long,
2050 unsigned long, unsigned long,
2051 unsigned long, unsigned long);
2052
2053struct vm_unmapped_area_info {
2054#define VM_UNMAPPED_AREA_TOPDOWN 1
2055 unsigned long flags;
2056 unsigned long length;
2057 unsigned long low_limit;
2058 unsigned long high_limit;
2059 unsigned long align_mask;
2060 unsigned long align_offset;
2061};
2062
2063extern unsigned long unmapped_area(struct vm_unmapped_area_info *info);
2064extern unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);
2065
2066/*
2067 * Search for an unmapped address range.
2068 *
2069 * We are looking for a range that:
2070 * - does not intersect with any VMA;
2071 * - is contained within the [low_limit, high_limit) interval;
2072 * - is at least the desired size.
2073 * - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
2074 */
2075static inline unsigned long
2076vm_unmapped_area(struct vm_unmapped_area_info *info)
2077{
2078 if (info->flags & VM_UNMAPPED_AREA_TOPDOWN)
2079 return unmapped_area_topdown(info);
2080 else
2081 return unmapped_area(info);
2082}
2083
2084/* truncate.c */
2085extern void truncate_inode_pages(struct address_space *, loff_t);
2086extern void truncate_inode_pages_range(struct address_space *,
2087 loff_t lstart, loff_t lend);
2088extern void truncate_inode_pages_final(struct address_space *);
2089
2090/* generic vm_area_ops exported for stackable file systems */
2091extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
2092extern void filemap_map_pages(struct vm_fault *vmf,
2093 pgoff_t start_pgoff, pgoff_t end_pgoff);
2094extern int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf);
2095
2096/* mm/page-writeback.c */
2097int write_one_page(struct page *page, int wait);
2098void task_dirty_inc(struct task_struct *tsk);
2099
2100/* readahead.c */
2101#define VM_MAX_READAHEAD 128 /* kbytes */
2102#define VM_MIN_READAHEAD 16 /* kbytes (includes current page) */
2103
2104int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
2105 pgoff_t offset, unsigned long nr_to_read);
2106
2107void page_cache_sync_readahead(struct address_space *mapping,
2108 struct file_ra_state *ra,
2109 struct file *filp,
2110 pgoff_t offset,
2111 unsigned long size);
2112
2113void page_cache_async_readahead(struct address_space *mapping,
2114 struct file_ra_state *ra,
2115 struct file *filp,
2116 struct page *pg,
2117 pgoff_t offset,
2118 unsigned long size);
2119
2120/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
2121extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
2122
2123/* CONFIG_STACK_GROWSUP still needs to to grow downwards at some places */
2124extern int expand_downwards(struct vm_area_struct *vma,
2125 unsigned long address);
2126#if VM_GROWSUP
2127extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
2128#else
2129 #define expand_upwards(vma, address) (0)
2130#endif
2131
2132/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
2133extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
2134extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
2135 struct vm_area_struct **pprev);
2136
2137/* Look up the first VMA which intersects the interval start_addr..end_addr-1,
2138 NULL if none. Assume start_addr < end_addr. */
2139static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
2140{
2141 struct vm_area_struct * vma = find_vma(mm,start_addr);
2142
2143 if (vma && end_addr <= vma->vm_start)
2144 vma = NULL;
2145 return vma;
2146}
2147
2148static inline unsigned long vma_pages(struct vm_area_struct *vma)
2149{
2150 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
2151}
2152
2153/* Look up the first VMA which exactly match the interval vm_start ... vm_end */
2154static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
2155 unsigned long vm_start, unsigned long vm_end)
2156{
2157 struct vm_area_struct *vma = find_vma(mm, vm_start);
2158
2159 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
2160 vma = NULL;
2161
2162 return vma;
2163}
2164
2165#ifdef CONFIG_MMU
2166pgprot_t vm_get_page_prot(unsigned long vm_flags);
2167void vma_set_page_prot(struct vm_area_struct *vma);
2168#else
2169static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
2170{
2171 return __pgprot(0);
2172}
2173static inline void vma_set_page_prot(struct vm_area_struct *vma)
2174{
2175 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
2176}
2177#endif
2178
2179#ifdef CONFIG_NUMA_BALANCING
2180unsigned long change_prot_numa(struct vm_area_struct *vma,
2181 unsigned long start, unsigned long end);
2182#endif
2183
2184struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
2185int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
2186 unsigned long pfn, unsigned long size, pgprot_t);
2187int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
2188int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2189 unsigned long pfn);
2190int vm_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
2191 unsigned long pfn, pgprot_t pgprot);
2192int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
2193 pfn_t pfn);
2194int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);
2195
2196
2197struct page *follow_page_mask(struct vm_area_struct *vma,
2198 unsigned long address, unsigned int foll_flags,
2199 unsigned int *page_mask);
2200
2201static inline struct page *follow_page(struct vm_area_struct *vma,
2202 unsigned long address, unsigned int foll_flags)
2203{
2204 unsigned int unused_page_mask;
2205 return follow_page_mask(vma, address, foll_flags, &unused_page_mask);
2206}
2207
2208#define FOLL_WRITE 0x01 /* check pte is writable */
2209#define FOLL_TOUCH 0x02 /* mark page accessed */
2210#define FOLL_GET 0x04 /* do get_page on page */
2211#define FOLL_DUMP 0x08 /* give error on hole if it would be zero */
2212#define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */
2213#define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO
2214 * and return without waiting upon it */
2215#define FOLL_POPULATE 0x40 /* fault in page */
2216#define FOLL_SPLIT 0x80 /* don't return transhuge pages, split them */
2217#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
2218#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
2219#define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
2220#define FOLL_TRIED 0x800 /* a retry, previous pass started an IO */
2221#define FOLL_MLOCK 0x1000 /* lock present pages */
2222#define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */
2223#define FOLL_COW 0x4000 /* internal GUP flag */
2224
2225typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
2226 void *data);
2227extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
2228 unsigned long size, pte_fn_t fn, void *data);
2229
2230
2231#ifdef CONFIG_PAGE_POISONING
2232extern bool page_poisoning_enabled(void);
2233extern void kernel_poison_pages(struct page *page, int numpages, int enable);
2234extern bool page_is_poisoned(struct page *page);
2235#else
2236static inline bool page_poisoning_enabled(void) { return false; }
2237static inline void kernel_poison_pages(struct page *page, int numpages,
2238 int enable) { }
2239static inline bool page_is_poisoned(struct page *page) { return false; }
2240#endif
2241
2242#ifdef CONFIG_DEBUG_PAGEALLOC
2243extern bool _debug_pagealloc_enabled;
2244extern void __kernel_map_pages(struct page *page, int numpages, int enable);
2245
2246static inline bool debug_pagealloc_enabled(void)
2247{
2248 return _debug_pagealloc_enabled;
2249}
2250
2251static inline void
2252kernel_map_pages(struct page *page, int numpages, int enable)
2253{
2254 if (!debug_pagealloc_enabled())
2255 return;
2256
2257 __kernel_map_pages(page, numpages, enable);
2258}
2259#ifdef CONFIG_HIBERNATION
2260extern bool kernel_page_present(struct page *page);
2261#endif /* CONFIG_HIBERNATION */
2262#else /* CONFIG_DEBUG_PAGEALLOC */
2263static inline void
2264kernel_map_pages(struct page *page, int numpages, int enable) {}
2265#ifdef CONFIG_HIBERNATION
2266static inline bool kernel_page_present(struct page *page) { return true; }
2267#endif /* CONFIG_HIBERNATION */
2268static inline bool debug_pagealloc_enabled(void)
2269{
2270 return false;
2271}
2272#endif /* CONFIG_DEBUG_PAGEALLOC */
2273
2274#ifdef __HAVE_ARCH_GATE_AREA
2275extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);
2276extern int in_gate_area_no_mm(unsigned long addr);
2277extern int in_gate_area(struct mm_struct *mm, unsigned long addr);
2278#else
2279static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
2280{
2281 return NULL;
2282}
2283static inline int in_gate_area_no_mm(unsigned long addr) { return 0; }
2284static inline int in_gate_area(struct mm_struct *mm, unsigned long addr)
2285{
2286 return 0;
2287}
2288#endif /* __HAVE_ARCH_GATE_AREA */
2289
2290extern bool process_shares_mm(struct task_struct *p, struct mm_struct *mm);
2291
2292#ifdef CONFIG_SYSCTL
2293extern int sysctl_drop_caches;
2294int drop_caches_sysctl_handler(struct ctl_table *, int,
2295 void __user *, size_t *, loff_t *);
2296#endif
2297
2298void drop_slab(void);
2299void drop_slab_node(int nid);
2300
2301#ifndef CONFIG_MMU
2302#define randomize_va_space 0
2303#else
2304extern int randomize_va_space;
2305#endif
2306
2307const char * arch_vma_name(struct vm_area_struct *vma);
2308void print_vma_addr(char *prefix, unsigned long rip);
2309
2310void sparse_mem_maps_populate_node(struct page **map_map,
2311 unsigned long pnum_begin,
2312 unsigned long pnum_end,
2313 unsigned long map_count,
2314 int nodeid);
2315
2316struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
2317pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
2318pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
2319pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
2320pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
2321void *vmemmap_alloc_block(unsigned long size, int node);
2322struct vmem_altmap;
2323void *__vmemmap_alloc_block_buf(unsigned long size, int node,
2324 struct vmem_altmap *altmap);
2325static inline void *vmemmap_alloc_block_buf(unsigned long size, int node)
2326{
2327 return __vmemmap_alloc_block_buf(size, node, NULL);
2328}
2329
2330void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
2331int vmemmap_populate_basepages(unsigned long start, unsigned long end,
2332 int node);
2333int vmemmap_populate(unsigned long start, unsigned long end, int node);
2334void vmemmap_populate_print_last(void);
2335#ifdef CONFIG_MEMORY_HOTPLUG
2336void vmemmap_free(unsigned long start, unsigned long end);
2337#endif
2338void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
2339 unsigned long size);
2340
2341enum mf_flags {
2342 MF_COUNT_INCREASED = 1 << 0,
2343 MF_ACTION_REQUIRED = 1 << 1,
2344 MF_MUST_KILL = 1 << 2,
2345 MF_SOFT_OFFLINE = 1 << 3,
2346};
2347extern int memory_failure(unsigned long pfn, int trapno, int flags);
2348extern void memory_failure_queue(unsigned long pfn, int trapno, int flags);
2349extern int unpoison_memory(unsigned long pfn);
2350extern int get_hwpoison_page(struct page *page);
2351#define put_hwpoison_page(page) put_page(page)
2352extern int sysctl_memory_failure_early_kill;
2353extern int sysctl_memory_failure_recovery;
2354extern void shake_page(struct page *p, int access);
2355extern atomic_long_t num_poisoned_pages;
2356extern int soft_offline_page(struct page *page, int flags);
2357
2358
2359/*
2360 * Error handlers for various types of pages.
2361 */
2362enum mf_result {
2363 MF_IGNORED, /* Error: cannot be handled */
2364 MF_FAILED, /* Error: handling failed */
2365 MF_DELAYED, /* Will be handled later */
2366 MF_RECOVERED, /* Successfully recovered */
2367};
2368
2369enum mf_action_page_type {
2370 MF_MSG_KERNEL,
2371 MF_MSG_KERNEL_HIGH_ORDER,
2372 MF_MSG_SLAB,
2373 MF_MSG_DIFFERENT_COMPOUND,
2374 MF_MSG_POISONED_HUGE,
2375 MF_MSG_HUGE,
2376 MF_MSG_FREE_HUGE,
2377 MF_MSG_UNMAP_FAILED,
2378 MF_MSG_DIRTY_SWAPCACHE,
2379 MF_MSG_CLEAN_SWAPCACHE,
2380 MF_MSG_DIRTY_MLOCKED_LRU,
2381 MF_MSG_CLEAN_MLOCKED_LRU,
2382 MF_MSG_DIRTY_UNEVICTABLE_LRU,
2383 MF_MSG_CLEAN_UNEVICTABLE_LRU,
2384 MF_MSG_DIRTY_LRU,
2385 MF_MSG_CLEAN_LRU,
2386 MF_MSG_TRUNCATED_LRU,
2387 MF_MSG_BUDDY,
2388 MF_MSG_BUDDY_2ND,
2389 MF_MSG_UNKNOWN,
2390};
2391
2392#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
2393extern void clear_huge_page(struct page *page,
2394 unsigned long addr,
2395 unsigned int pages_per_huge_page);
2396extern void copy_user_huge_page(struct page *dst, struct page *src,
2397 unsigned long addr, struct vm_area_struct *vma,
2398 unsigned int pages_per_huge_page);
2399#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
2400
2401extern struct page_ext_operations debug_guardpage_ops;
2402extern struct page_ext_operations page_poisoning_ops;
2403
2404#ifdef CONFIG_DEBUG_PAGEALLOC
2405extern unsigned int _debug_guardpage_minorder;
2406extern bool _debug_guardpage_enabled;
2407
2408static inline unsigned int debug_guardpage_minorder(void)
2409{
2410 return _debug_guardpage_minorder;
2411}
2412
2413static inline bool debug_guardpage_enabled(void)
2414{
2415 return _debug_guardpage_enabled;
2416}
2417
2418static inline bool page_is_guard(struct page *page)
2419{
2420 struct page_ext *page_ext;
2421
2422 if (!debug_guardpage_enabled())
2423 return false;
2424
2425 page_ext = lookup_page_ext(page);
2426 if (unlikely(!page_ext))
2427 return false;
2428
2429 return test_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
2430}
2431#else
2432static inline unsigned int debug_guardpage_minorder(void) { return 0; }
2433static inline bool debug_guardpage_enabled(void) { return false; }
2434static inline bool page_is_guard(struct page *page) { return false; }
2435#endif /* CONFIG_DEBUG_PAGEALLOC */
2436
2437#if MAX_NUMNODES > 1
2438void __init setup_nr_node_ids(void);
2439#else
2440static inline void setup_nr_node_ids(void) {}
2441#endif
2442
2443#endif /* __KERNEL__ */
2444#endif /* _LINUX_MM_H */
1#ifndef _LINUX_MM_H
2#define _LINUX_MM_H
3
4#include <linux/errno.h>
5
6#ifdef __KERNEL__
7
8#include <linux/mmdebug.h>
9#include <linux/gfp.h>
10#include <linux/bug.h>
11#include <linux/list.h>
12#include <linux/mmzone.h>
13#include <linux/rbtree.h>
14#include <linux/atomic.h>
15#include <linux/debug_locks.h>
16#include <linux/mm_types.h>
17#include <linux/range.h>
18#include <linux/pfn.h>
19#include <linux/bit_spinlock.h>
20#include <linux/shrinker.h>
21
22struct mempolicy;
23struct anon_vma;
24struct anon_vma_chain;
25struct file_ra_state;
26struct user_struct;
27struct writeback_control;
28
29#ifndef CONFIG_NEED_MULTIPLE_NODES /* Don't use mapnrs, do it properly */
30extern unsigned long max_mapnr;
31
32static inline void set_max_mapnr(unsigned long limit)
33{
34 max_mapnr = limit;
35}
36#else
37static inline void set_max_mapnr(unsigned long limit) { }
38#endif
39
40extern unsigned long totalram_pages;
41extern void * high_memory;
42extern int page_cluster;
43
44#ifdef CONFIG_SYSCTL
45extern int sysctl_legacy_va_layout;
46#else
47#define sysctl_legacy_va_layout 0
48#endif
49
50#include <asm/page.h>
51#include <asm/pgtable.h>
52#include <asm/processor.h>
53
54#ifndef __pa_symbol
55#define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0))
56#endif
57
58extern unsigned long sysctl_user_reserve_kbytes;
59extern unsigned long sysctl_admin_reserve_kbytes;
60
61extern int sysctl_overcommit_memory;
62extern int sysctl_overcommit_ratio;
63extern unsigned long sysctl_overcommit_kbytes;
64
65extern int overcommit_ratio_handler(struct ctl_table *, int, void __user *,
66 size_t *, loff_t *);
67extern int overcommit_kbytes_handler(struct ctl_table *, int, void __user *,
68 size_t *, loff_t *);
69
70#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
71
72/* to align the pointer to the (next) page boundary */
73#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
74
75/* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */
76#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)addr, PAGE_SIZE)
77
78/*
79 * Linux kernel virtual memory manager primitives.
80 * The idea being to have a "virtual" mm in the same way
81 * we have a virtual fs - giving a cleaner interface to the
82 * mm details, and allowing different kinds of memory mappings
83 * (from shared memory to executable loading to arbitrary
84 * mmap() functions).
85 */
86
87extern struct kmem_cache *vm_area_cachep;
88
89#ifndef CONFIG_MMU
90extern struct rb_root nommu_region_tree;
91extern struct rw_semaphore nommu_region_sem;
92
93extern unsigned int kobjsize(const void *objp);
94#endif
95
96/*
97 * vm_flags in vm_area_struct, see mm_types.h.
98 */
99#define VM_NONE 0x00000000
100
101#define VM_READ 0x00000001 /* currently active flags */
102#define VM_WRITE 0x00000002
103#define VM_EXEC 0x00000004
104#define VM_SHARED 0x00000008
105
106/* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */
107#define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */
108#define VM_MAYWRITE 0x00000020
109#define VM_MAYEXEC 0x00000040
110#define VM_MAYSHARE 0x00000080
111
112#define VM_GROWSDOWN 0x00000100 /* general info on the segment */
113#define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
114#define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */
115
116#define VM_LOCKED 0x00002000
117#define VM_IO 0x00004000 /* Memory mapped I/O or similar */
118
119 /* Used by sys_madvise() */
120#define VM_SEQ_READ 0x00008000 /* App will access data sequentially */
121#define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */
122
123#define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */
124#define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */
125#define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */
126#define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */
127#define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
128#define VM_NONLINEAR 0x00800000 /* Is non-linear (remap_file_pages) */
129#define VM_ARCH_1 0x01000000 /* Architecture-specific flag */
130#define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */
131
132#ifdef CONFIG_MEM_SOFT_DIRTY
133# define VM_SOFTDIRTY 0x08000000 /* Not soft dirty clean area */
134#else
135# define VM_SOFTDIRTY 0
136#endif
137
138#define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
139#define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */
140#define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */
141#define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */
142
143#if defined(CONFIG_X86)
144# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
145#elif defined(CONFIG_PPC)
146# define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */
147#elif defined(CONFIG_PARISC)
148# define VM_GROWSUP VM_ARCH_1
149#elif defined(CONFIG_METAG)
150# define VM_GROWSUP VM_ARCH_1
151#elif defined(CONFIG_IA64)
152# define VM_GROWSUP VM_ARCH_1
153#elif !defined(CONFIG_MMU)
154# define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */
155#endif
156
157#ifndef VM_GROWSUP
158# define VM_GROWSUP VM_NONE
159#endif
160
161/* Bits set in the VMA until the stack is in its final location */
162#define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ)
163
164#ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */
165#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
166#endif
167
168#ifdef CONFIG_STACK_GROWSUP
169#define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
170#else
171#define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
172#endif
173
174/*
175 * Special vmas that are non-mergable, non-mlock()able.
176 * Note: mm/huge_memory.c VM_NO_THP depends on this definition.
177 */
178#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP)
179
180/* This mask defines which mm->def_flags a process can inherit its parent */
181#define VM_INIT_DEF_MASK VM_NOHUGEPAGE
182
183/*
184 * mapping from the currently active vm_flags protection bits (the
185 * low four bits) to a page protection mask..
186 */
187extern pgprot_t protection_map[16];
188
189#define FAULT_FLAG_WRITE 0x01 /* Fault was a write access */
190#define FAULT_FLAG_NONLINEAR 0x02 /* Fault was via a nonlinear mapping */
191#define FAULT_FLAG_MKWRITE 0x04 /* Fault was mkwrite of existing pte */
192#define FAULT_FLAG_ALLOW_RETRY 0x08 /* Retry fault if blocking */
193#define FAULT_FLAG_RETRY_NOWAIT 0x10 /* Don't drop mmap_sem and wait when retrying */
194#define FAULT_FLAG_KILLABLE 0x20 /* The fault task is in SIGKILL killable region */
195#define FAULT_FLAG_TRIED 0x40 /* second try */
196#define FAULT_FLAG_USER 0x80 /* The fault originated in userspace */
197
198/*
199 * vm_fault is filled by the the pagefault handler and passed to the vma's
200 * ->fault function. The vma's ->fault is responsible for returning a bitmask
201 * of VM_FAULT_xxx flags that give details about how the fault was handled.
202 *
203 * pgoff should be used in favour of virtual_address, if possible. If pgoff
204 * is used, one may implement ->remap_pages to get nonlinear mapping support.
205 */
206struct vm_fault {
207 unsigned int flags; /* FAULT_FLAG_xxx flags */
208 pgoff_t pgoff; /* Logical page offset based on vma */
209 void __user *virtual_address; /* Faulting virtual address */
210
211 struct page *page; /* ->fault handlers should return a
212 * page here, unless VM_FAULT_NOPAGE
213 * is set (which is also implied by
214 * VM_FAULT_ERROR).
215 */
216 /* for ->map_pages() only */
217 pgoff_t max_pgoff; /* map pages for offset from pgoff till
218 * max_pgoff inclusive */
219 pte_t *pte; /* pte entry associated with ->pgoff */
220};
221
222/*
223 * These are the virtual MM functions - opening of an area, closing and
224 * unmapping it (needed to keep files on disk up-to-date etc), pointer
225 * to the functions called when a no-page or a wp-page exception occurs.
226 */
227struct vm_operations_struct {
228 void (*open)(struct vm_area_struct * area);
229 void (*close)(struct vm_area_struct * area);
230 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
231 void (*map_pages)(struct vm_area_struct *vma, struct vm_fault *vmf);
232
233 /* notification that a previously read-only page is about to become
234 * writable, if an error is returned it will cause a SIGBUS */
235 int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
236
237 /* called by access_process_vm when get_user_pages() fails, typically
238 * for use by special VMAs that can switch between memory and hardware
239 */
240 int (*access)(struct vm_area_struct *vma, unsigned long addr,
241 void *buf, int len, int write);
242#ifdef CONFIG_NUMA
243 /*
244 * set_policy() op must add a reference to any non-NULL @new mempolicy
245 * to hold the policy upon return. Caller should pass NULL @new to
246 * remove a policy and fall back to surrounding context--i.e. do not
247 * install a MPOL_DEFAULT policy, nor the task or system default
248 * mempolicy.
249 */
250 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
251
252 /*
253 * get_policy() op must add reference [mpol_get()] to any policy at
254 * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure
255 * in mm/mempolicy.c will do this automatically.
256 * get_policy() must NOT add a ref if the policy at (vma,addr) is not
257 * marked as MPOL_SHARED. vma policies are protected by the mmap_sem.
258 * If no [shared/vma] mempolicy exists at the addr, get_policy() op
259 * must return NULL--i.e., do not "fallback" to task or system default
260 * policy.
261 */
262 struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
263 unsigned long addr);
264 int (*migrate)(struct vm_area_struct *vma, const nodemask_t *from,
265 const nodemask_t *to, unsigned long flags);
266#endif
267 /* called by sys_remap_file_pages() to populate non-linear mapping */
268 int (*remap_pages)(struct vm_area_struct *vma, unsigned long addr,
269 unsigned long size, pgoff_t pgoff);
270};
271
272struct mmu_gather;
273struct inode;
274
275#define page_private(page) ((page)->private)
276#define set_page_private(page, v) ((page)->private = (v))
277
278/* It's valid only if the page is free path or free_list */
279static inline void set_freepage_migratetype(struct page *page, int migratetype)
280{
281 page->index = migratetype;
282}
283
284/* It's valid only if the page is free path or free_list */
285static inline int get_freepage_migratetype(struct page *page)
286{
287 return page->index;
288}
289
290/*
291 * FIXME: take this include out, include page-flags.h in
292 * files which need it (119 of them)
293 */
294#include <linux/page-flags.h>
295#include <linux/huge_mm.h>
296
297/*
298 * Methods to modify the page usage count.
299 *
300 * What counts for a page usage:
301 * - cache mapping (page->mapping)
302 * - private data (page->private)
303 * - page mapped in a task's page tables, each mapping
304 * is counted separately
305 *
306 * Also, many kernel routines increase the page count before a critical
307 * routine so they can be sure the page doesn't go away from under them.
308 */
309
310/*
311 * Drop a ref, return true if the refcount fell to zero (the page has no users)
312 */
313static inline int put_page_testzero(struct page *page)
314{
315 VM_BUG_ON_PAGE(atomic_read(&page->_count) == 0, page);
316 return atomic_dec_and_test(&page->_count);
317}
318
319/*
320 * Try to grab a ref unless the page has a refcount of zero, return false if
321 * that is the case.
322 * This can be called when MMU is off so it must not access
323 * any of the virtual mappings.
324 */
325static inline int get_page_unless_zero(struct page *page)
326{
327 return atomic_inc_not_zero(&page->_count);
328}
329
330/*
331 * Try to drop a ref unless the page has a refcount of one, return false if
332 * that is the case.
333 * This is to make sure that the refcount won't become zero after this drop.
334 * This can be called when MMU is off so it must not access
335 * any of the virtual mappings.
336 */
337static inline int put_page_unless_one(struct page *page)
338{
339 return atomic_add_unless(&page->_count, -1, 1);
340}
341
342extern int page_is_ram(unsigned long pfn);
343
344/* Support for virtually mapped pages */
345struct page *vmalloc_to_page(const void *addr);
346unsigned long vmalloc_to_pfn(const void *addr);
347
348/*
349 * Determine if an address is within the vmalloc range
350 *
351 * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there
352 * is no special casing required.
353 */
354static inline int is_vmalloc_addr(const void *x)
355{
356#ifdef CONFIG_MMU
357 unsigned long addr = (unsigned long)x;
358
359 return addr >= VMALLOC_START && addr < VMALLOC_END;
360#else
361 return 0;
362#endif
363}
364#ifdef CONFIG_MMU
365extern int is_vmalloc_or_module_addr(const void *x);
366#else
367static inline int is_vmalloc_or_module_addr(const void *x)
368{
369 return 0;
370}
371#endif
372
373extern void kvfree(const void *addr);
374
375static inline void compound_lock(struct page *page)
376{
377#ifdef CONFIG_TRANSPARENT_HUGEPAGE
378 VM_BUG_ON_PAGE(PageSlab(page), page);
379 bit_spin_lock(PG_compound_lock, &page->flags);
380#endif
381}
382
383static inline void compound_unlock(struct page *page)
384{
385#ifdef CONFIG_TRANSPARENT_HUGEPAGE
386 VM_BUG_ON_PAGE(PageSlab(page), page);
387 bit_spin_unlock(PG_compound_lock, &page->flags);
388#endif
389}
390
391static inline unsigned long compound_lock_irqsave(struct page *page)
392{
393 unsigned long uninitialized_var(flags);
394#ifdef CONFIG_TRANSPARENT_HUGEPAGE
395 local_irq_save(flags);
396 compound_lock(page);
397#endif
398 return flags;
399}
400
401static inline void compound_unlock_irqrestore(struct page *page,
402 unsigned long flags)
403{
404#ifdef CONFIG_TRANSPARENT_HUGEPAGE
405 compound_unlock(page);
406 local_irq_restore(flags);
407#endif
408}
409
410static inline struct page *compound_head(struct page *page)
411{
412 if (unlikely(PageTail(page))) {
413 struct page *head = page->first_page;
414
415 /*
416 * page->first_page may be a dangling pointer to an old
417 * compound page, so recheck that it is still a tail
418 * page before returning.
419 */
420 smp_rmb();
421 if (likely(PageTail(page)))
422 return head;
423 }
424 return page;
425}
426
427/*
428 * The atomic page->_mapcount, starts from -1: so that transitions
429 * both from it and to it can be tracked, using atomic_inc_and_test
430 * and atomic_add_negative(-1).
431 */
432static inline void page_mapcount_reset(struct page *page)
433{
434 atomic_set(&(page)->_mapcount, -1);
435}
436
437static inline int page_mapcount(struct page *page)
438{
439 return atomic_read(&(page)->_mapcount) + 1;
440}
441
442static inline int page_count(struct page *page)
443{
444 return atomic_read(&compound_head(page)->_count);
445}
446
447#ifdef CONFIG_HUGETLB_PAGE
448extern int PageHeadHuge(struct page *page_head);
449#else /* CONFIG_HUGETLB_PAGE */
450static inline int PageHeadHuge(struct page *page_head)
451{
452 return 0;
453}
454#endif /* CONFIG_HUGETLB_PAGE */
455
456static inline bool __compound_tail_refcounted(struct page *page)
457{
458 return !PageSlab(page) && !PageHeadHuge(page);
459}
460
461/*
462 * This takes a head page as parameter and tells if the
463 * tail page reference counting can be skipped.
464 *
465 * For this to be safe, PageSlab and PageHeadHuge must remain true on
466 * any given page where they return true here, until all tail pins
467 * have been released.
468 */
469static inline bool compound_tail_refcounted(struct page *page)
470{
471 VM_BUG_ON_PAGE(!PageHead(page), page);
472 return __compound_tail_refcounted(page);
473}
474
475static inline void get_huge_page_tail(struct page *page)
476{
477 /*
478 * __split_huge_page_refcount() cannot run from under us.
479 */
480 VM_BUG_ON_PAGE(!PageTail(page), page);
481 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
482 VM_BUG_ON_PAGE(atomic_read(&page->_count) != 0, page);
483 if (compound_tail_refcounted(page->first_page))
484 atomic_inc(&page->_mapcount);
485}
486
487extern bool __get_page_tail(struct page *page);
488
489static inline void get_page(struct page *page)
490{
491 if (unlikely(PageTail(page)))
492 if (likely(__get_page_tail(page)))
493 return;
494 /*
495 * Getting a normal page or the head of a compound page
496 * requires to already have an elevated page->_count.
497 */
498 VM_BUG_ON_PAGE(atomic_read(&page->_count) <= 0, page);
499 atomic_inc(&page->_count);
500}
501
502static inline struct page *virt_to_head_page(const void *x)
503{
504 struct page *page = virt_to_page(x);
505 return compound_head(page);
506}
507
508/*
509 * Setup the page count before being freed into the page allocator for
510 * the first time (boot or memory hotplug)
511 */
512static inline void init_page_count(struct page *page)
513{
514 atomic_set(&page->_count, 1);
515}
516
517/*
518 * PageBuddy() indicate that the page is free and in the buddy system
519 * (see mm/page_alloc.c).
520 *
521 * PAGE_BUDDY_MAPCOUNT_VALUE must be <= -2 but better not too close to
522 * -2 so that an underflow of the page_mapcount() won't be mistaken
523 * for a genuine PAGE_BUDDY_MAPCOUNT_VALUE. -128 can be created very
524 * efficiently by most CPU architectures.
525 */
526#define PAGE_BUDDY_MAPCOUNT_VALUE (-128)
527
528static inline int PageBuddy(struct page *page)
529{
530 return atomic_read(&page->_mapcount) == PAGE_BUDDY_MAPCOUNT_VALUE;
531}
532
533static inline void __SetPageBuddy(struct page *page)
534{
535 VM_BUG_ON_PAGE(atomic_read(&page->_mapcount) != -1, page);
536 atomic_set(&page->_mapcount, PAGE_BUDDY_MAPCOUNT_VALUE);
537}
538
539static inline void __ClearPageBuddy(struct page *page)
540{
541 VM_BUG_ON_PAGE(!PageBuddy(page), page);
542 atomic_set(&page->_mapcount, -1);
543}
544
545void put_page(struct page *page);
546void put_pages_list(struct list_head *pages);
547
548void split_page(struct page *page, unsigned int order);
549int split_free_page(struct page *page);
550
551/*
552 * Compound pages have a destructor function. Provide a
553 * prototype for that function and accessor functions.
554 * These are _only_ valid on the head of a PG_compound page.
555 */
556typedef void compound_page_dtor(struct page *);
557
558static inline void set_compound_page_dtor(struct page *page,
559 compound_page_dtor *dtor)
560{
561 page[1].lru.next = (void *)dtor;
562}
563
564static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
565{
566 return (compound_page_dtor *)page[1].lru.next;
567}
568
569static inline int compound_order(struct page *page)
570{
571 if (!PageHead(page))
572 return 0;
573 return (unsigned long)page[1].lru.prev;
574}
575
576static inline void set_compound_order(struct page *page, unsigned long order)
577{
578 page[1].lru.prev = (void *)order;
579}
580
581#ifdef CONFIG_MMU
582/*
583 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
584 * servicing faults for write access. In the normal case, do always want
585 * pte_mkwrite. But get_user_pages can cause write faults for mappings
586 * that do not have writing enabled, when used by access_process_vm.
587 */
588static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
589{
590 if (likely(vma->vm_flags & VM_WRITE))
591 pte = pte_mkwrite(pte);
592 return pte;
593}
594
595void do_set_pte(struct vm_area_struct *vma, unsigned long address,
596 struct page *page, pte_t *pte, bool write, bool anon);
597#endif
598
599/*
600 * Multiple processes may "see" the same page. E.g. for untouched
601 * mappings of /dev/null, all processes see the same page full of
602 * zeroes, and text pages of executables and shared libraries have
603 * only one copy in memory, at most, normally.
604 *
605 * For the non-reserved pages, page_count(page) denotes a reference count.
606 * page_count() == 0 means the page is free. page->lru is then used for
607 * freelist management in the buddy allocator.
608 * page_count() > 0 means the page has been allocated.
609 *
610 * Pages are allocated by the slab allocator in order to provide memory
611 * to kmalloc and kmem_cache_alloc. In this case, the management of the
612 * page, and the fields in 'struct page' are the responsibility of mm/slab.c
613 * unless a particular usage is carefully commented. (the responsibility of
614 * freeing the kmalloc memory is the caller's, of course).
615 *
616 * A page may be used by anyone else who does a __get_free_page().
617 * In this case, page_count still tracks the references, and should only
618 * be used through the normal accessor functions. The top bits of page->flags
619 * and page->virtual store page management information, but all other fields
620 * are unused and could be used privately, carefully. The management of this
621 * page is the responsibility of the one who allocated it, and those who have
622 * subsequently been given references to it.
623 *
624 * The other pages (we may call them "pagecache pages") are completely
625 * managed by the Linux memory manager: I/O, buffers, swapping etc.
626 * The following discussion applies only to them.
627 *
628 * A pagecache page contains an opaque `private' member, which belongs to the
629 * page's address_space. Usually, this is the address of a circular list of
630 * the page's disk buffers. PG_private must be set to tell the VM to call
631 * into the filesystem to release these pages.
632 *
633 * A page may belong to an inode's memory mapping. In this case, page->mapping
634 * is the pointer to the inode, and page->index is the file offset of the page,
635 * in units of PAGE_CACHE_SIZE.
636 *
637 * If pagecache pages are not associated with an inode, they are said to be
638 * anonymous pages. These may become associated with the swapcache, and in that
639 * case PG_swapcache is set, and page->private is an offset into the swapcache.
640 *
641 * In either case (swapcache or inode backed), the pagecache itself holds one
642 * reference to the page. Setting PG_private should also increment the
643 * refcount. The each user mapping also has a reference to the page.
644 *
645 * The pagecache pages are stored in a per-mapping radix tree, which is
646 * rooted at mapping->page_tree, and indexed by offset.
647 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space
648 * lists, we instead now tag pages as dirty/writeback in the radix tree.
649 *
650 * All pagecache pages may be subject to I/O:
651 * - inode pages may need to be read from disk,
652 * - inode pages which have been modified and are MAP_SHARED may need
653 * to be written back to the inode on disk,
654 * - anonymous pages (including MAP_PRIVATE file mappings) which have been
655 * modified may need to be swapped out to swap space and (later) to be read
656 * back into memory.
657 */
658
659/*
660 * The zone field is never updated after free_area_init_core()
661 * sets it, so none of the operations on it need to be atomic.
662 */
663
664/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */
665#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
666#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
667#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
668#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)
669
670/*
671 * Define the bit shifts to access each section. For non-existent
672 * sections we define the shift as 0; that plus a 0 mask ensures
673 * the compiler will optimise away reference to them.
674 */
675#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
676#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
677#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
678#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))
679
680/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
681#ifdef NODE_NOT_IN_PAGE_FLAGS
682#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
683#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
684 SECTIONS_PGOFF : ZONES_PGOFF)
685#else
686#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
687#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
688 NODES_PGOFF : ZONES_PGOFF)
689#endif
690
691#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
692
693#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
694#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
695#endif
696
697#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
698#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
699#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
700#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_SHIFT) - 1)
701#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
702
703static inline enum zone_type page_zonenum(const struct page *page)
704{
705 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
706}
707
708#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
709#define SECTION_IN_PAGE_FLAGS
710#endif
711
712/*
713 * The identification function is mainly used by the buddy allocator for
714 * determining if two pages could be buddies. We are not really identifying
715 * the zone since we could be using the section number id if we do not have
716 * node id available in page flags.
717 * We only guarantee that it will return the same value for two combinable
718 * pages in a zone.
719 */
720static inline int page_zone_id(struct page *page)
721{
722 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
723}
724
725static inline int zone_to_nid(struct zone *zone)
726{
727#ifdef CONFIG_NUMA
728 return zone->node;
729#else
730 return 0;
731#endif
732}
733
734#ifdef NODE_NOT_IN_PAGE_FLAGS
735extern int page_to_nid(const struct page *page);
736#else
737static inline int page_to_nid(const struct page *page)
738{
739 return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
740}
741#endif
742
743#ifdef CONFIG_NUMA_BALANCING
744static inline int cpu_pid_to_cpupid(int cpu, int pid)
745{
746 return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
747}
748
749static inline int cpupid_to_pid(int cpupid)
750{
751 return cpupid & LAST__PID_MASK;
752}
753
754static inline int cpupid_to_cpu(int cpupid)
755{
756 return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
757}
758
759static inline int cpupid_to_nid(int cpupid)
760{
761 return cpu_to_node(cpupid_to_cpu(cpupid));
762}
763
764static inline bool cpupid_pid_unset(int cpupid)
765{
766 return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
767}
768
769static inline bool cpupid_cpu_unset(int cpupid)
770{
771 return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
772}
773
774static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
775{
776 return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);
777}
778
779#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)
780#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
781static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
782{
783 return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);
784}
785
786static inline int page_cpupid_last(struct page *page)
787{
788 return page->_last_cpupid;
789}
790static inline void page_cpupid_reset_last(struct page *page)
791{
792 page->_last_cpupid = -1 & LAST_CPUPID_MASK;
793}
794#else
795static inline int page_cpupid_last(struct page *page)
796{
797 return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
798}
799
800extern int page_cpupid_xchg_last(struct page *page, int cpupid);
801
802static inline void page_cpupid_reset_last(struct page *page)
803{
804 int cpupid = (1 << LAST_CPUPID_SHIFT) - 1;
805
806 page->flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);
807 page->flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;
808}
809#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
810#else /* !CONFIG_NUMA_BALANCING */
811static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
812{
813 return page_to_nid(page); /* XXX */
814}
815
816static inline int page_cpupid_last(struct page *page)
817{
818 return page_to_nid(page); /* XXX */
819}
820
821static inline int cpupid_to_nid(int cpupid)
822{
823 return -1;
824}
825
826static inline int cpupid_to_pid(int cpupid)
827{
828 return -1;
829}
830
831static inline int cpupid_to_cpu(int cpupid)
832{
833 return -1;
834}
835
836static inline int cpu_pid_to_cpupid(int nid, int pid)
837{
838 return -1;
839}
840
841static inline bool cpupid_pid_unset(int cpupid)
842{
843 return 1;
844}
845
846static inline void page_cpupid_reset_last(struct page *page)
847{
848}
849
850static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
851{
852 return false;
853}
854#endif /* CONFIG_NUMA_BALANCING */
855
856static inline struct zone *page_zone(const struct page *page)
857{
858 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
859}
860
861#ifdef SECTION_IN_PAGE_FLAGS
862static inline void set_page_section(struct page *page, unsigned long section)
863{
864 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
865 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
866}
867
868static inline unsigned long page_to_section(const struct page *page)
869{
870 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
871}
872#endif
873
874static inline void set_page_zone(struct page *page, enum zone_type zone)
875{
876 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
877 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
878}
879
880static inline void set_page_node(struct page *page, unsigned long node)
881{
882 page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
883 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
884}
885
886static inline void set_page_links(struct page *page, enum zone_type zone,
887 unsigned long node, unsigned long pfn)
888{
889 set_page_zone(page, zone);
890 set_page_node(page, node);
891#ifdef SECTION_IN_PAGE_FLAGS
892 set_page_section(page, pfn_to_section_nr(pfn));
893#endif
894}
895
896/*
897 * Some inline functions in vmstat.h depend on page_zone()
898 */
899#include <linux/vmstat.h>
900
901static __always_inline void *lowmem_page_address(const struct page *page)
902{
903 return __va(PFN_PHYS(page_to_pfn(page)));
904}
905
906#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
907#define HASHED_PAGE_VIRTUAL
908#endif
909
910#if defined(WANT_PAGE_VIRTUAL)
911static inline void *page_address(const struct page *page)
912{
913 return page->virtual;
914}
915static inline void set_page_address(struct page *page, void *address)
916{
917 page->virtual = address;
918}
919#define page_address_init() do { } while(0)
920#endif
921
922#if defined(HASHED_PAGE_VIRTUAL)
923void *page_address(const struct page *page);
924void set_page_address(struct page *page, void *virtual);
925void page_address_init(void);
926#endif
927
928#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
929#define page_address(page) lowmem_page_address(page)
930#define set_page_address(page, address) do { } while(0)
931#define page_address_init() do { } while(0)
932#endif
933
934/*
935 * On an anonymous page mapped into a user virtual memory area,
936 * page->mapping points to its anon_vma, not to a struct address_space;
937 * with the PAGE_MAPPING_ANON bit set to distinguish it. See rmap.h.
938 *
939 * On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled,
940 * the PAGE_MAPPING_KSM bit may be set along with the PAGE_MAPPING_ANON bit;
941 * and then page->mapping points, not to an anon_vma, but to a private
942 * structure which KSM associates with that merged page. See ksm.h.
943 *
944 * PAGE_MAPPING_KSM without PAGE_MAPPING_ANON is currently never used.
945 *
946 * Please note that, confusingly, "page_mapping" refers to the inode
947 * address_space which maps the page from disk; whereas "page_mapped"
948 * refers to user virtual address space into which the page is mapped.
949 */
950#define PAGE_MAPPING_ANON 1
951#define PAGE_MAPPING_KSM 2
952#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM)
953
954extern struct address_space *page_mapping(struct page *page);
955
956/* Neutral page->mapping pointer to address_space or anon_vma or other */
957static inline void *page_rmapping(struct page *page)
958{
959 return (void *)((unsigned long)page->mapping & ~PAGE_MAPPING_FLAGS);
960}
961
962extern struct address_space *__page_file_mapping(struct page *);
963
964static inline
965struct address_space *page_file_mapping(struct page *page)
966{
967 if (unlikely(PageSwapCache(page)))
968 return __page_file_mapping(page);
969
970 return page->mapping;
971}
972
973static inline int PageAnon(struct page *page)
974{
975 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
976}
977
978/*
979 * Return the pagecache index of the passed page. Regular pagecache pages
980 * use ->index whereas swapcache pages use ->private
981 */
982static inline pgoff_t page_index(struct page *page)
983{
984 if (unlikely(PageSwapCache(page)))
985 return page_private(page);
986 return page->index;
987}
988
989extern pgoff_t __page_file_index(struct page *page);
990
991/*
992 * Return the file index of the page. Regular pagecache pages use ->index
993 * whereas swapcache pages use swp_offset(->private)
994 */
995static inline pgoff_t page_file_index(struct page *page)
996{
997 if (unlikely(PageSwapCache(page)))
998 return __page_file_index(page);
999
1000 return page->index;
1001}
1002
1003/*
1004 * Return true if this page is mapped into pagetables.
1005 */
1006static inline int page_mapped(struct page *page)
1007{
1008 return atomic_read(&(page)->_mapcount) >= 0;
1009}
1010
1011/*
1012 * Different kinds of faults, as returned by handle_mm_fault().
1013 * Used to decide whether a process gets delivered SIGBUS or
1014 * just gets major/minor fault counters bumped up.
1015 */
1016
1017#define VM_FAULT_MINOR 0 /* For backwards compat. Remove me quickly. */
1018
1019#define VM_FAULT_OOM 0x0001
1020#define VM_FAULT_SIGBUS 0x0002
1021#define VM_FAULT_MAJOR 0x0004
1022#define VM_FAULT_WRITE 0x0008 /* Special case for get_user_pages */
1023#define VM_FAULT_HWPOISON 0x0010 /* Hit poisoned small page */
1024#define VM_FAULT_HWPOISON_LARGE 0x0020 /* Hit poisoned large page. Index encoded in upper bits */
1025
1026#define VM_FAULT_NOPAGE 0x0100 /* ->fault installed the pte, not return page */
1027#define VM_FAULT_LOCKED 0x0200 /* ->fault locked the returned page */
1028#define VM_FAULT_RETRY 0x0400 /* ->fault blocked, must retry */
1029#define VM_FAULT_FALLBACK 0x0800 /* huge page fault failed, fall back to small */
1030
1031#define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */
1032
1033#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_HWPOISON | \
1034 VM_FAULT_FALLBACK | VM_FAULT_HWPOISON_LARGE)
1035
1036/* Encode hstate index for a hwpoisoned large page */
1037#define VM_FAULT_SET_HINDEX(x) ((x) << 12)
1038#define VM_FAULT_GET_HINDEX(x) (((x) >> 12) & 0xf)
1039
1040/*
1041 * Can be called by the pagefault handler when it gets a VM_FAULT_OOM.
1042 */
1043extern void pagefault_out_of_memory(void);
1044
1045#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
1046
1047/*
1048 * Flags passed to show_mem() and show_free_areas() to suppress output in
1049 * various contexts.
1050 */
1051#define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */
1052
1053extern void show_free_areas(unsigned int flags);
1054extern bool skip_free_areas_node(unsigned int flags, int nid);
1055
1056int shmem_zero_setup(struct vm_area_struct *);
1057#ifdef CONFIG_SHMEM
1058bool shmem_mapping(struct address_space *mapping);
1059#else
1060static inline bool shmem_mapping(struct address_space *mapping)
1061{
1062 return false;
1063}
1064#endif
1065
1066extern int can_do_mlock(void);
1067extern int user_shm_lock(size_t, struct user_struct *);
1068extern void user_shm_unlock(size_t, struct user_struct *);
1069
1070/*
1071 * Parameter block passed down to zap_pte_range in exceptional cases.
1072 */
1073struct zap_details {
1074 struct vm_area_struct *nonlinear_vma; /* Check page->index if set */
1075 struct address_space *check_mapping; /* Check page->mapping if set */
1076 pgoff_t first_index; /* Lowest page->index to unmap */
1077 pgoff_t last_index; /* Highest page->index to unmap */
1078};
1079
1080struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1081 pte_t pte);
1082
1083int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
1084 unsigned long size);
1085void zap_page_range(struct vm_area_struct *vma, unsigned long address,
1086 unsigned long size, struct zap_details *);
1087void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
1088 unsigned long start, unsigned long end);
1089
1090/**
1091 * mm_walk - callbacks for walk_page_range
1092 * @pgd_entry: if set, called for each non-empty PGD (top-level) entry
1093 * @pud_entry: if set, called for each non-empty PUD (2nd-level) entry
1094 * @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry
1095 * this handler is required to be able to handle
1096 * pmd_trans_huge() pmds. They may simply choose to
1097 * split_huge_page() instead of handling it explicitly.
1098 * @pte_entry: if set, called for each non-empty PTE (4th-level) entry
1099 * @pte_hole: if set, called for each hole at all levels
1100 * @hugetlb_entry: if set, called for each hugetlb entry
1101 * *Caution*: The caller must hold mmap_sem() if @hugetlb_entry
1102 * is used.
1103 *
1104 * (see walk_page_range for more details)
1105 */
1106struct mm_walk {
1107 int (*pgd_entry)(pgd_t *pgd, unsigned long addr,
1108 unsigned long next, struct mm_walk *walk);
1109 int (*pud_entry)(pud_t *pud, unsigned long addr,
1110 unsigned long next, struct mm_walk *walk);
1111 int (*pmd_entry)(pmd_t *pmd, unsigned long addr,
1112 unsigned long next, struct mm_walk *walk);
1113 int (*pte_entry)(pte_t *pte, unsigned long addr,
1114 unsigned long next, struct mm_walk *walk);
1115 int (*pte_hole)(unsigned long addr, unsigned long next,
1116 struct mm_walk *walk);
1117 int (*hugetlb_entry)(pte_t *pte, unsigned long hmask,
1118 unsigned long addr, unsigned long next,
1119 struct mm_walk *walk);
1120 struct mm_struct *mm;
1121 void *private;
1122};
1123
1124int walk_page_range(unsigned long addr, unsigned long end,
1125 struct mm_walk *walk);
1126void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
1127 unsigned long end, unsigned long floor, unsigned long ceiling);
1128int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
1129 struct vm_area_struct *vma);
1130void unmap_mapping_range(struct address_space *mapping,
1131 loff_t const holebegin, loff_t const holelen, int even_cows);
1132int follow_pfn(struct vm_area_struct *vma, unsigned long address,
1133 unsigned long *pfn);
1134int follow_phys(struct vm_area_struct *vma, unsigned long address,
1135 unsigned int flags, unsigned long *prot, resource_size_t *phys);
1136int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
1137 void *buf, int len, int write);
1138
1139static inline void unmap_shared_mapping_range(struct address_space *mapping,
1140 loff_t const holebegin, loff_t const holelen)
1141{
1142 unmap_mapping_range(mapping, holebegin, holelen, 0);
1143}
1144
1145extern void truncate_pagecache(struct inode *inode, loff_t new);
1146extern void truncate_setsize(struct inode *inode, loff_t newsize);
1147void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
1148int truncate_inode_page(struct address_space *mapping, struct page *page);
1149int generic_error_remove_page(struct address_space *mapping, struct page *page);
1150int invalidate_inode_page(struct page *page);
1151
1152#ifdef CONFIG_MMU
1153extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
1154 unsigned long address, unsigned int flags);
1155extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
1156 unsigned long address, unsigned int fault_flags);
1157#else
1158static inline int handle_mm_fault(struct mm_struct *mm,
1159 struct vm_area_struct *vma, unsigned long address,
1160 unsigned int flags)
1161{
1162 /* should never happen if there's no MMU */
1163 BUG();
1164 return VM_FAULT_SIGBUS;
1165}
1166static inline int fixup_user_fault(struct task_struct *tsk,
1167 struct mm_struct *mm, unsigned long address,
1168 unsigned int fault_flags)
1169{
1170 /* should never happen if there's no MMU */
1171 BUG();
1172 return -EFAULT;
1173}
1174#endif
1175
1176extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
1177extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
1178 void *buf, int len, int write);
1179
1180long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
1181 unsigned long start, unsigned long nr_pages,
1182 unsigned int foll_flags, struct page **pages,
1183 struct vm_area_struct **vmas, int *nonblocking);
1184long get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
1185 unsigned long start, unsigned long nr_pages,
1186 int write, int force, struct page **pages,
1187 struct vm_area_struct **vmas);
1188int get_user_pages_fast(unsigned long start, int nr_pages, int write,
1189 struct page **pages);
1190struct kvec;
1191int get_kernel_pages(const struct kvec *iov, int nr_pages, int write,
1192 struct page **pages);
1193int get_kernel_page(unsigned long start, int write, struct page **pages);
1194struct page *get_dump_page(unsigned long addr);
1195
1196extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
1197extern void do_invalidatepage(struct page *page, unsigned int offset,
1198 unsigned int length);
1199
1200int __set_page_dirty_nobuffers(struct page *page);
1201int __set_page_dirty_no_writeback(struct page *page);
1202int redirty_page_for_writepage(struct writeback_control *wbc,
1203 struct page *page);
1204void account_page_dirtied(struct page *page, struct address_space *mapping);
1205void account_page_writeback(struct page *page);
1206int set_page_dirty(struct page *page);
1207int set_page_dirty_lock(struct page *page);
1208int clear_page_dirty_for_io(struct page *page);
1209int get_cmdline(struct task_struct *task, char *buffer, int buflen);
1210
1211/* Is the vma a continuation of the stack vma above it? */
1212static inline int vma_growsdown(struct vm_area_struct *vma, unsigned long addr)
1213{
1214 return vma && (vma->vm_end == addr) && (vma->vm_flags & VM_GROWSDOWN);
1215}
1216
1217static inline int stack_guard_page_start(struct vm_area_struct *vma,
1218 unsigned long addr)
1219{
1220 return (vma->vm_flags & VM_GROWSDOWN) &&
1221 (vma->vm_start == addr) &&
1222 !vma_growsdown(vma->vm_prev, addr);
1223}
1224
1225/* Is the vma a continuation of the stack vma below it? */
1226static inline int vma_growsup(struct vm_area_struct *vma, unsigned long addr)
1227{
1228 return vma && (vma->vm_start == addr) && (vma->vm_flags & VM_GROWSUP);
1229}
1230
1231static inline int stack_guard_page_end(struct vm_area_struct *vma,
1232 unsigned long addr)
1233{
1234 return (vma->vm_flags & VM_GROWSUP) &&
1235 (vma->vm_end == addr) &&
1236 !vma_growsup(vma->vm_next, addr);
1237}
1238
1239extern pid_t
1240vm_is_stack(struct task_struct *task, struct vm_area_struct *vma, int in_group);
1241
1242extern unsigned long move_page_tables(struct vm_area_struct *vma,
1243 unsigned long old_addr, struct vm_area_struct *new_vma,
1244 unsigned long new_addr, unsigned long len,
1245 bool need_rmap_locks);
1246extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
1247 unsigned long end, pgprot_t newprot,
1248 int dirty_accountable, int prot_numa);
1249extern int mprotect_fixup(struct vm_area_struct *vma,
1250 struct vm_area_struct **pprev, unsigned long start,
1251 unsigned long end, unsigned long newflags);
1252
1253/*
1254 * doesn't attempt to fault and will return short.
1255 */
1256int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
1257 struct page **pages);
1258/*
1259 * per-process(per-mm_struct) statistics.
1260 */
1261static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
1262{
1263 long val = atomic_long_read(&mm->rss_stat.count[member]);
1264
1265#ifdef SPLIT_RSS_COUNTING
1266 /*
1267 * counter is updated in asynchronous manner and may go to minus.
1268 * But it's never be expected number for users.
1269 */
1270 if (val < 0)
1271 val = 0;
1272#endif
1273 return (unsigned long)val;
1274}
1275
1276static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
1277{
1278 atomic_long_add(value, &mm->rss_stat.count[member]);
1279}
1280
1281static inline void inc_mm_counter(struct mm_struct *mm, int member)
1282{
1283 atomic_long_inc(&mm->rss_stat.count[member]);
1284}
1285
1286static inline void dec_mm_counter(struct mm_struct *mm, int member)
1287{
1288 atomic_long_dec(&mm->rss_stat.count[member]);
1289}
1290
1291static inline unsigned long get_mm_rss(struct mm_struct *mm)
1292{
1293 return get_mm_counter(mm, MM_FILEPAGES) +
1294 get_mm_counter(mm, MM_ANONPAGES);
1295}
1296
1297static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
1298{
1299 return max(mm->hiwater_rss, get_mm_rss(mm));
1300}
1301
1302static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
1303{
1304 return max(mm->hiwater_vm, mm->total_vm);
1305}
1306
1307static inline void update_hiwater_rss(struct mm_struct *mm)
1308{
1309 unsigned long _rss = get_mm_rss(mm);
1310
1311 if ((mm)->hiwater_rss < _rss)
1312 (mm)->hiwater_rss = _rss;
1313}
1314
1315static inline void update_hiwater_vm(struct mm_struct *mm)
1316{
1317 if (mm->hiwater_vm < mm->total_vm)
1318 mm->hiwater_vm = mm->total_vm;
1319}
1320
1321static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
1322 struct mm_struct *mm)
1323{
1324 unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
1325
1326 if (*maxrss < hiwater_rss)
1327 *maxrss = hiwater_rss;
1328}
1329
1330#if defined(SPLIT_RSS_COUNTING)
1331void sync_mm_rss(struct mm_struct *mm);
1332#else
1333static inline void sync_mm_rss(struct mm_struct *mm)
1334{
1335}
1336#endif
1337
1338int vma_wants_writenotify(struct vm_area_struct *vma);
1339
1340extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
1341 spinlock_t **ptl);
1342static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
1343 spinlock_t **ptl)
1344{
1345 pte_t *ptep;
1346 __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
1347 return ptep;
1348}
1349
1350#ifdef __PAGETABLE_PUD_FOLDED
1351static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
1352 unsigned long address)
1353{
1354 return 0;
1355}
1356#else
1357int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
1358#endif
1359
1360#ifdef __PAGETABLE_PMD_FOLDED
1361static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
1362 unsigned long address)
1363{
1364 return 0;
1365}
1366#else
1367int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
1368#endif
1369
1370int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
1371 pmd_t *pmd, unsigned long address);
1372int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
1373
1374/*
1375 * The following ifdef needed to get the 4level-fixup.h header to work.
1376 * Remove it when 4level-fixup.h has been removed.
1377 */
1378#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
1379static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
1380{
1381 return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
1382 NULL: pud_offset(pgd, address);
1383}
1384
1385static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
1386{
1387 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
1388 NULL: pmd_offset(pud, address);
1389}
1390#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
1391
1392#if USE_SPLIT_PTE_PTLOCKS
1393#if ALLOC_SPLIT_PTLOCKS
1394void __init ptlock_cache_init(void);
1395extern bool ptlock_alloc(struct page *page);
1396extern void ptlock_free(struct page *page);
1397
1398static inline spinlock_t *ptlock_ptr(struct page *page)
1399{
1400 return page->ptl;
1401}
1402#else /* ALLOC_SPLIT_PTLOCKS */
1403static inline void ptlock_cache_init(void)
1404{
1405}
1406
1407static inline bool ptlock_alloc(struct page *page)
1408{
1409 return true;
1410}
1411
1412static inline void ptlock_free(struct page *page)
1413{
1414}
1415
1416static inline spinlock_t *ptlock_ptr(struct page *page)
1417{
1418 return &page->ptl;
1419}
1420#endif /* ALLOC_SPLIT_PTLOCKS */
1421
1422static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1423{
1424 return ptlock_ptr(pmd_page(*pmd));
1425}
1426
1427static inline bool ptlock_init(struct page *page)
1428{
1429 /*
1430 * prep_new_page() initialize page->private (and therefore page->ptl)
1431 * with 0. Make sure nobody took it in use in between.
1432 *
1433 * It can happen if arch try to use slab for page table allocation:
1434 * slab code uses page->slab_cache and page->first_page (for tail
1435 * pages), which share storage with page->ptl.
1436 */
1437 VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page);
1438 if (!ptlock_alloc(page))
1439 return false;
1440 spin_lock_init(ptlock_ptr(page));
1441 return true;
1442}
1443
1444/* Reset page->mapping so free_pages_check won't complain. */
1445static inline void pte_lock_deinit(struct page *page)
1446{
1447 page->mapping = NULL;
1448 ptlock_free(page);
1449}
1450
1451#else /* !USE_SPLIT_PTE_PTLOCKS */
1452/*
1453 * We use mm->page_table_lock to guard all pagetable pages of the mm.
1454 */
1455static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1456{
1457 return &mm->page_table_lock;
1458}
1459static inline void ptlock_cache_init(void) {}
1460static inline bool ptlock_init(struct page *page) { return true; }
1461static inline void pte_lock_deinit(struct page *page) {}
1462#endif /* USE_SPLIT_PTE_PTLOCKS */
1463
1464static inline void pgtable_init(void)
1465{
1466 ptlock_cache_init();
1467 pgtable_cache_init();
1468}
1469
1470static inline bool pgtable_page_ctor(struct page *page)
1471{
1472 inc_zone_page_state(page, NR_PAGETABLE);
1473 return ptlock_init(page);
1474}
1475
1476static inline void pgtable_page_dtor(struct page *page)
1477{
1478 pte_lock_deinit(page);
1479 dec_zone_page_state(page, NR_PAGETABLE);
1480}
1481
1482#define pte_offset_map_lock(mm, pmd, address, ptlp) \
1483({ \
1484 spinlock_t *__ptl = pte_lockptr(mm, pmd); \
1485 pte_t *__pte = pte_offset_map(pmd, address); \
1486 *(ptlp) = __ptl; \
1487 spin_lock(__ptl); \
1488 __pte; \
1489})
1490
1491#define pte_unmap_unlock(pte, ptl) do { \
1492 spin_unlock(ptl); \
1493 pte_unmap(pte); \
1494} while (0)
1495
1496#define pte_alloc_map(mm, vma, pmd, address) \
1497 ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, vma, \
1498 pmd, address))? \
1499 NULL: pte_offset_map(pmd, address))
1500
1501#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
1502 ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, NULL, \
1503 pmd, address))? \
1504 NULL: pte_offset_map_lock(mm, pmd, address, ptlp))
1505
1506#define pte_alloc_kernel(pmd, address) \
1507 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
1508 NULL: pte_offset_kernel(pmd, address))
1509
1510#if USE_SPLIT_PMD_PTLOCKS
1511
1512static struct page *pmd_to_page(pmd_t *pmd)
1513{
1514 unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
1515 return virt_to_page((void *)((unsigned long) pmd & mask));
1516}
1517
1518static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1519{
1520 return ptlock_ptr(pmd_to_page(pmd));
1521}
1522
1523static inline bool pgtable_pmd_page_ctor(struct page *page)
1524{
1525#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1526 page->pmd_huge_pte = NULL;
1527#endif
1528 return ptlock_init(page);
1529}
1530
1531static inline void pgtable_pmd_page_dtor(struct page *page)
1532{
1533#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1534 VM_BUG_ON_PAGE(page->pmd_huge_pte, page);
1535#endif
1536 ptlock_free(page);
1537}
1538
1539#define pmd_huge_pte(mm, pmd) (pmd_to_page(pmd)->pmd_huge_pte)
1540
1541#else
1542
1543static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1544{
1545 return &mm->page_table_lock;
1546}
1547
1548static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }
1549static inline void pgtable_pmd_page_dtor(struct page *page) {}
1550
1551#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
1552
1553#endif
1554
1555static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
1556{
1557 spinlock_t *ptl = pmd_lockptr(mm, pmd);
1558 spin_lock(ptl);
1559 return ptl;
1560}
1561
1562extern void free_area_init(unsigned long * zones_size);
1563extern void free_area_init_node(int nid, unsigned long * zones_size,
1564 unsigned long zone_start_pfn, unsigned long *zholes_size);
1565extern void free_initmem(void);
1566
1567/*
1568 * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK)
1569 * into the buddy system. The freed pages will be poisoned with pattern
1570 * "poison" if it's within range [0, UCHAR_MAX].
1571 * Return pages freed into the buddy system.
1572 */
1573extern unsigned long free_reserved_area(void *start, void *end,
1574 int poison, char *s);
1575
1576#ifdef CONFIG_HIGHMEM
1577/*
1578 * Free a highmem page into the buddy system, adjusting totalhigh_pages
1579 * and totalram_pages.
1580 */
1581extern void free_highmem_page(struct page *page);
1582#endif
1583
1584extern void adjust_managed_page_count(struct page *page, long count);
1585extern void mem_init_print_info(const char *str);
1586
1587/* Free the reserved page into the buddy system, so it gets managed. */
1588static inline void __free_reserved_page(struct page *page)
1589{
1590 ClearPageReserved(page);
1591 init_page_count(page);
1592 __free_page(page);
1593}
1594
1595static inline void free_reserved_page(struct page *page)
1596{
1597 __free_reserved_page(page);
1598 adjust_managed_page_count(page, 1);
1599}
1600
1601static inline void mark_page_reserved(struct page *page)
1602{
1603 SetPageReserved(page);
1604 adjust_managed_page_count(page, -1);
1605}
1606
1607/*
1608 * Default method to free all the __init memory into the buddy system.
1609 * The freed pages will be poisoned with pattern "poison" if it's within
1610 * range [0, UCHAR_MAX].
1611 * Return pages freed into the buddy system.
1612 */
1613static inline unsigned long free_initmem_default(int poison)
1614{
1615 extern char __init_begin[], __init_end[];
1616
1617 return free_reserved_area(&__init_begin, &__init_end,
1618 poison, "unused kernel");
1619}
1620
1621static inline unsigned long get_num_physpages(void)
1622{
1623 int nid;
1624 unsigned long phys_pages = 0;
1625
1626 for_each_online_node(nid)
1627 phys_pages += node_present_pages(nid);
1628
1629 return phys_pages;
1630}
1631
1632#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1633/*
1634 * With CONFIG_HAVE_MEMBLOCK_NODE_MAP set, an architecture may initialise its
1635 * zones, allocate the backing mem_map and account for memory holes in a more
1636 * architecture independent manner. This is a substitute for creating the
1637 * zone_sizes[] and zholes_size[] arrays and passing them to
1638 * free_area_init_node()
1639 *
1640 * An architecture is expected to register range of page frames backed by
1641 * physical memory with memblock_add[_node]() before calling
1642 * free_area_init_nodes() passing in the PFN each zone ends at. At a basic
1643 * usage, an architecture is expected to do something like
1644 *
1645 * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn,
1646 * max_highmem_pfn};
1647 * for_each_valid_physical_page_range()
1648 * memblock_add_node(base, size, nid)
1649 * free_area_init_nodes(max_zone_pfns);
1650 *
1651 * free_bootmem_with_active_regions() calls free_bootmem_node() for each
1652 * registered physical page range. Similarly
1653 * sparse_memory_present_with_active_regions() calls memory_present() for
1654 * each range when SPARSEMEM is enabled.
1655 *
1656 * See mm/page_alloc.c for more information on each function exposed by
1657 * CONFIG_HAVE_MEMBLOCK_NODE_MAP.
1658 */
1659extern void free_area_init_nodes(unsigned long *max_zone_pfn);
1660unsigned long node_map_pfn_alignment(void);
1661unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn,
1662 unsigned long end_pfn);
1663extern unsigned long absent_pages_in_range(unsigned long start_pfn,
1664 unsigned long end_pfn);
1665extern void get_pfn_range_for_nid(unsigned int nid,
1666 unsigned long *start_pfn, unsigned long *end_pfn);
1667extern unsigned long find_min_pfn_with_active_regions(void);
1668extern void free_bootmem_with_active_regions(int nid,
1669 unsigned long max_low_pfn);
1670extern void sparse_memory_present_with_active_regions(int nid);
1671
1672#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
1673
1674#if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \
1675 !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
1676static inline int __early_pfn_to_nid(unsigned long pfn)
1677{
1678 return 0;
1679}
1680#else
1681/* please see mm/page_alloc.c */
1682extern int __meminit early_pfn_to_nid(unsigned long pfn);
1683/* there is a per-arch backend function. */
1684extern int __meminit __early_pfn_to_nid(unsigned long pfn);
1685#endif
1686
1687extern void set_dma_reserve(unsigned long new_dma_reserve);
1688extern void memmap_init_zone(unsigned long, int, unsigned long,
1689 unsigned long, enum memmap_context);
1690extern void setup_per_zone_wmarks(void);
1691extern int __meminit init_per_zone_wmark_min(void);
1692extern void mem_init(void);
1693extern void __init mmap_init(void);
1694extern void show_mem(unsigned int flags);
1695extern void si_meminfo(struct sysinfo * val);
1696extern void si_meminfo_node(struct sysinfo *val, int nid);
1697
1698extern __printf(3, 4)
1699void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...);
1700
1701extern void setup_per_cpu_pageset(void);
1702
1703extern void zone_pcp_update(struct zone *zone);
1704extern void zone_pcp_reset(struct zone *zone);
1705
1706/* page_alloc.c */
1707extern int min_free_kbytes;
1708
1709/* nommu.c */
1710extern atomic_long_t mmap_pages_allocated;
1711extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
1712
1713/* interval_tree.c */
1714void vma_interval_tree_insert(struct vm_area_struct *node,
1715 struct rb_root *root);
1716void vma_interval_tree_insert_after(struct vm_area_struct *node,
1717 struct vm_area_struct *prev,
1718 struct rb_root *root);
1719void vma_interval_tree_remove(struct vm_area_struct *node,
1720 struct rb_root *root);
1721struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root *root,
1722 unsigned long start, unsigned long last);
1723struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
1724 unsigned long start, unsigned long last);
1725
1726#define vma_interval_tree_foreach(vma, root, start, last) \
1727 for (vma = vma_interval_tree_iter_first(root, start, last); \
1728 vma; vma = vma_interval_tree_iter_next(vma, start, last))
1729
1730static inline void vma_nonlinear_insert(struct vm_area_struct *vma,
1731 struct list_head *list)
1732{
1733 list_add_tail(&vma->shared.nonlinear, list);
1734}
1735
1736void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
1737 struct rb_root *root);
1738void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
1739 struct rb_root *root);
1740struct anon_vma_chain *anon_vma_interval_tree_iter_first(
1741 struct rb_root *root, unsigned long start, unsigned long last);
1742struct anon_vma_chain *anon_vma_interval_tree_iter_next(
1743 struct anon_vma_chain *node, unsigned long start, unsigned long last);
1744#ifdef CONFIG_DEBUG_VM_RB
1745void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
1746#endif
1747
1748#define anon_vma_interval_tree_foreach(avc, root, start, last) \
1749 for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
1750 avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
1751
1752/* mmap.c */
1753extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
1754extern int vma_adjust(struct vm_area_struct *vma, unsigned long start,
1755 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert);
1756extern struct vm_area_struct *vma_merge(struct mm_struct *,
1757 struct vm_area_struct *prev, unsigned long addr, unsigned long end,
1758 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
1759 struct mempolicy *);
1760extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
1761extern int split_vma(struct mm_struct *,
1762 struct vm_area_struct *, unsigned long addr, int new_below);
1763extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
1764extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
1765 struct rb_node **, struct rb_node *);
1766extern void unlink_file_vma(struct vm_area_struct *);
1767extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
1768 unsigned long addr, unsigned long len, pgoff_t pgoff,
1769 bool *need_rmap_locks);
1770extern void exit_mmap(struct mm_struct *);
1771
1772extern int mm_take_all_locks(struct mm_struct *mm);
1773extern void mm_drop_all_locks(struct mm_struct *mm);
1774
1775extern void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
1776extern struct file *get_mm_exe_file(struct mm_struct *mm);
1777
1778extern int may_expand_vm(struct mm_struct *mm, unsigned long npages);
1779extern struct vm_area_struct *_install_special_mapping(struct mm_struct *mm,
1780 unsigned long addr, unsigned long len,
1781 unsigned long flags, struct page **pages);
1782extern int install_special_mapping(struct mm_struct *mm,
1783 unsigned long addr, unsigned long len,
1784 unsigned long flags, struct page **pages);
1785
1786extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
1787
1788extern unsigned long mmap_region(struct file *file, unsigned long addr,
1789 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff);
1790extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
1791 unsigned long len, unsigned long prot, unsigned long flags,
1792 unsigned long pgoff, unsigned long *populate);
1793extern int do_munmap(struct mm_struct *, unsigned long, size_t);
1794
1795#ifdef CONFIG_MMU
1796extern int __mm_populate(unsigned long addr, unsigned long len,
1797 int ignore_errors);
1798static inline void mm_populate(unsigned long addr, unsigned long len)
1799{
1800 /* Ignore errors */
1801 (void) __mm_populate(addr, len, 1);
1802}
1803#else
1804static inline void mm_populate(unsigned long addr, unsigned long len) {}
1805#endif
1806
1807/* These take the mm semaphore themselves */
1808extern unsigned long vm_brk(unsigned long, unsigned long);
1809extern int vm_munmap(unsigned long, size_t);
1810extern unsigned long vm_mmap(struct file *, unsigned long,
1811 unsigned long, unsigned long,
1812 unsigned long, unsigned long);
1813
1814struct vm_unmapped_area_info {
1815#define VM_UNMAPPED_AREA_TOPDOWN 1
1816 unsigned long flags;
1817 unsigned long length;
1818 unsigned long low_limit;
1819 unsigned long high_limit;
1820 unsigned long align_mask;
1821 unsigned long align_offset;
1822};
1823
1824extern unsigned long unmapped_area(struct vm_unmapped_area_info *info);
1825extern unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);
1826
1827/*
1828 * Search for an unmapped address range.
1829 *
1830 * We are looking for a range that:
1831 * - does not intersect with any VMA;
1832 * - is contained within the [low_limit, high_limit) interval;
1833 * - is at least the desired size.
1834 * - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
1835 */
1836static inline unsigned long
1837vm_unmapped_area(struct vm_unmapped_area_info *info)
1838{
1839 if (!(info->flags & VM_UNMAPPED_AREA_TOPDOWN))
1840 return unmapped_area(info);
1841 else
1842 return unmapped_area_topdown(info);
1843}
1844
1845/* truncate.c */
1846extern void truncate_inode_pages(struct address_space *, loff_t);
1847extern void truncate_inode_pages_range(struct address_space *,
1848 loff_t lstart, loff_t lend);
1849extern void truncate_inode_pages_final(struct address_space *);
1850
1851/* generic vm_area_ops exported for stackable file systems */
1852extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
1853extern void filemap_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf);
1854extern int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf);
1855
1856/* mm/page-writeback.c */
1857int write_one_page(struct page *page, int wait);
1858void task_dirty_inc(struct task_struct *tsk);
1859
1860/* readahead.c */
1861#define VM_MAX_READAHEAD 128 /* kbytes */
1862#define VM_MIN_READAHEAD 16 /* kbytes (includes current page) */
1863
1864int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
1865 pgoff_t offset, unsigned long nr_to_read);
1866
1867void page_cache_sync_readahead(struct address_space *mapping,
1868 struct file_ra_state *ra,
1869 struct file *filp,
1870 pgoff_t offset,
1871 unsigned long size);
1872
1873void page_cache_async_readahead(struct address_space *mapping,
1874 struct file_ra_state *ra,
1875 struct file *filp,
1876 struct page *pg,
1877 pgoff_t offset,
1878 unsigned long size);
1879
1880unsigned long max_sane_readahead(unsigned long nr);
1881
1882/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
1883extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
1884
1885/* CONFIG_STACK_GROWSUP still needs to to grow downwards at some places */
1886extern int expand_downwards(struct vm_area_struct *vma,
1887 unsigned long address);
1888#if VM_GROWSUP
1889extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
1890#else
1891 #define expand_upwards(vma, address) do { } while (0)
1892#endif
1893
1894/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
1895extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
1896extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
1897 struct vm_area_struct **pprev);
1898
1899/* Look up the first VMA which intersects the interval start_addr..end_addr-1,
1900 NULL if none. Assume start_addr < end_addr. */
1901static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
1902{
1903 struct vm_area_struct * vma = find_vma(mm,start_addr);
1904
1905 if (vma && end_addr <= vma->vm_start)
1906 vma = NULL;
1907 return vma;
1908}
1909
1910static inline unsigned long vma_pages(struct vm_area_struct *vma)
1911{
1912 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
1913}
1914
1915/* Look up the first VMA which exactly match the interval vm_start ... vm_end */
1916static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
1917 unsigned long vm_start, unsigned long vm_end)
1918{
1919 struct vm_area_struct *vma = find_vma(mm, vm_start);
1920
1921 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
1922 vma = NULL;
1923
1924 return vma;
1925}
1926
1927#ifdef CONFIG_MMU
1928pgprot_t vm_get_page_prot(unsigned long vm_flags);
1929#else
1930static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
1931{
1932 return __pgprot(0);
1933}
1934#endif
1935
1936#ifdef CONFIG_NUMA_BALANCING
1937unsigned long change_prot_numa(struct vm_area_struct *vma,
1938 unsigned long start, unsigned long end);
1939#endif
1940
1941struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
1942int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
1943 unsigned long pfn, unsigned long size, pgprot_t);
1944int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
1945int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
1946 unsigned long pfn);
1947int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
1948 unsigned long pfn);
1949int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);
1950
1951
1952struct page *follow_page_mask(struct vm_area_struct *vma,
1953 unsigned long address, unsigned int foll_flags,
1954 unsigned int *page_mask);
1955
1956static inline struct page *follow_page(struct vm_area_struct *vma,
1957 unsigned long address, unsigned int foll_flags)
1958{
1959 unsigned int unused_page_mask;
1960 return follow_page_mask(vma, address, foll_flags, &unused_page_mask);
1961}
1962
1963#define FOLL_WRITE 0x01 /* check pte is writable */
1964#define FOLL_TOUCH 0x02 /* mark page accessed */
1965#define FOLL_GET 0x04 /* do get_page on page */
1966#define FOLL_DUMP 0x08 /* give error on hole if it would be zero */
1967#define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */
1968#define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO
1969 * and return without waiting upon it */
1970#define FOLL_MLOCK 0x40 /* mark page as mlocked */
1971#define FOLL_SPLIT 0x80 /* don't return transhuge pages, split them */
1972#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
1973#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
1974#define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
1975
1976typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
1977 void *data);
1978extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
1979 unsigned long size, pte_fn_t fn, void *data);
1980
1981#ifdef CONFIG_PROC_FS
1982void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
1983#else
1984static inline void vm_stat_account(struct mm_struct *mm,
1985 unsigned long flags, struct file *file, long pages)
1986{
1987 mm->total_vm += pages;
1988}
1989#endif /* CONFIG_PROC_FS */
1990
1991#ifdef CONFIG_DEBUG_PAGEALLOC
1992extern void kernel_map_pages(struct page *page, int numpages, int enable);
1993#ifdef CONFIG_HIBERNATION
1994extern bool kernel_page_present(struct page *page);
1995#endif /* CONFIG_HIBERNATION */
1996#else
1997static inline void
1998kernel_map_pages(struct page *page, int numpages, int enable) {}
1999#ifdef CONFIG_HIBERNATION
2000static inline bool kernel_page_present(struct page *page) { return true; }
2001#endif /* CONFIG_HIBERNATION */
2002#endif
2003
2004extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);
2005#ifdef __HAVE_ARCH_GATE_AREA
2006int in_gate_area_no_mm(unsigned long addr);
2007int in_gate_area(struct mm_struct *mm, unsigned long addr);
2008#else
2009int in_gate_area_no_mm(unsigned long addr);
2010#define in_gate_area(mm, addr) ({(void)mm; in_gate_area_no_mm(addr);})
2011#endif /* __HAVE_ARCH_GATE_AREA */
2012
2013#ifdef CONFIG_SYSCTL
2014extern int sysctl_drop_caches;
2015int drop_caches_sysctl_handler(struct ctl_table *, int,
2016 void __user *, size_t *, loff_t *);
2017#endif
2018
2019unsigned long shrink_slab(struct shrink_control *shrink,
2020 unsigned long nr_pages_scanned,
2021 unsigned long lru_pages);
2022
2023#ifndef CONFIG_MMU
2024#define randomize_va_space 0
2025#else
2026extern int randomize_va_space;
2027#endif
2028
2029const char * arch_vma_name(struct vm_area_struct *vma);
2030void print_vma_addr(char *prefix, unsigned long rip);
2031
2032void sparse_mem_maps_populate_node(struct page **map_map,
2033 unsigned long pnum_begin,
2034 unsigned long pnum_end,
2035 unsigned long map_count,
2036 int nodeid);
2037
2038struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
2039pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
2040pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
2041pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
2042pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
2043void *vmemmap_alloc_block(unsigned long size, int node);
2044void *vmemmap_alloc_block_buf(unsigned long size, int node);
2045void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
2046int vmemmap_populate_basepages(unsigned long start, unsigned long end,
2047 int node);
2048int vmemmap_populate(unsigned long start, unsigned long end, int node);
2049void vmemmap_populate_print_last(void);
2050#ifdef CONFIG_MEMORY_HOTPLUG
2051void vmemmap_free(unsigned long start, unsigned long end);
2052#endif
2053void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
2054 unsigned long size);
2055
2056enum mf_flags {
2057 MF_COUNT_INCREASED = 1 << 0,
2058 MF_ACTION_REQUIRED = 1 << 1,
2059 MF_MUST_KILL = 1 << 2,
2060 MF_SOFT_OFFLINE = 1 << 3,
2061};
2062extern int memory_failure(unsigned long pfn, int trapno, int flags);
2063extern void memory_failure_queue(unsigned long pfn, int trapno, int flags);
2064extern int unpoison_memory(unsigned long pfn);
2065extern int sysctl_memory_failure_early_kill;
2066extern int sysctl_memory_failure_recovery;
2067extern void shake_page(struct page *p, int access);
2068extern atomic_long_t num_poisoned_pages;
2069extern int soft_offline_page(struct page *page, int flags);
2070
2071#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
2072extern void clear_huge_page(struct page *page,
2073 unsigned long addr,
2074 unsigned int pages_per_huge_page);
2075extern void copy_user_huge_page(struct page *dst, struct page *src,
2076 unsigned long addr, struct vm_area_struct *vma,
2077 unsigned int pages_per_huge_page);
2078#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
2079
2080#ifdef CONFIG_DEBUG_PAGEALLOC
2081extern unsigned int _debug_guardpage_minorder;
2082
2083static inline unsigned int debug_guardpage_minorder(void)
2084{
2085 return _debug_guardpage_minorder;
2086}
2087
2088static inline bool page_is_guard(struct page *page)
2089{
2090 return test_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);
2091}
2092#else
2093static inline unsigned int debug_guardpage_minorder(void) { return 0; }
2094static inline bool page_is_guard(struct page *page) { return false; }
2095#endif /* CONFIG_DEBUG_PAGEALLOC */
2096
2097#if MAX_NUMNODES > 1
2098void __init setup_nr_node_ids(void);
2099#else
2100static inline void setup_nr_node_ids(void) {}
2101#endif
2102
2103#endif /* __KERNEL__ */
2104#endif /* _LINUX_MM_H */