Loading...
1#ifndef _LINUX_MM_H
2#define _LINUX_MM_H
3
4#include <linux/errno.h>
5
6#ifdef __KERNEL__
7
8#include <linux/mmdebug.h>
9#include <linux/gfp.h>
10#include <linux/bug.h>
11#include <linux/list.h>
12#include <linux/mmzone.h>
13#include <linux/rbtree.h>
14#include <linux/atomic.h>
15#include <linux/debug_locks.h>
16#include <linux/mm_types.h>
17#include <linux/range.h>
18#include <linux/pfn.h>
19#include <linux/percpu-refcount.h>
20#include <linux/bit_spinlock.h>
21#include <linux/shrinker.h>
22#include <linux/resource.h>
23#include <linux/page_ext.h>
24#include <linux/err.h>
25#include <linux/page_ref.h>
26
27struct mempolicy;
28struct anon_vma;
29struct anon_vma_chain;
30struct file_ra_state;
31struct user_struct;
32struct writeback_control;
33struct bdi_writeback;
34
35#ifndef CONFIG_NEED_MULTIPLE_NODES /* Don't use mapnrs, do it properly */
36extern unsigned long max_mapnr;
37
38static inline void set_max_mapnr(unsigned long limit)
39{
40 max_mapnr = limit;
41}
42#else
43static inline void set_max_mapnr(unsigned long limit) { }
44#endif
45
46extern unsigned long totalram_pages;
47extern void * high_memory;
48extern int page_cluster;
49
50#ifdef CONFIG_SYSCTL
51extern int sysctl_legacy_va_layout;
52#else
53#define sysctl_legacy_va_layout 0
54#endif
55
56#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
57extern const int mmap_rnd_bits_min;
58extern const int mmap_rnd_bits_max;
59extern int mmap_rnd_bits __read_mostly;
60#endif
61#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
62extern const int mmap_rnd_compat_bits_min;
63extern const int mmap_rnd_compat_bits_max;
64extern int mmap_rnd_compat_bits __read_mostly;
65#endif
66
67#include <asm/page.h>
68#include <asm/pgtable.h>
69#include <asm/processor.h>
70
71#ifndef __pa_symbol
72#define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0))
73#endif
74
75#ifndef page_to_virt
76#define page_to_virt(x) __va(PFN_PHYS(page_to_pfn(x)))
77#endif
78
79/*
80 * To prevent common memory management code establishing
81 * a zero page mapping on a read fault.
82 * This macro should be defined within <asm/pgtable.h>.
83 * s390 does this to prevent multiplexing of hardware bits
84 * related to the physical page in case of virtualization.
85 */
86#ifndef mm_forbids_zeropage
87#define mm_forbids_zeropage(X) (0)
88#endif
89
90/*
91 * Default maximum number of active map areas, this limits the number of vmas
92 * per mm struct. Users can overwrite this number by sysctl but there is a
93 * problem.
94 *
95 * When a program's coredump is generated as ELF format, a section is created
96 * per a vma. In ELF, the number of sections is represented in unsigned short.
97 * This means the number of sections should be smaller than 65535 at coredump.
98 * Because the kernel adds some informative sections to a image of program at
99 * generating coredump, we need some margin. The number of extra sections is
100 * 1-3 now and depends on arch. We use "5" as safe margin, here.
101 *
102 * ELF extended numbering allows more than 65535 sections, so 16-bit bound is
103 * not a hard limit any more. Although some userspace tools can be surprised by
104 * that.
105 */
106#define MAPCOUNT_ELF_CORE_MARGIN (5)
107#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
108
109extern int sysctl_max_map_count;
110
111extern unsigned long sysctl_user_reserve_kbytes;
112extern unsigned long sysctl_admin_reserve_kbytes;
113
114extern int sysctl_overcommit_memory;
115extern int sysctl_overcommit_ratio;
116extern unsigned long sysctl_overcommit_kbytes;
117
118extern int overcommit_ratio_handler(struct ctl_table *, int, void __user *,
119 size_t *, loff_t *);
120extern int overcommit_kbytes_handler(struct ctl_table *, int, void __user *,
121 size_t *, loff_t *);
122
123#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
124
125/* to align the pointer to the (next) page boundary */
126#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
127
128/* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */
129#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE)
130
131/*
132 * Linux kernel virtual memory manager primitives.
133 * The idea being to have a "virtual" mm in the same way
134 * we have a virtual fs - giving a cleaner interface to the
135 * mm details, and allowing different kinds of memory mappings
136 * (from shared memory to executable loading to arbitrary
137 * mmap() functions).
138 */
139
140extern struct kmem_cache *vm_area_cachep;
141
142#ifndef CONFIG_MMU
143extern struct rb_root nommu_region_tree;
144extern struct rw_semaphore nommu_region_sem;
145
146extern unsigned int kobjsize(const void *objp);
147#endif
148
149/*
150 * vm_flags in vm_area_struct, see mm_types.h.
151 * When changing, update also include/trace/events/mmflags.h
152 */
153#define VM_NONE 0x00000000
154
155#define VM_READ 0x00000001 /* currently active flags */
156#define VM_WRITE 0x00000002
157#define VM_EXEC 0x00000004
158#define VM_SHARED 0x00000008
159
160/* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */
161#define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */
162#define VM_MAYWRITE 0x00000020
163#define VM_MAYEXEC 0x00000040
164#define VM_MAYSHARE 0x00000080
165
166#define VM_GROWSDOWN 0x00000100 /* general info on the segment */
167#define VM_UFFD_MISSING 0x00000200 /* missing pages tracking */
168#define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
169#define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */
170#define VM_UFFD_WP 0x00001000 /* wrprotect pages tracking */
171
172#define VM_LOCKED 0x00002000
173#define VM_IO 0x00004000 /* Memory mapped I/O or similar */
174
175 /* Used by sys_madvise() */
176#define VM_SEQ_READ 0x00008000 /* App will access data sequentially */
177#define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */
178
179#define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */
180#define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */
181#define VM_LOCKONFAULT 0x00080000 /* Lock the pages covered when they are faulted in */
182#define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */
183#define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */
184#define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
185#define VM_ARCH_1 0x01000000 /* Architecture-specific flag */
186#define VM_ARCH_2 0x02000000
187#define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */
188
189#ifdef CONFIG_MEM_SOFT_DIRTY
190# define VM_SOFTDIRTY 0x08000000 /* Not soft dirty clean area */
191#else
192# define VM_SOFTDIRTY 0
193#endif
194
195#define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
196#define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */
197#define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */
198#define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */
199
200#ifdef CONFIG_ARCH_USES_HIGH_VMA_FLAGS
201#define VM_HIGH_ARCH_BIT_0 32 /* bit only usable on 64-bit architectures */
202#define VM_HIGH_ARCH_BIT_1 33 /* bit only usable on 64-bit architectures */
203#define VM_HIGH_ARCH_BIT_2 34 /* bit only usable on 64-bit architectures */
204#define VM_HIGH_ARCH_BIT_3 35 /* bit only usable on 64-bit architectures */
205#define VM_HIGH_ARCH_0 BIT(VM_HIGH_ARCH_BIT_0)
206#define VM_HIGH_ARCH_1 BIT(VM_HIGH_ARCH_BIT_1)
207#define VM_HIGH_ARCH_2 BIT(VM_HIGH_ARCH_BIT_2)
208#define VM_HIGH_ARCH_3 BIT(VM_HIGH_ARCH_BIT_3)
209#endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */
210
211#if defined(CONFIG_X86)
212# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
213#if defined (CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS)
214# define VM_PKEY_SHIFT VM_HIGH_ARCH_BIT_0
215# define VM_PKEY_BIT0 VM_HIGH_ARCH_0 /* A protection key is a 4-bit value */
216# define VM_PKEY_BIT1 VM_HIGH_ARCH_1
217# define VM_PKEY_BIT2 VM_HIGH_ARCH_2
218# define VM_PKEY_BIT3 VM_HIGH_ARCH_3
219#endif
220#elif defined(CONFIG_PPC)
221# define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */
222#elif defined(CONFIG_PARISC)
223# define VM_GROWSUP VM_ARCH_1
224#elif defined(CONFIG_METAG)
225# define VM_GROWSUP VM_ARCH_1
226#elif defined(CONFIG_IA64)
227# define VM_GROWSUP VM_ARCH_1
228#elif !defined(CONFIG_MMU)
229# define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */
230#endif
231
232#if defined(CONFIG_X86)
233/* MPX specific bounds table or bounds directory */
234# define VM_MPX VM_ARCH_2
235#endif
236
237#ifndef VM_GROWSUP
238# define VM_GROWSUP VM_NONE
239#endif
240
241/* Bits set in the VMA until the stack is in its final location */
242#define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ)
243
244#ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */
245#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
246#endif
247
248#ifdef CONFIG_STACK_GROWSUP
249#define VM_STACK VM_GROWSUP
250#else
251#define VM_STACK VM_GROWSDOWN
252#endif
253
254#define VM_STACK_FLAGS (VM_STACK | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
255
256/*
257 * Special vmas that are non-mergable, non-mlock()able.
258 * Note: mm/huge_memory.c VM_NO_THP depends on this definition.
259 */
260#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP)
261
262/* This mask defines which mm->def_flags a process can inherit its parent */
263#define VM_INIT_DEF_MASK VM_NOHUGEPAGE
264
265/* This mask is used to clear all the VMA flags used by mlock */
266#define VM_LOCKED_CLEAR_MASK (~(VM_LOCKED | VM_LOCKONFAULT))
267
268/*
269 * mapping from the currently active vm_flags protection bits (the
270 * low four bits) to a page protection mask..
271 */
272extern pgprot_t protection_map[16];
273
274#define FAULT_FLAG_WRITE 0x01 /* Fault was a write access */
275#define FAULT_FLAG_MKWRITE 0x02 /* Fault was mkwrite of existing pte */
276#define FAULT_FLAG_ALLOW_RETRY 0x04 /* Retry fault if blocking */
277#define FAULT_FLAG_RETRY_NOWAIT 0x08 /* Don't drop mmap_sem and wait when retrying */
278#define FAULT_FLAG_KILLABLE 0x10 /* The fault task is in SIGKILL killable region */
279#define FAULT_FLAG_TRIED 0x20 /* Second try */
280#define FAULT_FLAG_USER 0x40 /* The fault originated in userspace */
281#define FAULT_FLAG_REMOTE 0x80 /* faulting for non current tsk/mm */
282#define FAULT_FLAG_INSTRUCTION 0x100 /* The fault was during an instruction fetch */
283
284/*
285 * vm_fault is filled by the the pagefault handler and passed to the vma's
286 * ->fault function. The vma's ->fault is responsible for returning a bitmask
287 * of VM_FAULT_xxx flags that give details about how the fault was handled.
288 *
289 * MM layer fills up gfp_mask for page allocations but fault handler might
290 * alter it if its implementation requires a different allocation context.
291 *
292 * pgoff should be used in favour of virtual_address, if possible.
293 */
294struct vm_fault {
295 struct vm_area_struct *vma; /* Target VMA */
296 unsigned int flags; /* FAULT_FLAG_xxx flags */
297 gfp_t gfp_mask; /* gfp mask to be used for allocations */
298 pgoff_t pgoff; /* Logical page offset based on vma */
299 unsigned long address; /* Faulting virtual address */
300 pmd_t *pmd; /* Pointer to pmd entry matching
301 * the 'address' */
302 pte_t orig_pte; /* Value of PTE at the time of fault */
303
304 struct page *cow_page; /* Page handler may use for COW fault */
305 struct mem_cgroup *memcg; /* Cgroup cow_page belongs to */
306 struct page *page; /* ->fault handlers should return a
307 * page here, unless VM_FAULT_NOPAGE
308 * is set (which is also implied by
309 * VM_FAULT_ERROR).
310 */
311 /* These three entries are valid only while holding ptl lock */
312 pte_t *pte; /* Pointer to pte entry matching
313 * the 'address'. NULL if the page
314 * table hasn't been allocated.
315 */
316 spinlock_t *ptl; /* Page table lock.
317 * Protects pte page table if 'pte'
318 * is not NULL, otherwise pmd.
319 */
320 pgtable_t prealloc_pte; /* Pre-allocated pte page table.
321 * vm_ops->map_pages() calls
322 * alloc_set_pte() from atomic context.
323 * do_fault_around() pre-allocates
324 * page table to avoid allocation from
325 * atomic context.
326 */
327};
328
329/*
330 * These are the virtual MM functions - opening of an area, closing and
331 * unmapping it (needed to keep files on disk up-to-date etc), pointer
332 * to the functions called when a no-page or a wp-page exception occurs.
333 */
334struct vm_operations_struct {
335 void (*open)(struct vm_area_struct * area);
336 void (*close)(struct vm_area_struct * area);
337 int (*mremap)(struct vm_area_struct * area);
338 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
339 int (*pmd_fault)(struct vm_area_struct *, unsigned long address,
340 pmd_t *, unsigned int flags);
341 void (*map_pages)(struct vm_fault *vmf,
342 pgoff_t start_pgoff, pgoff_t end_pgoff);
343
344 /* notification that a previously read-only page is about to become
345 * writable, if an error is returned it will cause a SIGBUS */
346 int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
347
348 /* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */
349 int (*pfn_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
350
351 /* called by access_process_vm when get_user_pages() fails, typically
352 * for use by special VMAs that can switch between memory and hardware
353 */
354 int (*access)(struct vm_area_struct *vma, unsigned long addr,
355 void *buf, int len, int write);
356
357 /* Called by the /proc/PID/maps code to ask the vma whether it
358 * has a special name. Returning non-NULL will also cause this
359 * vma to be dumped unconditionally. */
360 const char *(*name)(struct vm_area_struct *vma);
361
362#ifdef CONFIG_NUMA
363 /*
364 * set_policy() op must add a reference to any non-NULL @new mempolicy
365 * to hold the policy upon return. Caller should pass NULL @new to
366 * remove a policy and fall back to surrounding context--i.e. do not
367 * install a MPOL_DEFAULT policy, nor the task or system default
368 * mempolicy.
369 */
370 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
371
372 /*
373 * get_policy() op must add reference [mpol_get()] to any policy at
374 * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure
375 * in mm/mempolicy.c will do this automatically.
376 * get_policy() must NOT add a ref if the policy at (vma,addr) is not
377 * marked as MPOL_SHARED. vma policies are protected by the mmap_sem.
378 * If no [shared/vma] mempolicy exists at the addr, get_policy() op
379 * must return NULL--i.e., do not "fallback" to task or system default
380 * policy.
381 */
382 struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
383 unsigned long addr);
384#endif
385 /*
386 * Called by vm_normal_page() for special PTEs to find the
387 * page for @addr. This is useful if the default behavior
388 * (using pte_page()) would not find the correct page.
389 */
390 struct page *(*find_special_page)(struct vm_area_struct *vma,
391 unsigned long addr);
392};
393
394struct mmu_gather;
395struct inode;
396
397#define page_private(page) ((page)->private)
398#define set_page_private(page, v) ((page)->private = (v))
399
400#if !defined(__HAVE_ARCH_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE)
401static inline int pmd_devmap(pmd_t pmd)
402{
403 return 0;
404}
405#endif
406
407/*
408 * FIXME: take this include out, include page-flags.h in
409 * files which need it (119 of them)
410 */
411#include <linux/page-flags.h>
412#include <linux/huge_mm.h>
413
414/*
415 * Methods to modify the page usage count.
416 *
417 * What counts for a page usage:
418 * - cache mapping (page->mapping)
419 * - private data (page->private)
420 * - page mapped in a task's page tables, each mapping
421 * is counted separately
422 *
423 * Also, many kernel routines increase the page count before a critical
424 * routine so they can be sure the page doesn't go away from under them.
425 */
426
427/*
428 * Drop a ref, return true if the refcount fell to zero (the page has no users)
429 */
430static inline int put_page_testzero(struct page *page)
431{
432 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
433 return page_ref_dec_and_test(page);
434}
435
436/*
437 * Try to grab a ref unless the page has a refcount of zero, return false if
438 * that is the case.
439 * This can be called when MMU is off so it must not access
440 * any of the virtual mappings.
441 */
442static inline int get_page_unless_zero(struct page *page)
443{
444 return page_ref_add_unless(page, 1, 0);
445}
446
447extern int page_is_ram(unsigned long pfn);
448
449enum {
450 REGION_INTERSECTS,
451 REGION_DISJOINT,
452 REGION_MIXED,
453};
454
455int region_intersects(resource_size_t offset, size_t size, unsigned long flags,
456 unsigned long desc);
457
458/* Support for virtually mapped pages */
459struct page *vmalloc_to_page(const void *addr);
460unsigned long vmalloc_to_pfn(const void *addr);
461
462/*
463 * Determine if an address is within the vmalloc range
464 *
465 * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there
466 * is no special casing required.
467 */
468static inline bool is_vmalloc_addr(const void *x)
469{
470#ifdef CONFIG_MMU
471 unsigned long addr = (unsigned long)x;
472
473 return addr >= VMALLOC_START && addr < VMALLOC_END;
474#else
475 return false;
476#endif
477}
478#ifdef CONFIG_MMU
479extern int is_vmalloc_or_module_addr(const void *x);
480#else
481static inline int is_vmalloc_or_module_addr(const void *x)
482{
483 return 0;
484}
485#endif
486
487extern void kvfree(const void *addr);
488
489static inline atomic_t *compound_mapcount_ptr(struct page *page)
490{
491 return &page[1].compound_mapcount;
492}
493
494static inline int compound_mapcount(struct page *page)
495{
496 VM_BUG_ON_PAGE(!PageCompound(page), page);
497 page = compound_head(page);
498 return atomic_read(compound_mapcount_ptr(page)) + 1;
499}
500
501/*
502 * The atomic page->_mapcount, starts from -1: so that transitions
503 * both from it and to it can be tracked, using atomic_inc_and_test
504 * and atomic_add_negative(-1).
505 */
506static inline void page_mapcount_reset(struct page *page)
507{
508 atomic_set(&(page)->_mapcount, -1);
509}
510
511int __page_mapcount(struct page *page);
512
513static inline int page_mapcount(struct page *page)
514{
515 VM_BUG_ON_PAGE(PageSlab(page), page);
516
517 if (unlikely(PageCompound(page)))
518 return __page_mapcount(page);
519 return atomic_read(&page->_mapcount) + 1;
520}
521
522#ifdef CONFIG_TRANSPARENT_HUGEPAGE
523int total_mapcount(struct page *page);
524int page_trans_huge_mapcount(struct page *page, int *total_mapcount);
525#else
526static inline int total_mapcount(struct page *page)
527{
528 return page_mapcount(page);
529}
530static inline int page_trans_huge_mapcount(struct page *page,
531 int *total_mapcount)
532{
533 int mapcount = page_mapcount(page);
534 if (total_mapcount)
535 *total_mapcount = mapcount;
536 return mapcount;
537}
538#endif
539
540static inline struct page *virt_to_head_page(const void *x)
541{
542 struct page *page = virt_to_page(x);
543
544 return compound_head(page);
545}
546
547void __put_page(struct page *page);
548
549void put_pages_list(struct list_head *pages);
550
551void split_page(struct page *page, unsigned int order);
552
553/*
554 * Compound pages have a destructor function. Provide a
555 * prototype for that function and accessor functions.
556 * These are _only_ valid on the head of a compound page.
557 */
558typedef void compound_page_dtor(struct page *);
559
560/* Keep the enum in sync with compound_page_dtors array in mm/page_alloc.c */
561enum compound_dtor_id {
562 NULL_COMPOUND_DTOR,
563 COMPOUND_PAGE_DTOR,
564#ifdef CONFIG_HUGETLB_PAGE
565 HUGETLB_PAGE_DTOR,
566#endif
567#ifdef CONFIG_TRANSPARENT_HUGEPAGE
568 TRANSHUGE_PAGE_DTOR,
569#endif
570 NR_COMPOUND_DTORS,
571};
572extern compound_page_dtor * const compound_page_dtors[];
573
574static inline void set_compound_page_dtor(struct page *page,
575 enum compound_dtor_id compound_dtor)
576{
577 VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page);
578 page[1].compound_dtor = compound_dtor;
579}
580
581static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
582{
583 VM_BUG_ON_PAGE(page[1].compound_dtor >= NR_COMPOUND_DTORS, page);
584 return compound_page_dtors[page[1].compound_dtor];
585}
586
587static inline unsigned int compound_order(struct page *page)
588{
589 if (!PageHead(page))
590 return 0;
591 return page[1].compound_order;
592}
593
594static inline void set_compound_order(struct page *page, unsigned int order)
595{
596 page[1].compound_order = order;
597}
598
599void free_compound_page(struct page *page);
600
601#ifdef CONFIG_MMU
602/*
603 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
604 * servicing faults for write access. In the normal case, do always want
605 * pte_mkwrite. But get_user_pages can cause write faults for mappings
606 * that do not have writing enabled, when used by access_process_vm.
607 */
608static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
609{
610 if (likely(vma->vm_flags & VM_WRITE))
611 pte = pte_mkwrite(pte);
612 return pte;
613}
614
615int alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg,
616 struct page *page);
617int finish_fault(struct vm_fault *vmf);
618int finish_mkwrite_fault(struct vm_fault *vmf);
619#endif
620
621/*
622 * Multiple processes may "see" the same page. E.g. for untouched
623 * mappings of /dev/null, all processes see the same page full of
624 * zeroes, and text pages of executables and shared libraries have
625 * only one copy in memory, at most, normally.
626 *
627 * For the non-reserved pages, page_count(page) denotes a reference count.
628 * page_count() == 0 means the page is free. page->lru is then used for
629 * freelist management in the buddy allocator.
630 * page_count() > 0 means the page has been allocated.
631 *
632 * Pages are allocated by the slab allocator in order to provide memory
633 * to kmalloc and kmem_cache_alloc. In this case, the management of the
634 * page, and the fields in 'struct page' are the responsibility of mm/slab.c
635 * unless a particular usage is carefully commented. (the responsibility of
636 * freeing the kmalloc memory is the caller's, of course).
637 *
638 * A page may be used by anyone else who does a __get_free_page().
639 * In this case, page_count still tracks the references, and should only
640 * be used through the normal accessor functions. The top bits of page->flags
641 * and page->virtual store page management information, but all other fields
642 * are unused and could be used privately, carefully. The management of this
643 * page is the responsibility of the one who allocated it, and those who have
644 * subsequently been given references to it.
645 *
646 * The other pages (we may call them "pagecache pages") are completely
647 * managed by the Linux memory manager: I/O, buffers, swapping etc.
648 * The following discussion applies only to them.
649 *
650 * A pagecache page contains an opaque `private' member, which belongs to the
651 * page's address_space. Usually, this is the address of a circular list of
652 * the page's disk buffers. PG_private must be set to tell the VM to call
653 * into the filesystem to release these pages.
654 *
655 * A page may belong to an inode's memory mapping. In this case, page->mapping
656 * is the pointer to the inode, and page->index is the file offset of the page,
657 * in units of PAGE_SIZE.
658 *
659 * If pagecache pages are not associated with an inode, they are said to be
660 * anonymous pages. These may become associated with the swapcache, and in that
661 * case PG_swapcache is set, and page->private is an offset into the swapcache.
662 *
663 * In either case (swapcache or inode backed), the pagecache itself holds one
664 * reference to the page. Setting PG_private should also increment the
665 * refcount. The each user mapping also has a reference to the page.
666 *
667 * The pagecache pages are stored in a per-mapping radix tree, which is
668 * rooted at mapping->page_tree, and indexed by offset.
669 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space
670 * lists, we instead now tag pages as dirty/writeback in the radix tree.
671 *
672 * All pagecache pages may be subject to I/O:
673 * - inode pages may need to be read from disk,
674 * - inode pages which have been modified and are MAP_SHARED may need
675 * to be written back to the inode on disk,
676 * - anonymous pages (including MAP_PRIVATE file mappings) which have been
677 * modified may need to be swapped out to swap space and (later) to be read
678 * back into memory.
679 */
680
681/*
682 * The zone field is never updated after free_area_init_core()
683 * sets it, so none of the operations on it need to be atomic.
684 */
685
686/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */
687#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
688#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
689#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
690#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)
691
692/*
693 * Define the bit shifts to access each section. For non-existent
694 * sections we define the shift as 0; that plus a 0 mask ensures
695 * the compiler will optimise away reference to them.
696 */
697#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
698#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
699#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
700#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))
701
702/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
703#ifdef NODE_NOT_IN_PAGE_FLAGS
704#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
705#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
706 SECTIONS_PGOFF : ZONES_PGOFF)
707#else
708#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
709#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
710 NODES_PGOFF : ZONES_PGOFF)
711#endif
712
713#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
714
715#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
716#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
717#endif
718
719#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
720#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
721#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
722#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_SHIFT) - 1)
723#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
724
725static inline enum zone_type page_zonenum(const struct page *page)
726{
727 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
728}
729
730#ifdef CONFIG_ZONE_DEVICE
731void get_zone_device_page(struct page *page);
732void put_zone_device_page(struct page *page);
733static inline bool is_zone_device_page(const struct page *page)
734{
735 return page_zonenum(page) == ZONE_DEVICE;
736}
737#else
738static inline void get_zone_device_page(struct page *page)
739{
740}
741static inline void put_zone_device_page(struct page *page)
742{
743}
744static inline bool is_zone_device_page(const struct page *page)
745{
746 return false;
747}
748#endif
749
750static inline void get_page(struct page *page)
751{
752 page = compound_head(page);
753 /*
754 * Getting a normal page or the head of a compound page
755 * requires to already have an elevated page->_refcount.
756 */
757 VM_BUG_ON_PAGE(page_ref_count(page) <= 0, page);
758 page_ref_inc(page);
759
760 if (unlikely(is_zone_device_page(page)))
761 get_zone_device_page(page);
762}
763
764static inline void put_page(struct page *page)
765{
766 page = compound_head(page);
767
768 if (put_page_testzero(page))
769 __put_page(page);
770
771 if (unlikely(is_zone_device_page(page)))
772 put_zone_device_page(page);
773}
774
775#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
776#define SECTION_IN_PAGE_FLAGS
777#endif
778
779/*
780 * The identification function is mainly used by the buddy allocator for
781 * determining if two pages could be buddies. We are not really identifying
782 * the zone since we could be using the section number id if we do not have
783 * node id available in page flags.
784 * We only guarantee that it will return the same value for two combinable
785 * pages in a zone.
786 */
787static inline int page_zone_id(struct page *page)
788{
789 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
790}
791
792static inline int zone_to_nid(struct zone *zone)
793{
794#ifdef CONFIG_NUMA
795 return zone->node;
796#else
797 return 0;
798#endif
799}
800
801#ifdef NODE_NOT_IN_PAGE_FLAGS
802extern int page_to_nid(const struct page *page);
803#else
804static inline int page_to_nid(const struct page *page)
805{
806 return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
807}
808#endif
809
810#ifdef CONFIG_NUMA_BALANCING
811static inline int cpu_pid_to_cpupid(int cpu, int pid)
812{
813 return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
814}
815
816static inline int cpupid_to_pid(int cpupid)
817{
818 return cpupid & LAST__PID_MASK;
819}
820
821static inline int cpupid_to_cpu(int cpupid)
822{
823 return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
824}
825
826static inline int cpupid_to_nid(int cpupid)
827{
828 return cpu_to_node(cpupid_to_cpu(cpupid));
829}
830
831static inline bool cpupid_pid_unset(int cpupid)
832{
833 return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
834}
835
836static inline bool cpupid_cpu_unset(int cpupid)
837{
838 return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
839}
840
841static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
842{
843 return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);
844}
845
846#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)
847#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
848static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
849{
850 return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);
851}
852
853static inline int page_cpupid_last(struct page *page)
854{
855 return page->_last_cpupid;
856}
857static inline void page_cpupid_reset_last(struct page *page)
858{
859 page->_last_cpupid = -1 & LAST_CPUPID_MASK;
860}
861#else
862static inline int page_cpupid_last(struct page *page)
863{
864 return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
865}
866
867extern int page_cpupid_xchg_last(struct page *page, int cpupid);
868
869static inline void page_cpupid_reset_last(struct page *page)
870{
871 page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
872}
873#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
874#else /* !CONFIG_NUMA_BALANCING */
875static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
876{
877 return page_to_nid(page); /* XXX */
878}
879
880static inline int page_cpupid_last(struct page *page)
881{
882 return page_to_nid(page); /* XXX */
883}
884
885static inline int cpupid_to_nid(int cpupid)
886{
887 return -1;
888}
889
890static inline int cpupid_to_pid(int cpupid)
891{
892 return -1;
893}
894
895static inline int cpupid_to_cpu(int cpupid)
896{
897 return -1;
898}
899
900static inline int cpu_pid_to_cpupid(int nid, int pid)
901{
902 return -1;
903}
904
905static inline bool cpupid_pid_unset(int cpupid)
906{
907 return 1;
908}
909
910static inline void page_cpupid_reset_last(struct page *page)
911{
912}
913
914static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
915{
916 return false;
917}
918#endif /* CONFIG_NUMA_BALANCING */
919
920static inline struct zone *page_zone(const struct page *page)
921{
922 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
923}
924
925static inline pg_data_t *page_pgdat(const struct page *page)
926{
927 return NODE_DATA(page_to_nid(page));
928}
929
930#ifdef SECTION_IN_PAGE_FLAGS
931static inline void set_page_section(struct page *page, unsigned long section)
932{
933 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
934 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
935}
936
937static inline unsigned long page_to_section(const struct page *page)
938{
939 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
940}
941#endif
942
943static inline void set_page_zone(struct page *page, enum zone_type zone)
944{
945 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
946 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
947}
948
949static inline void set_page_node(struct page *page, unsigned long node)
950{
951 page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
952 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
953}
954
955static inline void set_page_links(struct page *page, enum zone_type zone,
956 unsigned long node, unsigned long pfn)
957{
958 set_page_zone(page, zone);
959 set_page_node(page, node);
960#ifdef SECTION_IN_PAGE_FLAGS
961 set_page_section(page, pfn_to_section_nr(pfn));
962#endif
963}
964
965#ifdef CONFIG_MEMCG
966static inline struct mem_cgroup *page_memcg(struct page *page)
967{
968 return page->mem_cgroup;
969}
970static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
971{
972 WARN_ON_ONCE(!rcu_read_lock_held());
973 return READ_ONCE(page->mem_cgroup);
974}
975#else
976static inline struct mem_cgroup *page_memcg(struct page *page)
977{
978 return NULL;
979}
980static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
981{
982 WARN_ON_ONCE(!rcu_read_lock_held());
983 return NULL;
984}
985#endif
986
987/*
988 * Some inline functions in vmstat.h depend on page_zone()
989 */
990#include <linux/vmstat.h>
991
992static __always_inline void *lowmem_page_address(const struct page *page)
993{
994 return page_to_virt(page);
995}
996
997#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
998#define HASHED_PAGE_VIRTUAL
999#endif
1000
1001#if defined(WANT_PAGE_VIRTUAL)
1002static inline void *page_address(const struct page *page)
1003{
1004 return page->virtual;
1005}
1006static inline void set_page_address(struct page *page, void *address)
1007{
1008 page->virtual = address;
1009}
1010#define page_address_init() do { } while(0)
1011#endif
1012
1013#if defined(HASHED_PAGE_VIRTUAL)
1014void *page_address(const struct page *page);
1015void set_page_address(struct page *page, void *virtual);
1016void page_address_init(void);
1017#endif
1018
1019#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
1020#define page_address(page) lowmem_page_address(page)
1021#define set_page_address(page, address) do { } while(0)
1022#define page_address_init() do { } while(0)
1023#endif
1024
1025extern void *page_rmapping(struct page *page);
1026extern struct anon_vma *page_anon_vma(struct page *page);
1027extern struct address_space *page_mapping(struct page *page);
1028
1029extern struct address_space *__page_file_mapping(struct page *);
1030
1031static inline
1032struct address_space *page_file_mapping(struct page *page)
1033{
1034 if (unlikely(PageSwapCache(page)))
1035 return __page_file_mapping(page);
1036
1037 return page->mapping;
1038}
1039
1040extern pgoff_t __page_file_index(struct page *page);
1041
1042/*
1043 * Return the pagecache index of the passed page. Regular pagecache pages
1044 * use ->index whereas swapcache pages use swp_offset(->private)
1045 */
1046static inline pgoff_t page_index(struct page *page)
1047{
1048 if (unlikely(PageSwapCache(page)))
1049 return __page_file_index(page);
1050 return page->index;
1051}
1052
1053bool page_mapped(struct page *page);
1054struct address_space *page_mapping(struct page *page);
1055
1056/*
1057 * Return true only if the page has been allocated with
1058 * ALLOC_NO_WATERMARKS and the low watermark was not
1059 * met implying that the system is under some pressure.
1060 */
1061static inline bool page_is_pfmemalloc(struct page *page)
1062{
1063 /*
1064 * Page index cannot be this large so this must be
1065 * a pfmemalloc page.
1066 */
1067 return page->index == -1UL;
1068}
1069
1070/*
1071 * Only to be called by the page allocator on a freshly allocated
1072 * page.
1073 */
1074static inline void set_page_pfmemalloc(struct page *page)
1075{
1076 page->index = -1UL;
1077}
1078
1079static inline void clear_page_pfmemalloc(struct page *page)
1080{
1081 page->index = 0;
1082}
1083
1084/*
1085 * Different kinds of faults, as returned by handle_mm_fault().
1086 * Used to decide whether a process gets delivered SIGBUS or
1087 * just gets major/minor fault counters bumped up.
1088 */
1089
1090#define VM_FAULT_OOM 0x0001
1091#define VM_FAULT_SIGBUS 0x0002
1092#define VM_FAULT_MAJOR 0x0004
1093#define VM_FAULT_WRITE 0x0008 /* Special case for get_user_pages */
1094#define VM_FAULT_HWPOISON 0x0010 /* Hit poisoned small page */
1095#define VM_FAULT_HWPOISON_LARGE 0x0020 /* Hit poisoned large page. Index encoded in upper bits */
1096#define VM_FAULT_SIGSEGV 0x0040
1097
1098#define VM_FAULT_NOPAGE 0x0100 /* ->fault installed the pte, not return page */
1099#define VM_FAULT_LOCKED 0x0200 /* ->fault locked the returned page */
1100#define VM_FAULT_RETRY 0x0400 /* ->fault blocked, must retry */
1101#define VM_FAULT_FALLBACK 0x0800 /* huge page fault failed, fall back to small */
1102#define VM_FAULT_DONE_COW 0x1000 /* ->fault has fully handled COW */
1103
1104#define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */
1105
1106#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | \
1107 VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE | \
1108 VM_FAULT_FALLBACK)
1109
1110/* Encode hstate index for a hwpoisoned large page */
1111#define VM_FAULT_SET_HINDEX(x) ((x) << 12)
1112#define VM_FAULT_GET_HINDEX(x) (((x) >> 12) & 0xf)
1113
1114/*
1115 * Can be called by the pagefault handler when it gets a VM_FAULT_OOM.
1116 */
1117extern void pagefault_out_of_memory(void);
1118
1119#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
1120
1121/*
1122 * Flags passed to show_mem() and show_free_areas() to suppress output in
1123 * various contexts.
1124 */
1125#define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */
1126
1127extern void show_free_areas(unsigned int flags);
1128extern bool skip_free_areas_node(unsigned int flags, int nid);
1129
1130int shmem_zero_setup(struct vm_area_struct *);
1131#ifdef CONFIG_SHMEM
1132bool shmem_mapping(struct address_space *mapping);
1133#else
1134static inline bool shmem_mapping(struct address_space *mapping)
1135{
1136 return false;
1137}
1138#endif
1139
1140extern bool can_do_mlock(void);
1141extern int user_shm_lock(size_t, struct user_struct *);
1142extern void user_shm_unlock(size_t, struct user_struct *);
1143
1144/*
1145 * Parameter block passed down to zap_pte_range in exceptional cases.
1146 */
1147struct zap_details {
1148 struct address_space *check_mapping; /* Check page->mapping if set */
1149 pgoff_t first_index; /* Lowest page->index to unmap */
1150 pgoff_t last_index; /* Highest page->index to unmap */
1151 bool ignore_dirty; /* Ignore dirty pages */
1152 bool check_swap_entries; /* Check also swap entries */
1153};
1154
1155struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1156 pte_t pte);
1157struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
1158 pmd_t pmd);
1159
1160int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
1161 unsigned long size);
1162void zap_page_range(struct vm_area_struct *vma, unsigned long address,
1163 unsigned long size, struct zap_details *);
1164void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
1165 unsigned long start, unsigned long end);
1166
1167/**
1168 * mm_walk - callbacks for walk_page_range
1169 * @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry
1170 * this handler is required to be able to handle
1171 * pmd_trans_huge() pmds. They may simply choose to
1172 * split_huge_page() instead of handling it explicitly.
1173 * @pte_entry: if set, called for each non-empty PTE (4th-level) entry
1174 * @pte_hole: if set, called for each hole at all levels
1175 * @hugetlb_entry: if set, called for each hugetlb entry
1176 * @test_walk: caller specific callback function to determine whether
1177 * we walk over the current vma or not. Returning 0
1178 * value means "do page table walk over the current vma,"
1179 * and a negative one means "abort current page table walk
1180 * right now." 1 means "skip the current vma."
1181 * @mm: mm_struct representing the target process of page table walk
1182 * @vma: vma currently walked (NULL if walking outside vmas)
1183 * @private: private data for callbacks' usage
1184 *
1185 * (see the comment on walk_page_range() for more details)
1186 */
1187struct mm_walk {
1188 int (*pmd_entry)(pmd_t *pmd, unsigned long addr,
1189 unsigned long next, struct mm_walk *walk);
1190 int (*pte_entry)(pte_t *pte, unsigned long addr,
1191 unsigned long next, struct mm_walk *walk);
1192 int (*pte_hole)(unsigned long addr, unsigned long next,
1193 struct mm_walk *walk);
1194 int (*hugetlb_entry)(pte_t *pte, unsigned long hmask,
1195 unsigned long addr, unsigned long next,
1196 struct mm_walk *walk);
1197 int (*test_walk)(unsigned long addr, unsigned long next,
1198 struct mm_walk *walk);
1199 struct mm_struct *mm;
1200 struct vm_area_struct *vma;
1201 void *private;
1202};
1203
1204int walk_page_range(unsigned long addr, unsigned long end,
1205 struct mm_walk *walk);
1206int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk);
1207void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
1208 unsigned long end, unsigned long floor, unsigned long ceiling);
1209int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
1210 struct vm_area_struct *vma);
1211void unmap_mapping_range(struct address_space *mapping,
1212 loff_t const holebegin, loff_t const holelen, int even_cows);
1213int follow_pte_pmd(struct mm_struct *mm, unsigned long address,
1214 pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp);
1215int follow_pfn(struct vm_area_struct *vma, unsigned long address,
1216 unsigned long *pfn);
1217int follow_phys(struct vm_area_struct *vma, unsigned long address,
1218 unsigned int flags, unsigned long *prot, resource_size_t *phys);
1219int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
1220 void *buf, int len, int write);
1221
1222static inline void unmap_shared_mapping_range(struct address_space *mapping,
1223 loff_t const holebegin, loff_t const holelen)
1224{
1225 unmap_mapping_range(mapping, holebegin, holelen, 0);
1226}
1227
1228extern void truncate_pagecache(struct inode *inode, loff_t new);
1229extern void truncate_setsize(struct inode *inode, loff_t newsize);
1230void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
1231void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
1232int truncate_inode_page(struct address_space *mapping, struct page *page);
1233int generic_error_remove_page(struct address_space *mapping, struct page *page);
1234int invalidate_inode_page(struct page *page);
1235
1236#ifdef CONFIG_MMU
1237extern int handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
1238 unsigned int flags);
1239extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
1240 unsigned long address, unsigned int fault_flags,
1241 bool *unlocked);
1242#else
1243static inline int handle_mm_fault(struct vm_area_struct *vma,
1244 unsigned long address, unsigned int flags)
1245{
1246 /* should never happen if there's no MMU */
1247 BUG();
1248 return VM_FAULT_SIGBUS;
1249}
1250static inline int fixup_user_fault(struct task_struct *tsk,
1251 struct mm_struct *mm, unsigned long address,
1252 unsigned int fault_flags, bool *unlocked)
1253{
1254 /* should never happen if there's no MMU */
1255 BUG();
1256 return -EFAULT;
1257}
1258#endif
1259
1260extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len,
1261 unsigned int gup_flags);
1262extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
1263 void *buf, int len, unsigned int gup_flags);
1264extern int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
1265 unsigned long addr, void *buf, int len, unsigned int gup_flags);
1266
1267long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
1268 unsigned long start, unsigned long nr_pages,
1269 unsigned int gup_flags, struct page **pages,
1270 struct vm_area_struct **vmas, int *locked);
1271long get_user_pages(unsigned long start, unsigned long nr_pages,
1272 unsigned int gup_flags, struct page **pages,
1273 struct vm_area_struct **vmas);
1274long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
1275 unsigned int gup_flags, struct page **pages, int *locked);
1276long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
1277 struct page **pages, unsigned int gup_flags);
1278int get_user_pages_fast(unsigned long start, int nr_pages, int write,
1279 struct page **pages);
1280
1281/* Container for pinned pfns / pages */
1282struct frame_vector {
1283 unsigned int nr_allocated; /* Number of frames we have space for */
1284 unsigned int nr_frames; /* Number of frames stored in ptrs array */
1285 bool got_ref; /* Did we pin pages by getting page ref? */
1286 bool is_pfns; /* Does array contain pages or pfns? */
1287 void *ptrs[0]; /* Array of pinned pfns / pages. Use
1288 * pfns_vector_pages() or pfns_vector_pfns()
1289 * for access */
1290};
1291
1292struct frame_vector *frame_vector_create(unsigned int nr_frames);
1293void frame_vector_destroy(struct frame_vector *vec);
1294int get_vaddr_frames(unsigned long start, unsigned int nr_pfns,
1295 unsigned int gup_flags, struct frame_vector *vec);
1296void put_vaddr_frames(struct frame_vector *vec);
1297int frame_vector_to_pages(struct frame_vector *vec);
1298void frame_vector_to_pfns(struct frame_vector *vec);
1299
1300static inline unsigned int frame_vector_count(struct frame_vector *vec)
1301{
1302 return vec->nr_frames;
1303}
1304
1305static inline struct page **frame_vector_pages(struct frame_vector *vec)
1306{
1307 if (vec->is_pfns) {
1308 int err = frame_vector_to_pages(vec);
1309
1310 if (err)
1311 return ERR_PTR(err);
1312 }
1313 return (struct page **)(vec->ptrs);
1314}
1315
1316static inline unsigned long *frame_vector_pfns(struct frame_vector *vec)
1317{
1318 if (!vec->is_pfns)
1319 frame_vector_to_pfns(vec);
1320 return (unsigned long *)(vec->ptrs);
1321}
1322
1323struct kvec;
1324int get_kernel_pages(const struct kvec *iov, int nr_pages, int write,
1325 struct page **pages);
1326int get_kernel_page(unsigned long start, int write, struct page **pages);
1327struct page *get_dump_page(unsigned long addr);
1328
1329extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
1330extern void do_invalidatepage(struct page *page, unsigned int offset,
1331 unsigned int length);
1332
1333int __set_page_dirty_nobuffers(struct page *page);
1334int __set_page_dirty_no_writeback(struct page *page);
1335int redirty_page_for_writepage(struct writeback_control *wbc,
1336 struct page *page);
1337void account_page_dirtied(struct page *page, struct address_space *mapping);
1338void account_page_cleaned(struct page *page, struct address_space *mapping,
1339 struct bdi_writeback *wb);
1340int set_page_dirty(struct page *page);
1341int set_page_dirty_lock(struct page *page);
1342void cancel_dirty_page(struct page *page);
1343int clear_page_dirty_for_io(struct page *page);
1344
1345int get_cmdline(struct task_struct *task, char *buffer, int buflen);
1346
1347/* Is the vma a continuation of the stack vma above it? */
1348static inline int vma_growsdown(struct vm_area_struct *vma, unsigned long addr)
1349{
1350 return vma && (vma->vm_end == addr) && (vma->vm_flags & VM_GROWSDOWN);
1351}
1352
1353static inline bool vma_is_anonymous(struct vm_area_struct *vma)
1354{
1355 return !vma->vm_ops;
1356}
1357
1358static inline int stack_guard_page_start(struct vm_area_struct *vma,
1359 unsigned long addr)
1360{
1361 return (vma->vm_flags & VM_GROWSDOWN) &&
1362 (vma->vm_start == addr) &&
1363 !vma_growsdown(vma->vm_prev, addr);
1364}
1365
1366/* Is the vma a continuation of the stack vma below it? */
1367static inline int vma_growsup(struct vm_area_struct *vma, unsigned long addr)
1368{
1369 return vma && (vma->vm_start == addr) && (vma->vm_flags & VM_GROWSUP);
1370}
1371
1372static inline int stack_guard_page_end(struct vm_area_struct *vma,
1373 unsigned long addr)
1374{
1375 return (vma->vm_flags & VM_GROWSUP) &&
1376 (vma->vm_end == addr) &&
1377 !vma_growsup(vma->vm_next, addr);
1378}
1379
1380int vma_is_stack_for_current(struct vm_area_struct *vma);
1381
1382extern unsigned long move_page_tables(struct vm_area_struct *vma,
1383 unsigned long old_addr, struct vm_area_struct *new_vma,
1384 unsigned long new_addr, unsigned long len,
1385 bool need_rmap_locks);
1386extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
1387 unsigned long end, pgprot_t newprot,
1388 int dirty_accountable, int prot_numa);
1389extern int mprotect_fixup(struct vm_area_struct *vma,
1390 struct vm_area_struct **pprev, unsigned long start,
1391 unsigned long end, unsigned long newflags);
1392
1393/*
1394 * doesn't attempt to fault and will return short.
1395 */
1396int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
1397 struct page **pages);
1398/*
1399 * per-process(per-mm_struct) statistics.
1400 */
1401static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
1402{
1403 long val = atomic_long_read(&mm->rss_stat.count[member]);
1404
1405#ifdef SPLIT_RSS_COUNTING
1406 /*
1407 * counter is updated in asynchronous manner and may go to minus.
1408 * But it's never be expected number for users.
1409 */
1410 if (val < 0)
1411 val = 0;
1412#endif
1413 return (unsigned long)val;
1414}
1415
1416static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
1417{
1418 atomic_long_add(value, &mm->rss_stat.count[member]);
1419}
1420
1421static inline void inc_mm_counter(struct mm_struct *mm, int member)
1422{
1423 atomic_long_inc(&mm->rss_stat.count[member]);
1424}
1425
1426static inline void dec_mm_counter(struct mm_struct *mm, int member)
1427{
1428 atomic_long_dec(&mm->rss_stat.count[member]);
1429}
1430
1431/* Optimized variant when page is already known not to be PageAnon */
1432static inline int mm_counter_file(struct page *page)
1433{
1434 if (PageSwapBacked(page))
1435 return MM_SHMEMPAGES;
1436 return MM_FILEPAGES;
1437}
1438
1439static inline int mm_counter(struct page *page)
1440{
1441 if (PageAnon(page))
1442 return MM_ANONPAGES;
1443 return mm_counter_file(page);
1444}
1445
1446static inline unsigned long get_mm_rss(struct mm_struct *mm)
1447{
1448 return get_mm_counter(mm, MM_FILEPAGES) +
1449 get_mm_counter(mm, MM_ANONPAGES) +
1450 get_mm_counter(mm, MM_SHMEMPAGES);
1451}
1452
1453static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
1454{
1455 return max(mm->hiwater_rss, get_mm_rss(mm));
1456}
1457
1458static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
1459{
1460 return max(mm->hiwater_vm, mm->total_vm);
1461}
1462
1463static inline void update_hiwater_rss(struct mm_struct *mm)
1464{
1465 unsigned long _rss = get_mm_rss(mm);
1466
1467 if ((mm)->hiwater_rss < _rss)
1468 (mm)->hiwater_rss = _rss;
1469}
1470
1471static inline void update_hiwater_vm(struct mm_struct *mm)
1472{
1473 if (mm->hiwater_vm < mm->total_vm)
1474 mm->hiwater_vm = mm->total_vm;
1475}
1476
1477static inline void reset_mm_hiwater_rss(struct mm_struct *mm)
1478{
1479 mm->hiwater_rss = get_mm_rss(mm);
1480}
1481
1482static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
1483 struct mm_struct *mm)
1484{
1485 unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
1486
1487 if (*maxrss < hiwater_rss)
1488 *maxrss = hiwater_rss;
1489}
1490
1491#if defined(SPLIT_RSS_COUNTING)
1492void sync_mm_rss(struct mm_struct *mm);
1493#else
1494static inline void sync_mm_rss(struct mm_struct *mm)
1495{
1496}
1497#endif
1498
1499#ifndef __HAVE_ARCH_PTE_DEVMAP
1500static inline int pte_devmap(pte_t pte)
1501{
1502 return 0;
1503}
1504#endif
1505
1506int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot);
1507
1508extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
1509 spinlock_t **ptl);
1510static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
1511 spinlock_t **ptl)
1512{
1513 pte_t *ptep;
1514 __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
1515 return ptep;
1516}
1517
1518#ifdef __PAGETABLE_PUD_FOLDED
1519static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
1520 unsigned long address)
1521{
1522 return 0;
1523}
1524#else
1525int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
1526#endif
1527
1528#if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)
1529static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
1530 unsigned long address)
1531{
1532 return 0;
1533}
1534
1535static inline void mm_nr_pmds_init(struct mm_struct *mm) {}
1536
1537static inline unsigned long mm_nr_pmds(struct mm_struct *mm)
1538{
1539 return 0;
1540}
1541
1542static inline void mm_inc_nr_pmds(struct mm_struct *mm) {}
1543static inline void mm_dec_nr_pmds(struct mm_struct *mm) {}
1544
1545#else
1546int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
1547
1548static inline void mm_nr_pmds_init(struct mm_struct *mm)
1549{
1550 atomic_long_set(&mm->nr_pmds, 0);
1551}
1552
1553static inline unsigned long mm_nr_pmds(struct mm_struct *mm)
1554{
1555 return atomic_long_read(&mm->nr_pmds);
1556}
1557
1558static inline void mm_inc_nr_pmds(struct mm_struct *mm)
1559{
1560 atomic_long_inc(&mm->nr_pmds);
1561}
1562
1563static inline void mm_dec_nr_pmds(struct mm_struct *mm)
1564{
1565 atomic_long_dec(&mm->nr_pmds);
1566}
1567#endif
1568
1569int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
1570int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
1571
1572/*
1573 * The following ifdef needed to get the 4level-fixup.h header to work.
1574 * Remove it when 4level-fixup.h has been removed.
1575 */
1576#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
1577static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
1578{
1579 return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
1580 NULL: pud_offset(pgd, address);
1581}
1582
1583static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
1584{
1585 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
1586 NULL: pmd_offset(pud, address);
1587}
1588#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
1589
1590#if USE_SPLIT_PTE_PTLOCKS
1591#if ALLOC_SPLIT_PTLOCKS
1592void __init ptlock_cache_init(void);
1593extern bool ptlock_alloc(struct page *page);
1594extern void ptlock_free(struct page *page);
1595
1596static inline spinlock_t *ptlock_ptr(struct page *page)
1597{
1598 return page->ptl;
1599}
1600#else /* ALLOC_SPLIT_PTLOCKS */
1601static inline void ptlock_cache_init(void)
1602{
1603}
1604
1605static inline bool ptlock_alloc(struct page *page)
1606{
1607 return true;
1608}
1609
1610static inline void ptlock_free(struct page *page)
1611{
1612}
1613
1614static inline spinlock_t *ptlock_ptr(struct page *page)
1615{
1616 return &page->ptl;
1617}
1618#endif /* ALLOC_SPLIT_PTLOCKS */
1619
1620static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1621{
1622 return ptlock_ptr(pmd_page(*pmd));
1623}
1624
1625static inline bool ptlock_init(struct page *page)
1626{
1627 /*
1628 * prep_new_page() initialize page->private (and therefore page->ptl)
1629 * with 0. Make sure nobody took it in use in between.
1630 *
1631 * It can happen if arch try to use slab for page table allocation:
1632 * slab code uses page->slab_cache, which share storage with page->ptl.
1633 */
1634 VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page);
1635 if (!ptlock_alloc(page))
1636 return false;
1637 spin_lock_init(ptlock_ptr(page));
1638 return true;
1639}
1640
1641/* Reset page->mapping so free_pages_check won't complain. */
1642static inline void pte_lock_deinit(struct page *page)
1643{
1644 page->mapping = NULL;
1645 ptlock_free(page);
1646}
1647
1648#else /* !USE_SPLIT_PTE_PTLOCKS */
1649/*
1650 * We use mm->page_table_lock to guard all pagetable pages of the mm.
1651 */
1652static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1653{
1654 return &mm->page_table_lock;
1655}
1656static inline void ptlock_cache_init(void) {}
1657static inline bool ptlock_init(struct page *page) { return true; }
1658static inline void pte_lock_deinit(struct page *page) {}
1659#endif /* USE_SPLIT_PTE_PTLOCKS */
1660
1661static inline void pgtable_init(void)
1662{
1663 ptlock_cache_init();
1664 pgtable_cache_init();
1665}
1666
1667static inline bool pgtable_page_ctor(struct page *page)
1668{
1669 if (!ptlock_init(page))
1670 return false;
1671 inc_zone_page_state(page, NR_PAGETABLE);
1672 return true;
1673}
1674
1675static inline void pgtable_page_dtor(struct page *page)
1676{
1677 pte_lock_deinit(page);
1678 dec_zone_page_state(page, NR_PAGETABLE);
1679}
1680
1681#define pte_offset_map_lock(mm, pmd, address, ptlp) \
1682({ \
1683 spinlock_t *__ptl = pte_lockptr(mm, pmd); \
1684 pte_t *__pte = pte_offset_map(pmd, address); \
1685 *(ptlp) = __ptl; \
1686 spin_lock(__ptl); \
1687 __pte; \
1688})
1689
1690#define pte_unmap_unlock(pte, ptl) do { \
1691 spin_unlock(ptl); \
1692 pte_unmap(pte); \
1693} while (0)
1694
1695#define pte_alloc(mm, pmd, address) \
1696 (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd, address))
1697
1698#define pte_alloc_map(mm, pmd, address) \
1699 (pte_alloc(mm, pmd, address) ? NULL : pte_offset_map(pmd, address))
1700
1701#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
1702 (pte_alloc(mm, pmd, address) ? \
1703 NULL : pte_offset_map_lock(mm, pmd, address, ptlp))
1704
1705#define pte_alloc_kernel(pmd, address) \
1706 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
1707 NULL: pte_offset_kernel(pmd, address))
1708
1709#if USE_SPLIT_PMD_PTLOCKS
1710
1711static struct page *pmd_to_page(pmd_t *pmd)
1712{
1713 unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
1714 return virt_to_page((void *)((unsigned long) pmd & mask));
1715}
1716
1717static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1718{
1719 return ptlock_ptr(pmd_to_page(pmd));
1720}
1721
1722static inline bool pgtable_pmd_page_ctor(struct page *page)
1723{
1724#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1725 page->pmd_huge_pte = NULL;
1726#endif
1727 return ptlock_init(page);
1728}
1729
1730static inline void pgtable_pmd_page_dtor(struct page *page)
1731{
1732#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1733 VM_BUG_ON_PAGE(page->pmd_huge_pte, page);
1734#endif
1735 ptlock_free(page);
1736}
1737
1738#define pmd_huge_pte(mm, pmd) (pmd_to_page(pmd)->pmd_huge_pte)
1739
1740#else
1741
1742static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1743{
1744 return &mm->page_table_lock;
1745}
1746
1747static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }
1748static inline void pgtable_pmd_page_dtor(struct page *page) {}
1749
1750#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
1751
1752#endif
1753
1754static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
1755{
1756 spinlock_t *ptl = pmd_lockptr(mm, pmd);
1757 spin_lock(ptl);
1758 return ptl;
1759}
1760
1761extern void __init pagecache_init(void);
1762
1763extern void free_area_init(unsigned long * zones_size);
1764extern void free_area_init_node(int nid, unsigned long * zones_size,
1765 unsigned long zone_start_pfn, unsigned long *zholes_size);
1766extern void free_initmem(void);
1767
1768/*
1769 * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK)
1770 * into the buddy system. The freed pages will be poisoned with pattern
1771 * "poison" if it's within range [0, UCHAR_MAX].
1772 * Return pages freed into the buddy system.
1773 */
1774extern unsigned long free_reserved_area(void *start, void *end,
1775 int poison, char *s);
1776
1777#ifdef CONFIG_HIGHMEM
1778/*
1779 * Free a highmem page into the buddy system, adjusting totalhigh_pages
1780 * and totalram_pages.
1781 */
1782extern void free_highmem_page(struct page *page);
1783#endif
1784
1785extern void adjust_managed_page_count(struct page *page, long count);
1786extern void mem_init_print_info(const char *str);
1787
1788extern void reserve_bootmem_region(phys_addr_t start, phys_addr_t end);
1789
1790/* Free the reserved page into the buddy system, so it gets managed. */
1791static inline void __free_reserved_page(struct page *page)
1792{
1793 ClearPageReserved(page);
1794 init_page_count(page);
1795 __free_page(page);
1796}
1797
1798static inline void free_reserved_page(struct page *page)
1799{
1800 __free_reserved_page(page);
1801 adjust_managed_page_count(page, 1);
1802}
1803
1804static inline void mark_page_reserved(struct page *page)
1805{
1806 SetPageReserved(page);
1807 adjust_managed_page_count(page, -1);
1808}
1809
1810/*
1811 * Default method to free all the __init memory into the buddy system.
1812 * The freed pages will be poisoned with pattern "poison" if it's within
1813 * range [0, UCHAR_MAX].
1814 * Return pages freed into the buddy system.
1815 */
1816static inline unsigned long free_initmem_default(int poison)
1817{
1818 extern char __init_begin[], __init_end[];
1819
1820 return free_reserved_area(&__init_begin, &__init_end,
1821 poison, "unused kernel");
1822}
1823
1824static inline unsigned long get_num_physpages(void)
1825{
1826 int nid;
1827 unsigned long phys_pages = 0;
1828
1829 for_each_online_node(nid)
1830 phys_pages += node_present_pages(nid);
1831
1832 return phys_pages;
1833}
1834
1835#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1836/*
1837 * With CONFIG_HAVE_MEMBLOCK_NODE_MAP set, an architecture may initialise its
1838 * zones, allocate the backing mem_map and account for memory holes in a more
1839 * architecture independent manner. This is a substitute for creating the
1840 * zone_sizes[] and zholes_size[] arrays and passing them to
1841 * free_area_init_node()
1842 *
1843 * An architecture is expected to register range of page frames backed by
1844 * physical memory with memblock_add[_node]() before calling
1845 * free_area_init_nodes() passing in the PFN each zone ends at. At a basic
1846 * usage, an architecture is expected to do something like
1847 *
1848 * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn,
1849 * max_highmem_pfn};
1850 * for_each_valid_physical_page_range()
1851 * memblock_add_node(base, size, nid)
1852 * free_area_init_nodes(max_zone_pfns);
1853 *
1854 * free_bootmem_with_active_regions() calls free_bootmem_node() for each
1855 * registered physical page range. Similarly
1856 * sparse_memory_present_with_active_regions() calls memory_present() for
1857 * each range when SPARSEMEM is enabled.
1858 *
1859 * See mm/page_alloc.c for more information on each function exposed by
1860 * CONFIG_HAVE_MEMBLOCK_NODE_MAP.
1861 */
1862extern void free_area_init_nodes(unsigned long *max_zone_pfn);
1863unsigned long node_map_pfn_alignment(void);
1864unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn,
1865 unsigned long end_pfn);
1866extern unsigned long absent_pages_in_range(unsigned long start_pfn,
1867 unsigned long end_pfn);
1868extern void get_pfn_range_for_nid(unsigned int nid,
1869 unsigned long *start_pfn, unsigned long *end_pfn);
1870extern unsigned long find_min_pfn_with_active_regions(void);
1871extern void free_bootmem_with_active_regions(int nid,
1872 unsigned long max_low_pfn);
1873extern void sparse_memory_present_with_active_regions(int nid);
1874
1875#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
1876
1877#if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \
1878 !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
1879static inline int __early_pfn_to_nid(unsigned long pfn,
1880 struct mminit_pfnnid_cache *state)
1881{
1882 return 0;
1883}
1884#else
1885/* please see mm/page_alloc.c */
1886extern int __meminit early_pfn_to_nid(unsigned long pfn);
1887/* there is a per-arch backend function. */
1888extern int __meminit __early_pfn_to_nid(unsigned long pfn,
1889 struct mminit_pfnnid_cache *state);
1890#endif
1891
1892extern void set_dma_reserve(unsigned long new_dma_reserve);
1893extern void memmap_init_zone(unsigned long, int, unsigned long,
1894 unsigned long, enum memmap_context);
1895extern void setup_per_zone_wmarks(void);
1896extern int __meminit init_per_zone_wmark_min(void);
1897extern void mem_init(void);
1898extern void __init mmap_init(void);
1899extern void show_mem(unsigned int flags);
1900extern long si_mem_available(void);
1901extern void si_meminfo(struct sysinfo * val);
1902extern void si_meminfo_node(struct sysinfo *val, int nid);
1903#ifdef __HAVE_ARCH_RESERVED_KERNEL_PAGES
1904extern unsigned long arch_reserved_kernel_pages(void);
1905#endif
1906
1907extern __printf(2, 3)
1908void warn_alloc(gfp_t gfp_mask, const char *fmt, ...);
1909
1910extern void setup_per_cpu_pageset(void);
1911
1912extern void zone_pcp_update(struct zone *zone);
1913extern void zone_pcp_reset(struct zone *zone);
1914
1915/* page_alloc.c */
1916extern int min_free_kbytes;
1917extern int watermark_scale_factor;
1918
1919/* nommu.c */
1920extern atomic_long_t mmap_pages_allocated;
1921extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
1922
1923/* interval_tree.c */
1924void vma_interval_tree_insert(struct vm_area_struct *node,
1925 struct rb_root *root);
1926void vma_interval_tree_insert_after(struct vm_area_struct *node,
1927 struct vm_area_struct *prev,
1928 struct rb_root *root);
1929void vma_interval_tree_remove(struct vm_area_struct *node,
1930 struct rb_root *root);
1931struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root *root,
1932 unsigned long start, unsigned long last);
1933struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
1934 unsigned long start, unsigned long last);
1935
1936#define vma_interval_tree_foreach(vma, root, start, last) \
1937 for (vma = vma_interval_tree_iter_first(root, start, last); \
1938 vma; vma = vma_interval_tree_iter_next(vma, start, last))
1939
1940void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
1941 struct rb_root *root);
1942void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
1943 struct rb_root *root);
1944struct anon_vma_chain *anon_vma_interval_tree_iter_first(
1945 struct rb_root *root, unsigned long start, unsigned long last);
1946struct anon_vma_chain *anon_vma_interval_tree_iter_next(
1947 struct anon_vma_chain *node, unsigned long start, unsigned long last);
1948#ifdef CONFIG_DEBUG_VM_RB
1949void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
1950#endif
1951
1952#define anon_vma_interval_tree_foreach(avc, root, start, last) \
1953 for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
1954 avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
1955
1956/* mmap.c */
1957extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
1958extern int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
1959 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert,
1960 struct vm_area_struct *expand);
1961static inline int vma_adjust(struct vm_area_struct *vma, unsigned long start,
1962 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
1963{
1964 return __vma_adjust(vma, start, end, pgoff, insert, NULL);
1965}
1966extern struct vm_area_struct *vma_merge(struct mm_struct *,
1967 struct vm_area_struct *prev, unsigned long addr, unsigned long end,
1968 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
1969 struct mempolicy *, struct vm_userfaultfd_ctx);
1970extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
1971extern int split_vma(struct mm_struct *,
1972 struct vm_area_struct *, unsigned long addr, int new_below);
1973extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
1974extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
1975 struct rb_node **, struct rb_node *);
1976extern void unlink_file_vma(struct vm_area_struct *);
1977extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
1978 unsigned long addr, unsigned long len, pgoff_t pgoff,
1979 bool *need_rmap_locks);
1980extern void exit_mmap(struct mm_struct *);
1981
1982static inline int check_data_rlimit(unsigned long rlim,
1983 unsigned long new,
1984 unsigned long start,
1985 unsigned long end_data,
1986 unsigned long start_data)
1987{
1988 if (rlim < RLIM_INFINITY) {
1989 if (((new - start) + (end_data - start_data)) > rlim)
1990 return -ENOSPC;
1991 }
1992
1993 return 0;
1994}
1995
1996extern int mm_take_all_locks(struct mm_struct *mm);
1997extern void mm_drop_all_locks(struct mm_struct *mm);
1998
1999extern void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
2000extern struct file *get_mm_exe_file(struct mm_struct *mm);
2001extern struct file *get_task_exe_file(struct task_struct *task);
2002
2003extern bool may_expand_vm(struct mm_struct *, vm_flags_t, unsigned long npages);
2004extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages);
2005
2006extern bool vma_is_special_mapping(const struct vm_area_struct *vma,
2007 const struct vm_special_mapping *sm);
2008extern struct vm_area_struct *_install_special_mapping(struct mm_struct *mm,
2009 unsigned long addr, unsigned long len,
2010 unsigned long flags,
2011 const struct vm_special_mapping *spec);
2012/* This is an obsolete alternative to _install_special_mapping. */
2013extern int install_special_mapping(struct mm_struct *mm,
2014 unsigned long addr, unsigned long len,
2015 unsigned long flags, struct page **pages);
2016
2017extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
2018
2019extern unsigned long mmap_region(struct file *file, unsigned long addr,
2020 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff);
2021extern unsigned long do_mmap(struct file *file, unsigned long addr,
2022 unsigned long len, unsigned long prot, unsigned long flags,
2023 vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate);
2024extern int do_munmap(struct mm_struct *, unsigned long, size_t);
2025
2026static inline unsigned long
2027do_mmap_pgoff(struct file *file, unsigned long addr,
2028 unsigned long len, unsigned long prot, unsigned long flags,
2029 unsigned long pgoff, unsigned long *populate)
2030{
2031 return do_mmap(file, addr, len, prot, flags, 0, pgoff, populate);
2032}
2033
2034#ifdef CONFIG_MMU
2035extern int __mm_populate(unsigned long addr, unsigned long len,
2036 int ignore_errors);
2037static inline void mm_populate(unsigned long addr, unsigned long len)
2038{
2039 /* Ignore errors */
2040 (void) __mm_populate(addr, len, 1);
2041}
2042#else
2043static inline void mm_populate(unsigned long addr, unsigned long len) {}
2044#endif
2045
2046/* These take the mm semaphore themselves */
2047extern int __must_check vm_brk(unsigned long, unsigned long);
2048extern int vm_munmap(unsigned long, size_t);
2049extern unsigned long __must_check vm_mmap(struct file *, unsigned long,
2050 unsigned long, unsigned long,
2051 unsigned long, unsigned long);
2052
2053struct vm_unmapped_area_info {
2054#define VM_UNMAPPED_AREA_TOPDOWN 1
2055 unsigned long flags;
2056 unsigned long length;
2057 unsigned long low_limit;
2058 unsigned long high_limit;
2059 unsigned long align_mask;
2060 unsigned long align_offset;
2061};
2062
2063extern unsigned long unmapped_area(struct vm_unmapped_area_info *info);
2064extern unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);
2065
2066/*
2067 * Search for an unmapped address range.
2068 *
2069 * We are looking for a range that:
2070 * - does not intersect with any VMA;
2071 * - is contained within the [low_limit, high_limit) interval;
2072 * - is at least the desired size.
2073 * - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
2074 */
2075static inline unsigned long
2076vm_unmapped_area(struct vm_unmapped_area_info *info)
2077{
2078 if (info->flags & VM_UNMAPPED_AREA_TOPDOWN)
2079 return unmapped_area_topdown(info);
2080 else
2081 return unmapped_area(info);
2082}
2083
2084/* truncate.c */
2085extern void truncate_inode_pages(struct address_space *, loff_t);
2086extern void truncate_inode_pages_range(struct address_space *,
2087 loff_t lstart, loff_t lend);
2088extern void truncate_inode_pages_final(struct address_space *);
2089
2090/* generic vm_area_ops exported for stackable file systems */
2091extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
2092extern void filemap_map_pages(struct vm_fault *vmf,
2093 pgoff_t start_pgoff, pgoff_t end_pgoff);
2094extern int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf);
2095
2096/* mm/page-writeback.c */
2097int write_one_page(struct page *page, int wait);
2098void task_dirty_inc(struct task_struct *tsk);
2099
2100/* readahead.c */
2101#define VM_MAX_READAHEAD 128 /* kbytes */
2102#define VM_MIN_READAHEAD 16 /* kbytes (includes current page) */
2103
2104int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
2105 pgoff_t offset, unsigned long nr_to_read);
2106
2107void page_cache_sync_readahead(struct address_space *mapping,
2108 struct file_ra_state *ra,
2109 struct file *filp,
2110 pgoff_t offset,
2111 unsigned long size);
2112
2113void page_cache_async_readahead(struct address_space *mapping,
2114 struct file_ra_state *ra,
2115 struct file *filp,
2116 struct page *pg,
2117 pgoff_t offset,
2118 unsigned long size);
2119
2120/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
2121extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
2122
2123/* CONFIG_STACK_GROWSUP still needs to to grow downwards at some places */
2124extern int expand_downwards(struct vm_area_struct *vma,
2125 unsigned long address);
2126#if VM_GROWSUP
2127extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
2128#else
2129 #define expand_upwards(vma, address) (0)
2130#endif
2131
2132/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
2133extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
2134extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
2135 struct vm_area_struct **pprev);
2136
2137/* Look up the first VMA which intersects the interval start_addr..end_addr-1,
2138 NULL if none. Assume start_addr < end_addr. */
2139static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
2140{
2141 struct vm_area_struct * vma = find_vma(mm,start_addr);
2142
2143 if (vma && end_addr <= vma->vm_start)
2144 vma = NULL;
2145 return vma;
2146}
2147
2148static inline unsigned long vma_pages(struct vm_area_struct *vma)
2149{
2150 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
2151}
2152
2153/* Look up the first VMA which exactly match the interval vm_start ... vm_end */
2154static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
2155 unsigned long vm_start, unsigned long vm_end)
2156{
2157 struct vm_area_struct *vma = find_vma(mm, vm_start);
2158
2159 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
2160 vma = NULL;
2161
2162 return vma;
2163}
2164
2165#ifdef CONFIG_MMU
2166pgprot_t vm_get_page_prot(unsigned long vm_flags);
2167void vma_set_page_prot(struct vm_area_struct *vma);
2168#else
2169static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
2170{
2171 return __pgprot(0);
2172}
2173static inline void vma_set_page_prot(struct vm_area_struct *vma)
2174{
2175 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
2176}
2177#endif
2178
2179#ifdef CONFIG_NUMA_BALANCING
2180unsigned long change_prot_numa(struct vm_area_struct *vma,
2181 unsigned long start, unsigned long end);
2182#endif
2183
2184struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
2185int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
2186 unsigned long pfn, unsigned long size, pgprot_t);
2187int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
2188int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2189 unsigned long pfn);
2190int vm_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
2191 unsigned long pfn, pgprot_t pgprot);
2192int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
2193 pfn_t pfn);
2194int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);
2195
2196
2197struct page *follow_page_mask(struct vm_area_struct *vma,
2198 unsigned long address, unsigned int foll_flags,
2199 unsigned int *page_mask);
2200
2201static inline struct page *follow_page(struct vm_area_struct *vma,
2202 unsigned long address, unsigned int foll_flags)
2203{
2204 unsigned int unused_page_mask;
2205 return follow_page_mask(vma, address, foll_flags, &unused_page_mask);
2206}
2207
2208#define FOLL_WRITE 0x01 /* check pte is writable */
2209#define FOLL_TOUCH 0x02 /* mark page accessed */
2210#define FOLL_GET 0x04 /* do get_page on page */
2211#define FOLL_DUMP 0x08 /* give error on hole if it would be zero */
2212#define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */
2213#define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO
2214 * and return without waiting upon it */
2215#define FOLL_POPULATE 0x40 /* fault in page */
2216#define FOLL_SPLIT 0x80 /* don't return transhuge pages, split them */
2217#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
2218#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
2219#define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
2220#define FOLL_TRIED 0x800 /* a retry, previous pass started an IO */
2221#define FOLL_MLOCK 0x1000 /* lock present pages */
2222#define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */
2223#define FOLL_COW 0x4000 /* internal GUP flag */
2224
2225typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
2226 void *data);
2227extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
2228 unsigned long size, pte_fn_t fn, void *data);
2229
2230
2231#ifdef CONFIG_PAGE_POISONING
2232extern bool page_poisoning_enabled(void);
2233extern void kernel_poison_pages(struct page *page, int numpages, int enable);
2234extern bool page_is_poisoned(struct page *page);
2235#else
2236static inline bool page_poisoning_enabled(void) { return false; }
2237static inline void kernel_poison_pages(struct page *page, int numpages,
2238 int enable) { }
2239static inline bool page_is_poisoned(struct page *page) { return false; }
2240#endif
2241
2242#ifdef CONFIG_DEBUG_PAGEALLOC
2243extern bool _debug_pagealloc_enabled;
2244extern void __kernel_map_pages(struct page *page, int numpages, int enable);
2245
2246static inline bool debug_pagealloc_enabled(void)
2247{
2248 return _debug_pagealloc_enabled;
2249}
2250
2251static inline void
2252kernel_map_pages(struct page *page, int numpages, int enable)
2253{
2254 if (!debug_pagealloc_enabled())
2255 return;
2256
2257 __kernel_map_pages(page, numpages, enable);
2258}
2259#ifdef CONFIG_HIBERNATION
2260extern bool kernel_page_present(struct page *page);
2261#endif /* CONFIG_HIBERNATION */
2262#else /* CONFIG_DEBUG_PAGEALLOC */
2263static inline void
2264kernel_map_pages(struct page *page, int numpages, int enable) {}
2265#ifdef CONFIG_HIBERNATION
2266static inline bool kernel_page_present(struct page *page) { return true; }
2267#endif /* CONFIG_HIBERNATION */
2268static inline bool debug_pagealloc_enabled(void)
2269{
2270 return false;
2271}
2272#endif /* CONFIG_DEBUG_PAGEALLOC */
2273
2274#ifdef __HAVE_ARCH_GATE_AREA
2275extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);
2276extern int in_gate_area_no_mm(unsigned long addr);
2277extern int in_gate_area(struct mm_struct *mm, unsigned long addr);
2278#else
2279static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
2280{
2281 return NULL;
2282}
2283static inline int in_gate_area_no_mm(unsigned long addr) { return 0; }
2284static inline int in_gate_area(struct mm_struct *mm, unsigned long addr)
2285{
2286 return 0;
2287}
2288#endif /* __HAVE_ARCH_GATE_AREA */
2289
2290extern bool process_shares_mm(struct task_struct *p, struct mm_struct *mm);
2291
2292#ifdef CONFIG_SYSCTL
2293extern int sysctl_drop_caches;
2294int drop_caches_sysctl_handler(struct ctl_table *, int,
2295 void __user *, size_t *, loff_t *);
2296#endif
2297
2298void drop_slab(void);
2299void drop_slab_node(int nid);
2300
2301#ifndef CONFIG_MMU
2302#define randomize_va_space 0
2303#else
2304extern int randomize_va_space;
2305#endif
2306
2307const char * arch_vma_name(struct vm_area_struct *vma);
2308void print_vma_addr(char *prefix, unsigned long rip);
2309
2310void sparse_mem_maps_populate_node(struct page **map_map,
2311 unsigned long pnum_begin,
2312 unsigned long pnum_end,
2313 unsigned long map_count,
2314 int nodeid);
2315
2316struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
2317pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
2318pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
2319pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
2320pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
2321void *vmemmap_alloc_block(unsigned long size, int node);
2322struct vmem_altmap;
2323void *__vmemmap_alloc_block_buf(unsigned long size, int node,
2324 struct vmem_altmap *altmap);
2325static inline void *vmemmap_alloc_block_buf(unsigned long size, int node)
2326{
2327 return __vmemmap_alloc_block_buf(size, node, NULL);
2328}
2329
2330void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
2331int vmemmap_populate_basepages(unsigned long start, unsigned long end,
2332 int node);
2333int vmemmap_populate(unsigned long start, unsigned long end, int node);
2334void vmemmap_populate_print_last(void);
2335#ifdef CONFIG_MEMORY_HOTPLUG
2336void vmemmap_free(unsigned long start, unsigned long end);
2337#endif
2338void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
2339 unsigned long size);
2340
2341enum mf_flags {
2342 MF_COUNT_INCREASED = 1 << 0,
2343 MF_ACTION_REQUIRED = 1 << 1,
2344 MF_MUST_KILL = 1 << 2,
2345 MF_SOFT_OFFLINE = 1 << 3,
2346};
2347extern int memory_failure(unsigned long pfn, int trapno, int flags);
2348extern void memory_failure_queue(unsigned long pfn, int trapno, int flags);
2349extern int unpoison_memory(unsigned long pfn);
2350extern int get_hwpoison_page(struct page *page);
2351#define put_hwpoison_page(page) put_page(page)
2352extern int sysctl_memory_failure_early_kill;
2353extern int sysctl_memory_failure_recovery;
2354extern void shake_page(struct page *p, int access);
2355extern atomic_long_t num_poisoned_pages;
2356extern int soft_offline_page(struct page *page, int flags);
2357
2358
2359/*
2360 * Error handlers for various types of pages.
2361 */
2362enum mf_result {
2363 MF_IGNORED, /* Error: cannot be handled */
2364 MF_FAILED, /* Error: handling failed */
2365 MF_DELAYED, /* Will be handled later */
2366 MF_RECOVERED, /* Successfully recovered */
2367};
2368
2369enum mf_action_page_type {
2370 MF_MSG_KERNEL,
2371 MF_MSG_KERNEL_HIGH_ORDER,
2372 MF_MSG_SLAB,
2373 MF_MSG_DIFFERENT_COMPOUND,
2374 MF_MSG_POISONED_HUGE,
2375 MF_MSG_HUGE,
2376 MF_MSG_FREE_HUGE,
2377 MF_MSG_UNMAP_FAILED,
2378 MF_MSG_DIRTY_SWAPCACHE,
2379 MF_MSG_CLEAN_SWAPCACHE,
2380 MF_MSG_DIRTY_MLOCKED_LRU,
2381 MF_MSG_CLEAN_MLOCKED_LRU,
2382 MF_MSG_DIRTY_UNEVICTABLE_LRU,
2383 MF_MSG_CLEAN_UNEVICTABLE_LRU,
2384 MF_MSG_DIRTY_LRU,
2385 MF_MSG_CLEAN_LRU,
2386 MF_MSG_TRUNCATED_LRU,
2387 MF_MSG_BUDDY,
2388 MF_MSG_BUDDY_2ND,
2389 MF_MSG_UNKNOWN,
2390};
2391
2392#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
2393extern void clear_huge_page(struct page *page,
2394 unsigned long addr,
2395 unsigned int pages_per_huge_page);
2396extern void copy_user_huge_page(struct page *dst, struct page *src,
2397 unsigned long addr, struct vm_area_struct *vma,
2398 unsigned int pages_per_huge_page);
2399#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
2400
2401extern struct page_ext_operations debug_guardpage_ops;
2402extern struct page_ext_operations page_poisoning_ops;
2403
2404#ifdef CONFIG_DEBUG_PAGEALLOC
2405extern unsigned int _debug_guardpage_minorder;
2406extern bool _debug_guardpage_enabled;
2407
2408static inline unsigned int debug_guardpage_minorder(void)
2409{
2410 return _debug_guardpage_minorder;
2411}
2412
2413static inline bool debug_guardpage_enabled(void)
2414{
2415 return _debug_guardpage_enabled;
2416}
2417
2418static inline bool page_is_guard(struct page *page)
2419{
2420 struct page_ext *page_ext;
2421
2422 if (!debug_guardpage_enabled())
2423 return false;
2424
2425 page_ext = lookup_page_ext(page);
2426 if (unlikely(!page_ext))
2427 return false;
2428
2429 return test_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
2430}
2431#else
2432static inline unsigned int debug_guardpage_minorder(void) { return 0; }
2433static inline bool debug_guardpage_enabled(void) { return false; }
2434static inline bool page_is_guard(struct page *page) { return false; }
2435#endif /* CONFIG_DEBUG_PAGEALLOC */
2436
2437#if MAX_NUMNODES > 1
2438void __init setup_nr_node_ids(void);
2439#else
2440static inline void setup_nr_node_ids(void) {}
2441#endif
2442
2443#endif /* __KERNEL__ */
2444#endif /* _LINUX_MM_H */
1#ifndef _LINUX_MM_H
2#define _LINUX_MM_H
3
4#include <linux/errno.h>
5
6#ifdef __KERNEL__
7
8#include <linux/mmdebug.h>
9#include <linux/gfp.h>
10#include <linux/bug.h>
11#include <linux/list.h>
12#include <linux/mmzone.h>
13#include <linux/rbtree.h>
14#include <linux/atomic.h>
15#include <linux/debug_locks.h>
16#include <linux/mm_types.h>
17#include <linux/range.h>
18#include <linux/pfn.h>
19#include <linux/percpu-refcount.h>
20#include <linux/bit_spinlock.h>
21#include <linux/shrinker.h>
22#include <linux/resource.h>
23#include <linux/page_ext.h>
24#include <linux/err.h>
25#include <linux/page_ref.h>
26
27struct mempolicy;
28struct anon_vma;
29struct anon_vma_chain;
30struct file_ra_state;
31struct user_struct;
32struct writeback_control;
33struct bdi_writeback;
34
35#ifndef CONFIG_NEED_MULTIPLE_NODES /* Don't use mapnrs, do it properly */
36extern unsigned long max_mapnr;
37
38static inline void set_max_mapnr(unsigned long limit)
39{
40 max_mapnr = limit;
41}
42#else
43static inline void set_max_mapnr(unsigned long limit) { }
44#endif
45
46extern unsigned long totalram_pages;
47extern void * high_memory;
48extern int page_cluster;
49
50#ifdef CONFIG_SYSCTL
51extern int sysctl_legacy_va_layout;
52#else
53#define sysctl_legacy_va_layout 0
54#endif
55
56#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
57extern const int mmap_rnd_bits_min;
58extern const int mmap_rnd_bits_max;
59extern int mmap_rnd_bits __read_mostly;
60#endif
61#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
62extern const int mmap_rnd_compat_bits_min;
63extern const int mmap_rnd_compat_bits_max;
64extern int mmap_rnd_compat_bits __read_mostly;
65#endif
66
67#include <asm/page.h>
68#include <asm/pgtable.h>
69#include <asm/processor.h>
70
71#ifndef __pa_symbol
72#define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0))
73#endif
74
75/*
76 * To prevent common memory management code establishing
77 * a zero page mapping on a read fault.
78 * This macro should be defined within <asm/pgtable.h>.
79 * s390 does this to prevent multiplexing of hardware bits
80 * related to the physical page in case of virtualization.
81 */
82#ifndef mm_forbids_zeropage
83#define mm_forbids_zeropage(X) (0)
84#endif
85
86/*
87 * Default maximum number of active map areas, this limits the number of vmas
88 * per mm struct. Users can overwrite this number by sysctl but there is a
89 * problem.
90 *
91 * When a program's coredump is generated as ELF format, a section is created
92 * per a vma. In ELF, the number of sections is represented in unsigned short.
93 * This means the number of sections should be smaller than 65535 at coredump.
94 * Because the kernel adds some informative sections to a image of program at
95 * generating coredump, we need some margin. The number of extra sections is
96 * 1-3 now and depends on arch. We use "5" as safe margin, here.
97 *
98 * ELF extended numbering allows more than 65535 sections, so 16-bit bound is
99 * not a hard limit any more. Although some userspace tools can be surprised by
100 * that.
101 */
102#define MAPCOUNT_ELF_CORE_MARGIN (5)
103#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
104
105extern int sysctl_max_map_count;
106
107extern unsigned long sysctl_user_reserve_kbytes;
108extern unsigned long sysctl_admin_reserve_kbytes;
109
110extern int sysctl_overcommit_memory;
111extern int sysctl_overcommit_ratio;
112extern unsigned long sysctl_overcommit_kbytes;
113
114extern int overcommit_ratio_handler(struct ctl_table *, int, void __user *,
115 size_t *, loff_t *);
116extern int overcommit_kbytes_handler(struct ctl_table *, int, void __user *,
117 size_t *, loff_t *);
118
119#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
120
121/* to align the pointer to the (next) page boundary */
122#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
123
124/* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */
125#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)addr, PAGE_SIZE)
126
127/*
128 * Linux kernel virtual memory manager primitives.
129 * The idea being to have a "virtual" mm in the same way
130 * we have a virtual fs - giving a cleaner interface to the
131 * mm details, and allowing different kinds of memory mappings
132 * (from shared memory to executable loading to arbitrary
133 * mmap() functions).
134 */
135
136extern struct kmem_cache *vm_area_cachep;
137
138#ifndef CONFIG_MMU
139extern struct rb_root nommu_region_tree;
140extern struct rw_semaphore nommu_region_sem;
141
142extern unsigned int kobjsize(const void *objp);
143#endif
144
145/*
146 * vm_flags in vm_area_struct, see mm_types.h.
147 * When changing, update also include/trace/events/mmflags.h
148 */
149#define VM_NONE 0x00000000
150
151#define VM_READ 0x00000001 /* currently active flags */
152#define VM_WRITE 0x00000002
153#define VM_EXEC 0x00000004
154#define VM_SHARED 0x00000008
155
156/* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */
157#define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */
158#define VM_MAYWRITE 0x00000020
159#define VM_MAYEXEC 0x00000040
160#define VM_MAYSHARE 0x00000080
161
162#define VM_GROWSDOWN 0x00000100 /* general info on the segment */
163#define VM_UFFD_MISSING 0x00000200 /* missing pages tracking */
164#define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
165#define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */
166#define VM_UFFD_WP 0x00001000 /* wrprotect pages tracking */
167
168#define VM_LOCKED 0x00002000
169#define VM_IO 0x00004000 /* Memory mapped I/O or similar */
170
171 /* Used by sys_madvise() */
172#define VM_SEQ_READ 0x00008000 /* App will access data sequentially */
173#define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */
174
175#define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */
176#define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */
177#define VM_LOCKONFAULT 0x00080000 /* Lock the pages covered when they are faulted in */
178#define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */
179#define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */
180#define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
181#define VM_ARCH_1 0x01000000 /* Architecture-specific flag */
182#define VM_ARCH_2 0x02000000
183#define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */
184
185#ifdef CONFIG_MEM_SOFT_DIRTY
186# define VM_SOFTDIRTY 0x08000000 /* Not soft dirty clean area */
187#else
188# define VM_SOFTDIRTY 0
189#endif
190
191#define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
192#define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */
193#define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */
194#define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */
195
196#ifdef CONFIG_ARCH_USES_HIGH_VMA_FLAGS
197#define VM_HIGH_ARCH_BIT_0 32 /* bit only usable on 64-bit architectures */
198#define VM_HIGH_ARCH_BIT_1 33 /* bit only usable on 64-bit architectures */
199#define VM_HIGH_ARCH_BIT_2 34 /* bit only usable on 64-bit architectures */
200#define VM_HIGH_ARCH_BIT_3 35 /* bit only usable on 64-bit architectures */
201#define VM_HIGH_ARCH_0 BIT(VM_HIGH_ARCH_BIT_0)
202#define VM_HIGH_ARCH_1 BIT(VM_HIGH_ARCH_BIT_1)
203#define VM_HIGH_ARCH_2 BIT(VM_HIGH_ARCH_BIT_2)
204#define VM_HIGH_ARCH_3 BIT(VM_HIGH_ARCH_BIT_3)
205#endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */
206
207#if defined(CONFIG_X86)
208# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
209#if defined (CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS)
210# define VM_PKEY_SHIFT VM_HIGH_ARCH_BIT_0
211# define VM_PKEY_BIT0 VM_HIGH_ARCH_0 /* A protection key is a 4-bit value */
212# define VM_PKEY_BIT1 VM_HIGH_ARCH_1
213# define VM_PKEY_BIT2 VM_HIGH_ARCH_2
214# define VM_PKEY_BIT3 VM_HIGH_ARCH_3
215#endif
216#elif defined(CONFIG_PPC)
217# define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */
218#elif defined(CONFIG_PARISC)
219# define VM_GROWSUP VM_ARCH_1
220#elif defined(CONFIG_METAG)
221# define VM_GROWSUP VM_ARCH_1
222#elif defined(CONFIG_IA64)
223# define VM_GROWSUP VM_ARCH_1
224#elif !defined(CONFIG_MMU)
225# define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */
226#endif
227
228#if defined(CONFIG_X86)
229/* MPX specific bounds table or bounds directory */
230# define VM_MPX VM_ARCH_2
231#endif
232
233#ifndef VM_GROWSUP
234# define VM_GROWSUP VM_NONE
235#endif
236
237/* Bits set in the VMA until the stack is in its final location */
238#define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ)
239
240#ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */
241#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
242#endif
243
244#ifdef CONFIG_STACK_GROWSUP
245#define VM_STACK VM_GROWSUP
246#else
247#define VM_STACK VM_GROWSDOWN
248#endif
249
250#define VM_STACK_FLAGS (VM_STACK | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
251
252/*
253 * Special vmas that are non-mergable, non-mlock()able.
254 * Note: mm/huge_memory.c VM_NO_THP depends on this definition.
255 */
256#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP)
257
258/* This mask defines which mm->def_flags a process can inherit its parent */
259#define VM_INIT_DEF_MASK VM_NOHUGEPAGE
260
261/* This mask is used to clear all the VMA flags used by mlock */
262#define VM_LOCKED_CLEAR_MASK (~(VM_LOCKED | VM_LOCKONFAULT))
263
264/*
265 * mapping from the currently active vm_flags protection bits (the
266 * low four bits) to a page protection mask..
267 */
268extern pgprot_t protection_map[16];
269
270#define FAULT_FLAG_WRITE 0x01 /* Fault was a write access */
271#define FAULT_FLAG_MKWRITE 0x02 /* Fault was mkwrite of existing pte */
272#define FAULT_FLAG_ALLOW_RETRY 0x04 /* Retry fault if blocking */
273#define FAULT_FLAG_RETRY_NOWAIT 0x08 /* Don't drop mmap_sem and wait when retrying */
274#define FAULT_FLAG_KILLABLE 0x10 /* The fault task is in SIGKILL killable region */
275#define FAULT_FLAG_TRIED 0x20 /* Second try */
276#define FAULT_FLAG_USER 0x40 /* The fault originated in userspace */
277#define FAULT_FLAG_REMOTE 0x80 /* faulting for non current tsk/mm */
278#define FAULT_FLAG_INSTRUCTION 0x100 /* The fault was during an instruction fetch */
279
280/*
281 * vm_fault is filled by the the pagefault handler and passed to the vma's
282 * ->fault function. The vma's ->fault is responsible for returning a bitmask
283 * of VM_FAULT_xxx flags that give details about how the fault was handled.
284 *
285 * MM layer fills up gfp_mask for page allocations but fault handler might
286 * alter it if its implementation requires a different allocation context.
287 *
288 * pgoff should be used in favour of virtual_address, if possible.
289 */
290struct vm_fault {
291 unsigned int flags; /* FAULT_FLAG_xxx flags */
292 gfp_t gfp_mask; /* gfp mask to be used for allocations */
293 pgoff_t pgoff; /* Logical page offset based on vma */
294 void __user *virtual_address; /* Faulting virtual address */
295
296 struct page *cow_page; /* Handler may choose to COW */
297 struct page *page; /* ->fault handlers should return a
298 * page here, unless VM_FAULT_NOPAGE
299 * is set (which is also implied by
300 * VM_FAULT_ERROR).
301 */
302 /* for ->map_pages() only */
303 pgoff_t max_pgoff; /* map pages for offset from pgoff till
304 * max_pgoff inclusive */
305 pte_t *pte; /* pte entry associated with ->pgoff */
306};
307
308/*
309 * These are the virtual MM functions - opening of an area, closing and
310 * unmapping it (needed to keep files on disk up-to-date etc), pointer
311 * to the functions called when a no-page or a wp-page exception occurs.
312 */
313struct vm_operations_struct {
314 void (*open)(struct vm_area_struct * area);
315 void (*close)(struct vm_area_struct * area);
316 int (*mremap)(struct vm_area_struct * area);
317 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
318 int (*pmd_fault)(struct vm_area_struct *, unsigned long address,
319 pmd_t *, unsigned int flags);
320 void (*map_pages)(struct vm_area_struct *vma, struct vm_fault *vmf);
321
322 /* notification that a previously read-only page is about to become
323 * writable, if an error is returned it will cause a SIGBUS */
324 int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
325
326 /* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */
327 int (*pfn_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
328
329 /* called by access_process_vm when get_user_pages() fails, typically
330 * for use by special VMAs that can switch between memory and hardware
331 */
332 int (*access)(struct vm_area_struct *vma, unsigned long addr,
333 void *buf, int len, int write);
334
335 /* Called by the /proc/PID/maps code to ask the vma whether it
336 * has a special name. Returning non-NULL will also cause this
337 * vma to be dumped unconditionally. */
338 const char *(*name)(struct vm_area_struct *vma);
339
340#ifdef CONFIG_NUMA
341 /*
342 * set_policy() op must add a reference to any non-NULL @new mempolicy
343 * to hold the policy upon return. Caller should pass NULL @new to
344 * remove a policy and fall back to surrounding context--i.e. do not
345 * install a MPOL_DEFAULT policy, nor the task or system default
346 * mempolicy.
347 */
348 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
349
350 /*
351 * get_policy() op must add reference [mpol_get()] to any policy at
352 * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure
353 * in mm/mempolicy.c will do this automatically.
354 * get_policy() must NOT add a ref if the policy at (vma,addr) is not
355 * marked as MPOL_SHARED. vma policies are protected by the mmap_sem.
356 * If no [shared/vma] mempolicy exists at the addr, get_policy() op
357 * must return NULL--i.e., do not "fallback" to task or system default
358 * policy.
359 */
360 struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
361 unsigned long addr);
362#endif
363 /*
364 * Called by vm_normal_page() for special PTEs to find the
365 * page for @addr. This is useful if the default behavior
366 * (using pte_page()) would not find the correct page.
367 */
368 struct page *(*find_special_page)(struct vm_area_struct *vma,
369 unsigned long addr);
370};
371
372struct mmu_gather;
373struct inode;
374
375#define page_private(page) ((page)->private)
376#define set_page_private(page, v) ((page)->private = (v))
377
378#if !defined(__HAVE_ARCH_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE)
379static inline int pmd_devmap(pmd_t pmd)
380{
381 return 0;
382}
383#endif
384
385/*
386 * FIXME: take this include out, include page-flags.h in
387 * files which need it (119 of them)
388 */
389#include <linux/page-flags.h>
390#include <linux/huge_mm.h>
391
392/*
393 * Methods to modify the page usage count.
394 *
395 * What counts for a page usage:
396 * - cache mapping (page->mapping)
397 * - private data (page->private)
398 * - page mapped in a task's page tables, each mapping
399 * is counted separately
400 *
401 * Also, many kernel routines increase the page count before a critical
402 * routine so they can be sure the page doesn't go away from under them.
403 */
404
405/*
406 * Drop a ref, return true if the refcount fell to zero (the page has no users)
407 */
408static inline int put_page_testzero(struct page *page)
409{
410 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
411 return page_ref_dec_and_test(page);
412}
413
414/*
415 * Try to grab a ref unless the page has a refcount of zero, return false if
416 * that is the case.
417 * This can be called when MMU is off so it must not access
418 * any of the virtual mappings.
419 */
420static inline int get_page_unless_zero(struct page *page)
421{
422 return page_ref_add_unless(page, 1, 0);
423}
424
425extern int page_is_ram(unsigned long pfn);
426
427enum {
428 REGION_INTERSECTS,
429 REGION_DISJOINT,
430 REGION_MIXED,
431};
432
433int region_intersects(resource_size_t offset, size_t size, unsigned long flags,
434 unsigned long desc);
435
436/* Support for virtually mapped pages */
437struct page *vmalloc_to_page(const void *addr);
438unsigned long vmalloc_to_pfn(const void *addr);
439
440/*
441 * Determine if an address is within the vmalloc range
442 *
443 * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there
444 * is no special casing required.
445 */
446static inline int is_vmalloc_addr(const void *x)
447{
448#ifdef CONFIG_MMU
449 unsigned long addr = (unsigned long)x;
450
451 return addr >= VMALLOC_START && addr < VMALLOC_END;
452#else
453 return 0;
454#endif
455}
456#ifdef CONFIG_MMU
457extern int is_vmalloc_or_module_addr(const void *x);
458#else
459static inline int is_vmalloc_or_module_addr(const void *x)
460{
461 return 0;
462}
463#endif
464
465extern void kvfree(const void *addr);
466
467static inline atomic_t *compound_mapcount_ptr(struct page *page)
468{
469 return &page[1].compound_mapcount;
470}
471
472static inline int compound_mapcount(struct page *page)
473{
474 if (!PageCompound(page))
475 return 0;
476 page = compound_head(page);
477 return atomic_read(compound_mapcount_ptr(page)) + 1;
478}
479
480/*
481 * The atomic page->_mapcount, starts from -1: so that transitions
482 * both from it and to it can be tracked, using atomic_inc_and_test
483 * and atomic_add_negative(-1).
484 */
485static inline void page_mapcount_reset(struct page *page)
486{
487 atomic_set(&(page)->_mapcount, -1);
488}
489
490int __page_mapcount(struct page *page);
491
492static inline int page_mapcount(struct page *page)
493{
494 VM_BUG_ON_PAGE(PageSlab(page), page);
495
496 if (unlikely(PageCompound(page)))
497 return __page_mapcount(page);
498 return atomic_read(&page->_mapcount) + 1;
499}
500
501#ifdef CONFIG_TRANSPARENT_HUGEPAGE
502int total_mapcount(struct page *page);
503int page_trans_huge_mapcount(struct page *page, int *total_mapcount);
504#else
505static inline int total_mapcount(struct page *page)
506{
507 return page_mapcount(page);
508}
509static inline int page_trans_huge_mapcount(struct page *page,
510 int *total_mapcount)
511{
512 int mapcount = page_mapcount(page);
513 if (total_mapcount)
514 *total_mapcount = mapcount;
515 return mapcount;
516}
517#endif
518
519static inline struct page *virt_to_head_page(const void *x)
520{
521 struct page *page = virt_to_page(x);
522
523 return compound_head(page);
524}
525
526void __put_page(struct page *page);
527
528void put_pages_list(struct list_head *pages);
529
530void split_page(struct page *page, unsigned int order);
531int split_free_page(struct page *page);
532
533/*
534 * Compound pages have a destructor function. Provide a
535 * prototype for that function and accessor functions.
536 * These are _only_ valid on the head of a compound page.
537 */
538typedef void compound_page_dtor(struct page *);
539
540/* Keep the enum in sync with compound_page_dtors array in mm/page_alloc.c */
541enum compound_dtor_id {
542 NULL_COMPOUND_DTOR,
543 COMPOUND_PAGE_DTOR,
544#ifdef CONFIG_HUGETLB_PAGE
545 HUGETLB_PAGE_DTOR,
546#endif
547#ifdef CONFIG_TRANSPARENT_HUGEPAGE
548 TRANSHUGE_PAGE_DTOR,
549#endif
550 NR_COMPOUND_DTORS,
551};
552extern compound_page_dtor * const compound_page_dtors[];
553
554static inline void set_compound_page_dtor(struct page *page,
555 enum compound_dtor_id compound_dtor)
556{
557 VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page);
558 page[1].compound_dtor = compound_dtor;
559}
560
561static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
562{
563 VM_BUG_ON_PAGE(page[1].compound_dtor >= NR_COMPOUND_DTORS, page);
564 return compound_page_dtors[page[1].compound_dtor];
565}
566
567static inline unsigned int compound_order(struct page *page)
568{
569 if (!PageHead(page))
570 return 0;
571 return page[1].compound_order;
572}
573
574static inline void set_compound_order(struct page *page, unsigned int order)
575{
576 page[1].compound_order = order;
577}
578
579void free_compound_page(struct page *page);
580
581#ifdef CONFIG_MMU
582/*
583 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
584 * servicing faults for write access. In the normal case, do always want
585 * pte_mkwrite. But get_user_pages can cause write faults for mappings
586 * that do not have writing enabled, when used by access_process_vm.
587 */
588static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
589{
590 if (likely(vma->vm_flags & VM_WRITE))
591 pte = pte_mkwrite(pte);
592 return pte;
593}
594
595void do_set_pte(struct vm_area_struct *vma, unsigned long address,
596 struct page *page, pte_t *pte, bool write, bool anon);
597#endif
598
599/*
600 * Multiple processes may "see" the same page. E.g. for untouched
601 * mappings of /dev/null, all processes see the same page full of
602 * zeroes, and text pages of executables and shared libraries have
603 * only one copy in memory, at most, normally.
604 *
605 * For the non-reserved pages, page_count(page) denotes a reference count.
606 * page_count() == 0 means the page is free. page->lru is then used for
607 * freelist management in the buddy allocator.
608 * page_count() > 0 means the page has been allocated.
609 *
610 * Pages are allocated by the slab allocator in order to provide memory
611 * to kmalloc and kmem_cache_alloc. In this case, the management of the
612 * page, and the fields in 'struct page' are the responsibility of mm/slab.c
613 * unless a particular usage is carefully commented. (the responsibility of
614 * freeing the kmalloc memory is the caller's, of course).
615 *
616 * A page may be used by anyone else who does a __get_free_page().
617 * In this case, page_count still tracks the references, and should only
618 * be used through the normal accessor functions. The top bits of page->flags
619 * and page->virtual store page management information, but all other fields
620 * are unused and could be used privately, carefully. The management of this
621 * page is the responsibility of the one who allocated it, and those who have
622 * subsequently been given references to it.
623 *
624 * The other pages (we may call them "pagecache pages") are completely
625 * managed by the Linux memory manager: I/O, buffers, swapping etc.
626 * The following discussion applies only to them.
627 *
628 * A pagecache page contains an opaque `private' member, which belongs to the
629 * page's address_space. Usually, this is the address of a circular list of
630 * the page's disk buffers. PG_private must be set to tell the VM to call
631 * into the filesystem to release these pages.
632 *
633 * A page may belong to an inode's memory mapping. In this case, page->mapping
634 * is the pointer to the inode, and page->index is the file offset of the page,
635 * in units of PAGE_SIZE.
636 *
637 * If pagecache pages are not associated with an inode, they are said to be
638 * anonymous pages. These may become associated with the swapcache, and in that
639 * case PG_swapcache is set, and page->private is an offset into the swapcache.
640 *
641 * In either case (swapcache or inode backed), the pagecache itself holds one
642 * reference to the page. Setting PG_private should also increment the
643 * refcount. The each user mapping also has a reference to the page.
644 *
645 * The pagecache pages are stored in a per-mapping radix tree, which is
646 * rooted at mapping->page_tree, and indexed by offset.
647 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space
648 * lists, we instead now tag pages as dirty/writeback in the radix tree.
649 *
650 * All pagecache pages may be subject to I/O:
651 * - inode pages may need to be read from disk,
652 * - inode pages which have been modified and are MAP_SHARED may need
653 * to be written back to the inode on disk,
654 * - anonymous pages (including MAP_PRIVATE file mappings) which have been
655 * modified may need to be swapped out to swap space and (later) to be read
656 * back into memory.
657 */
658
659/*
660 * The zone field is never updated after free_area_init_core()
661 * sets it, so none of the operations on it need to be atomic.
662 */
663
664/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */
665#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
666#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
667#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
668#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)
669
670/*
671 * Define the bit shifts to access each section. For non-existent
672 * sections we define the shift as 0; that plus a 0 mask ensures
673 * the compiler will optimise away reference to them.
674 */
675#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
676#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
677#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
678#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))
679
680/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
681#ifdef NODE_NOT_IN_PAGE_FLAGS
682#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
683#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
684 SECTIONS_PGOFF : ZONES_PGOFF)
685#else
686#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
687#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
688 NODES_PGOFF : ZONES_PGOFF)
689#endif
690
691#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
692
693#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
694#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
695#endif
696
697#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
698#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
699#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
700#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_SHIFT) - 1)
701#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
702
703static inline enum zone_type page_zonenum(const struct page *page)
704{
705 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
706}
707
708#ifdef CONFIG_ZONE_DEVICE
709void get_zone_device_page(struct page *page);
710void put_zone_device_page(struct page *page);
711static inline bool is_zone_device_page(const struct page *page)
712{
713 return page_zonenum(page) == ZONE_DEVICE;
714}
715#else
716static inline void get_zone_device_page(struct page *page)
717{
718}
719static inline void put_zone_device_page(struct page *page)
720{
721}
722static inline bool is_zone_device_page(const struct page *page)
723{
724 return false;
725}
726#endif
727
728static inline void get_page(struct page *page)
729{
730 page = compound_head(page);
731 /*
732 * Getting a normal page or the head of a compound page
733 * requires to already have an elevated page->_count.
734 */
735 VM_BUG_ON_PAGE(page_ref_count(page) <= 0, page);
736 page_ref_inc(page);
737
738 if (unlikely(is_zone_device_page(page)))
739 get_zone_device_page(page);
740}
741
742static inline void put_page(struct page *page)
743{
744 page = compound_head(page);
745
746 if (put_page_testzero(page))
747 __put_page(page);
748
749 if (unlikely(is_zone_device_page(page)))
750 put_zone_device_page(page);
751}
752
753#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
754#define SECTION_IN_PAGE_FLAGS
755#endif
756
757/*
758 * The identification function is mainly used by the buddy allocator for
759 * determining if two pages could be buddies. We are not really identifying
760 * the zone since we could be using the section number id if we do not have
761 * node id available in page flags.
762 * We only guarantee that it will return the same value for two combinable
763 * pages in a zone.
764 */
765static inline int page_zone_id(struct page *page)
766{
767 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
768}
769
770static inline int zone_to_nid(struct zone *zone)
771{
772#ifdef CONFIG_NUMA
773 return zone->node;
774#else
775 return 0;
776#endif
777}
778
779#ifdef NODE_NOT_IN_PAGE_FLAGS
780extern int page_to_nid(const struct page *page);
781#else
782static inline int page_to_nid(const struct page *page)
783{
784 return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
785}
786#endif
787
788#ifdef CONFIG_NUMA_BALANCING
789static inline int cpu_pid_to_cpupid(int cpu, int pid)
790{
791 return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
792}
793
794static inline int cpupid_to_pid(int cpupid)
795{
796 return cpupid & LAST__PID_MASK;
797}
798
799static inline int cpupid_to_cpu(int cpupid)
800{
801 return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
802}
803
804static inline int cpupid_to_nid(int cpupid)
805{
806 return cpu_to_node(cpupid_to_cpu(cpupid));
807}
808
809static inline bool cpupid_pid_unset(int cpupid)
810{
811 return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
812}
813
814static inline bool cpupid_cpu_unset(int cpupid)
815{
816 return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
817}
818
819static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
820{
821 return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);
822}
823
824#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)
825#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
826static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
827{
828 return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);
829}
830
831static inline int page_cpupid_last(struct page *page)
832{
833 return page->_last_cpupid;
834}
835static inline void page_cpupid_reset_last(struct page *page)
836{
837 page->_last_cpupid = -1 & LAST_CPUPID_MASK;
838}
839#else
840static inline int page_cpupid_last(struct page *page)
841{
842 return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
843}
844
845extern int page_cpupid_xchg_last(struct page *page, int cpupid);
846
847static inline void page_cpupid_reset_last(struct page *page)
848{
849 int cpupid = (1 << LAST_CPUPID_SHIFT) - 1;
850
851 page->flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);
852 page->flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;
853}
854#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
855#else /* !CONFIG_NUMA_BALANCING */
856static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
857{
858 return page_to_nid(page); /* XXX */
859}
860
861static inline int page_cpupid_last(struct page *page)
862{
863 return page_to_nid(page); /* XXX */
864}
865
866static inline int cpupid_to_nid(int cpupid)
867{
868 return -1;
869}
870
871static inline int cpupid_to_pid(int cpupid)
872{
873 return -1;
874}
875
876static inline int cpupid_to_cpu(int cpupid)
877{
878 return -1;
879}
880
881static inline int cpu_pid_to_cpupid(int nid, int pid)
882{
883 return -1;
884}
885
886static inline bool cpupid_pid_unset(int cpupid)
887{
888 return 1;
889}
890
891static inline void page_cpupid_reset_last(struct page *page)
892{
893}
894
895static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
896{
897 return false;
898}
899#endif /* CONFIG_NUMA_BALANCING */
900
901static inline struct zone *page_zone(const struct page *page)
902{
903 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
904}
905
906#ifdef SECTION_IN_PAGE_FLAGS
907static inline void set_page_section(struct page *page, unsigned long section)
908{
909 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
910 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
911}
912
913static inline unsigned long page_to_section(const struct page *page)
914{
915 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
916}
917#endif
918
919static inline void set_page_zone(struct page *page, enum zone_type zone)
920{
921 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
922 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
923}
924
925static inline void set_page_node(struct page *page, unsigned long node)
926{
927 page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
928 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
929}
930
931static inline void set_page_links(struct page *page, enum zone_type zone,
932 unsigned long node, unsigned long pfn)
933{
934 set_page_zone(page, zone);
935 set_page_node(page, node);
936#ifdef SECTION_IN_PAGE_FLAGS
937 set_page_section(page, pfn_to_section_nr(pfn));
938#endif
939}
940
941#ifdef CONFIG_MEMCG
942static inline struct mem_cgroup *page_memcg(struct page *page)
943{
944 return page->mem_cgroup;
945}
946#else
947static inline struct mem_cgroup *page_memcg(struct page *page)
948{
949 return NULL;
950}
951#endif
952
953/*
954 * Some inline functions in vmstat.h depend on page_zone()
955 */
956#include <linux/vmstat.h>
957
958static __always_inline void *lowmem_page_address(const struct page *page)
959{
960 return __va(PFN_PHYS(page_to_pfn(page)));
961}
962
963#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
964#define HASHED_PAGE_VIRTUAL
965#endif
966
967#if defined(WANT_PAGE_VIRTUAL)
968static inline void *page_address(const struct page *page)
969{
970 return page->virtual;
971}
972static inline void set_page_address(struct page *page, void *address)
973{
974 page->virtual = address;
975}
976#define page_address_init() do { } while(0)
977#endif
978
979#if defined(HASHED_PAGE_VIRTUAL)
980void *page_address(const struct page *page);
981void set_page_address(struct page *page, void *virtual);
982void page_address_init(void);
983#endif
984
985#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
986#define page_address(page) lowmem_page_address(page)
987#define set_page_address(page, address) do { } while(0)
988#define page_address_init() do { } while(0)
989#endif
990
991extern void *page_rmapping(struct page *page);
992extern struct anon_vma *page_anon_vma(struct page *page);
993extern struct address_space *page_mapping(struct page *page);
994
995extern struct address_space *__page_file_mapping(struct page *);
996
997static inline
998struct address_space *page_file_mapping(struct page *page)
999{
1000 if (unlikely(PageSwapCache(page)))
1001 return __page_file_mapping(page);
1002
1003 return page->mapping;
1004}
1005
1006/*
1007 * Return the pagecache index of the passed page. Regular pagecache pages
1008 * use ->index whereas swapcache pages use ->private
1009 */
1010static inline pgoff_t page_index(struct page *page)
1011{
1012 if (unlikely(PageSwapCache(page)))
1013 return page_private(page);
1014 return page->index;
1015}
1016
1017extern pgoff_t __page_file_index(struct page *page);
1018
1019/*
1020 * Return the file index of the page. Regular pagecache pages use ->index
1021 * whereas swapcache pages use swp_offset(->private)
1022 */
1023static inline pgoff_t page_file_index(struct page *page)
1024{
1025 if (unlikely(PageSwapCache(page)))
1026 return __page_file_index(page);
1027
1028 return page->index;
1029}
1030
1031/*
1032 * Return true if this page is mapped into pagetables.
1033 * For compound page it returns true if any subpage of compound page is mapped.
1034 */
1035static inline bool page_mapped(struct page *page)
1036{
1037 int i;
1038 if (likely(!PageCompound(page)))
1039 return atomic_read(&page->_mapcount) >= 0;
1040 page = compound_head(page);
1041 if (atomic_read(compound_mapcount_ptr(page)) >= 0)
1042 return true;
1043 if (PageHuge(page))
1044 return false;
1045 for (i = 0; i < hpage_nr_pages(page); i++) {
1046 if (atomic_read(&page[i]._mapcount) >= 0)
1047 return true;
1048 }
1049 return false;
1050}
1051
1052/*
1053 * Return true only if the page has been allocated with
1054 * ALLOC_NO_WATERMARKS and the low watermark was not
1055 * met implying that the system is under some pressure.
1056 */
1057static inline bool page_is_pfmemalloc(struct page *page)
1058{
1059 /*
1060 * Page index cannot be this large so this must be
1061 * a pfmemalloc page.
1062 */
1063 return page->index == -1UL;
1064}
1065
1066/*
1067 * Only to be called by the page allocator on a freshly allocated
1068 * page.
1069 */
1070static inline void set_page_pfmemalloc(struct page *page)
1071{
1072 page->index = -1UL;
1073}
1074
1075static inline void clear_page_pfmemalloc(struct page *page)
1076{
1077 page->index = 0;
1078}
1079
1080/*
1081 * Different kinds of faults, as returned by handle_mm_fault().
1082 * Used to decide whether a process gets delivered SIGBUS or
1083 * just gets major/minor fault counters bumped up.
1084 */
1085
1086#define VM_FAULT_OOM 0x0001
1087#define VM_FAULT_SIGBUS 0x0002
1088#define VM_FAULT_MAJOR 0x0004
1089#define VM_FAULT_WRITE 0x0008 /* Special case for get_user_pages */
1090#define VM_FAULT_HWPOISON 0x0010 /* Hit poisoned small page */
1091#define VM_FAULT_HWPOISON_LARGE 0x0020 /* Hit poisoned large page. Index encoded in upper bits */
1092#define VM_FAULT_SIGSEGV 0x0040
1093
1094#define VM_FAULT_NOPAGE 0x0100 /* ->fault installed the pte, not return page */
1095#define VM_FAULT_LOCKED 0x0200 /* ->fault locked the returned page */
1096#define VM_FAULT_RETRY 0x0400 /* ->fault blocked, must retry */
1097#define VM_FAULT_FALLBACK 0x0800 /* huge page fault failed, fall back to small */
1098
1099#define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */
1100
1101#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | \
1102 VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE | \
1103 VM_FAULT_FALLBACK)
1104
1105/* Encode hstate index for a hwpoisoned large page */
1106#define VM_FAULT_SET_HINDEX(x) ((x) << 12)
1107#define VM_FAULT_GET_HINDEX(x) (((x) >> 12) & 0xf)
1108
1109/*
1110 * Can be called by the pagefault handler when it gets a VM_FAULT_OOM.
1111 */
1112extern void pagefault_out_of_memory(void);
1113
1114#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
1115
1116/*
1117 * Flags passed to show_mem() and show_free_areas() to suppress output in
1118 * various contexts.
1119 */
1120#define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */
1121
1122extern void show_free_areas(unsigned int flags);
1123extern bool skip_free_areas_node(unsigned int flags, int nid);
1124
1125int shmem_zero_setup(struct vm_area_struct *);
1126#ifdef CONFIG_SHMEM
1127bool shmem_mapping(struct address_space *mapping);
1128#else
1129static inline bool shmem_mapping(struct address_space *mapping)
1130{
1131 return false;
1132}
1133#endif
1134
1135extern bool can_do_mlock(void);
1136extern int user_shm_lock(size_t, struct user_struct *);
1137extern void user_shm_unlock(size_t, struct user_struct *);
1138
1139/*
1140 * Parameter block passed down to zap_pte_range in exceptional cases.
1141 */
1142struct zap_details {
1143 struct address_space *check_mapping; /* Check page->mapping if set */
1144 pgoff_t first_index; /* Lowest page->index to unmap */
1145 pgoff_t last_index; /* Highest page->index to unmap */
1146 bool ignore_dirty; /* Ignore dirty pages */
1147 bool check_swap_entries; /* Check also swap entries */
1148};
1149
1150struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1151 pte_t pte);
1152struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
1153 pmd_t pmd);
1154
1155int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
1156 unsigned long size);
1157void zap_page_range(struct vm_area_struct *vma, unsigned long address,
1158 unsigned long size, struct zap_details *);
1159void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
1160 unsigned long start, unsigned long end);
1161
1162/**
1163 * mm_walk - callbacks for walk_page_range
1164 * @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry
1165 * this handler is required to be able to handle
1166 * pmd_trans_huge() pmds. They may simply choose to
1167 * split_huge_page() instead of handling it explicitly.
1168 * @pte_entry: if set, called for each non-empty PTE (4th-level) entry
1169 * @pte_hole: if set, called for each hole at all levels
1170 * @hugetlb_entry: if set, called for each hugetlb entry
1171 * @test_walk: caller specific callback function to determine whether
1172 * we walk over the current vma or not. A positive returned
1173 * value means "do page table walk over the current vma,"
1174 * and a negative one means "abort current page table walk
1175 * right now." 0 means "skip the current vma."
1176 * @mm: mm_struct representing the target process of page table walk
1177 * @vma: vma currently walked (NULL if walking outside vmas)
1178 * @private: private data for callbacks' usage
1179 *
1180 * (see the comment on walk_page_range() for more details)
1181 */
1182struct mm_walk {
1183 int (*pmd_entry)(pmd_t *pmd, unsigned long addr,
1184 unsigned long next, struct mm_walk *walk);
1185 int (*pte_entry)(pte_t *pte, unsigned long addr,
1186 unsigned long next, struct mm_walk *walk);
1187 int (*pte_hole)(unsigned long addr, unsigned long next,
1188 struct mm_walk *walk);
1189 int (*hugetlb_entry)(pte_t *pte, unsigned long hmask,
1190 unsigned long addr, unsigned long next,
1191 struct mm_walk *walk);
1192 int (*test_walk)(unsigned long addr, unsigned long next,
1193 struct mm_walk *walk);
1194 struct mm_struct *mm;
1195 struct vm_area_struct *vma;
1196 void *private;
1197};
1198
1199int walk_page_range(unsigned long addr, unsigned long end,
1200 struct mm_walk *walk);
1201int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk);
1202void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
1203 unsigned long end, unsigned long floor, unsigned long ceiling);
1204int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
1205 struct vm_area_struct *vma);
1206void unmap_mapping_range(struct address_space *mapping,
1207 loff_t const holebegin, loff_t const holelen, int even_cows);
1208int follow_pfn(struct vm_area_struct *vma, unsigned long address,
1209 unsigned long *pfn);
1210int follow_phys(struct vm_area_struct *vma, unsigned long address,
1211 unsigned int flags, unsigned long *prot, resource_size_t *phys);
1212int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
1213 void *buf, int len, int write);
1214
1215static inline void unmap_shared_mapping_range(struct address_space *mapping,
1216 loff_t const holebegin, loff_t const holelen)
1217{
1218 unmap_mapping_range(mapping, holebegin, holelen, 0);
1219}
1220
1221extern void truncate_pagecache(struct inode *inode, loff_t new);
1222extern void truncate_setsize(struct inode *inode, loff_t newsize);
1223void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
1224void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
1225int truncate_inode_page(struct address_space *mapping, struct page *page);
1226int generic_error_remove_page(struct address_space *mapping, struct page *page);
1227int invalidate_inode_page(struct page *page);
1228
1229#ifdef CONFIG_MMU
1230extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
1231 unsigned long address, unsigned int flags);
1232extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
1233 unsigned long address, unsigned int fault_flags,
1234 bool *unlocked);
1235#else
1236static inline int handle_mm_fault(struct mm_struct *mm,
1237 struct vm_area_struct *vma, unsigned long address,
1238 unsigned int flags)
1239{
1240 /* should never happen if there's no MMU */
1241 BUG();
1242 return VM_FAULT_SIGBUS;
1243}
1244static inline int fixup_user_fault(struct task_struct *tsk,
1245 struct mm_struct *mm, unsigned long address,
1246 unsigned int fault_flags, bool *unlocked)
1247{
1248 /* should never happen if there's no MMU */
1249 BUG();
1250 return -EFAULT;
1251}
1252#endif
1253
1254extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
1255extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
1256 void *buf, int len, int write);
1257
1258long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
1259 unsigned long start, unsigned long nr_pages,
1260 unsigned int foll_flags, struct page **pages,
1261 struct vm_area_struct **vmas, int *nonblocking);
1262long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
1263 unsigned long start, unsigned long nr_pages,
1264 int write, int force, struct page **pages,
1265 struct vm_area_struct **vmas);
1266long get_user_pages(unsigned long start, unsigned long nr_pages,
1267 int write, int force, struct page **pages,
1268 struct vm_area_struct **vmas);
1269long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
1270 int write, int force, struct page **pages, int *locked);
1271long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
1272 unsigned long start, unsigned long nr_pages,
1273 int write, int force, struct page **pages,
1274 unsigned int gup_flags);
1275long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
1276 int write, int force, struct page **pages);
1277int get_user_pages_fast(unsigned long start, int nr_pages, int write,
1278 struct page **pages);
1279
1280/* Container for pinned pfns / pages */
1281struct frame_vector {
1282 unsigned int nr_allocated; /* Number of frames we have space for */
1283 unsigned int nr_frames; /* Number of frames stored in ptrs array */
1284 bool got_ref; /* Did we pin pages by getting page ref? */
1285 bool is_pfns; /* Does array contain pages or pfns? */
1286 void *ptrs[0]; /* Array of pinned pfns / pages. Use
1287 * pfns_vector_pages() or pfns_vector_pfns()
1288 * for access */
1289};
1290
1291struct frame_vector *frame_vector_create(unsigned int nr_frames);
1292void frame_vector_destroy(struct frame_vector *vec);
1293int get_vaddr_frames(unsigned long start, unsigned int nr_pfns,
1294 bool write, bool force, struct frame_vector *vec);
1295void put_vaddr_frames(struct frame_vector *vec);
1296int frame_vector_to_pages(struct frame_vector *vec);
1297void frame_vector_to_pfns(struct frame_vector *vec);
1298
1299static inline unsigned int frame_vector_count(struct frame_vector *vec)
1300{
1301 return vec->nr_frames;
1302}
1303
1304static inline struct page **frame_vector_pages(struct frame_vector *vec)
1305{
1306 if (vec->is_pfns) {
1307 int err = frame_vector_to_pages(vec);
1308
1309 if (err)
1310 return ERR_PTR(err);
1311 }
1312 return (struct page **)(vec->ptrs);
1313}
1314
1315static inline unsigned long *frame_vector_pfns(struct frame_vector *vec)
1316{
1317 if (!vec->is_pfns)
1318 frame_vector_to_pfns(vec);
1319 return (unsigned long *)(vec->ptrs);
1320}
1321
1322struct kvec;
1323int get_kernel_pages(const struct kvec *iov, int nr_pages, int write,
1324 struct page **pages);
1325int get_kernel_page(unsigned long start, int write, struct page **pages);
1326struct page *get_dump_page(unsigned long addr);
1327
1328extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
1329extern void do_invalidatepage(struct page *page, unsigned int offset,
1330 unsigned int length);
1331
1332int __set_page_dirty_nobuffers(struct page *page);
1333int __set_page_dirty_no_writeback(struct page *page);
1334int redirty_page_for_writepage(struct writeback_control *wbc,
1335 struct page *page);
1336void account_page_dirtied(struct page *page, struct address_space *mapping);
1337void account_page_cleaned(struct page *page, struct address_space *mapping,
1338 struct bdi_writeback *wb);
1339int set_page_dirty(struct page *page);
1340int set_page_dirty_lock(struct page *page);
1341void cancel_dirty_page(struct page *page);
1342int clear_page_dirty_for_io(struct page *page);
1343
1344int get_cmdline(struct task_struct *task, char *buffer, int buflen);
1345
1346/* Is the vma a continuation of the stack vma above it? */
1347static inline int vma_growsdown(struct vm_area_struct *vma, unsigned long addr)
1348{
1349 return vma && (vma->vm_end == addr) && (vma->vm_flags & VM_GROWSDOWN);
1350}
1351
1352static inline bool vma_is_anonymous(struct vm_area_struct *vma)
1353{
1354 return !vma->vm_ops;
1355}
1356
1357static inline int stack_guard_page_start(struct vm_area_struct *vma,
1358 unsigned long addr)
1359{
1360 return (vma->vm_flags & VM_GROWSDOWN) &&
1361 (vma->vm_start == addr) &&
1362 !vma_growsdown(vma->vm_prev, addr);
1363}
1364
1365/* Is the vma a continuation of the stack vma below it? */
1366static inline int vma_growsup(struct vm_area_struct *vma, unsigned long addr)
1367{
1368 return vma && (vma->vm_start == addr) && (vma->vm_flags & VM_GROWSUP);
1369}
1370
1371static inline int stack_guard_page_end(struct vm_area_struct *vma,
1372 unsigned long addr)
1373{
1374 return (vma->vm_flags & VM_GROWSUP) &&
1375 (vma->vm_end == addr) &&
1376 !vma_growsup(vma->vm_next, addr);
1377}
1378
1379int vma_is_stack_for_task(struct vm_area_struct *vma, struct task_struct *t);
1380
1381extern unsigned long move_page_tables(struct vm_area_struct *vma,
1382 unsigned long old_addr, struct vm_area_struct *new_vma,
1383 unsigned long new_addr, unsigned long len,
1384 bool need_rmap_locks);
1385extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
1386 unsigned long end, pgprot_t newprot,
1387 int dirty_accountable, int prot_numa);
1388extern int mprotect_fixup(struct vm_area_struct *vma,
1389 struct vm_area_struct **pprev, unsigned long start,
1390 unsigned long end, unsigned long newflags);
1391
1392/*
1393 * doesn't attempt to fault and will return short.
1394 */
1395int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
1396 struct page **pages);
1397/*
1398 * per-process(per-mm_struct) statistics.
1399 */
1400static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
1401{
1402 long val = atomic_long_read(&mm->rss_stat.count[member]);
1403
1404#ifdef SPLIT_RSS_COUNTING
1405 /*
1406 * counter is updated in asynchronous manner and may go to minus.
1407 * But it's never be expected number for users.
1408 */
1409 if (val < 0)
1410 val = 0;
1411#endif
1412 return (unsigned long)val;
1413}
1414
1415static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
1416{
1417 atomic_long_add(value, &mm->rss_stat.count[member]);
1418}
1419
1420static inline void inc_mm_counter(struct mm_struct *mm, int member)
1421{
1422 atomic_long_inc(&mm->rss_stat.count[member]);
1423}
1424
1425static inline void dec_mm_counter(struct mm_struct *mm, int member)
1426{
1427 atomic_long_dec(&mm->rss_stat.count[member]);
1428}
1429
1430/* Optimized variant when page is already known not to be PageAnon */
1431static inline int mm_counter_file(struct page *page)
1432{
1433 if (PageSwapBacked(page))
1434 return MM_SHMEMPAGES;
1435 return MM_FILEPAGES;
1436}
1437
1438static inline int mm_counter(struct page *page)
1439{
1440 if (PageAnon(page))
1441 return MM_ANONPAGES;
1442 return mm_counter_file(page);
1443}
1444
1445static inline unsigned long get_mm_rss(struct mm_struct *mm)
1446{
1447 return get_mm_counter(mm, MM_FILEPAGES) +
1448 get_mm_counter(mm, MM_ANONPAGES) +
1449 get_mm_counter(mm, MM_SHMEMPAGES);
1450}
1451
1452static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
1453{
1454 return max(mm->hiwater_rss, get_mm_rss(mm));
1455}
1456
1457static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
1458{
1459 return max(mm->hiwater_vm, mm->total_vm);
1460}
1461
1462static inline void update_hiwater_rss(struct mm_struct *mm)
1463{
1464 unsigned long _rss = get_mm_rss(mm);
1465
1466 if ((mm)->hiwater_rss < _rss)
1467 (mm)->hiwater_rss = _rss;
1468}
1469
1470static inline void update_hiwater_vm(struct mm_struct *mm)
1471{
1472 if (mm->hiwater_vm < mm->total_vm)
1473 mm->hiwater_vm = mm->total_vm;
1474}
1475
1476static inline void reset_mm_hiwater_rss(struct mm_struct *mm)
1477{
1478 mm->hiwater_rss = get_mm_rss(mm);
1479}
1480
1481static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
1482 struct mm_struct *mm)
1483{
1484 unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
1485
1486 if (*maxrss < hiwater_rss)
1487 *maxrss = hiwater_rss;
1488}
1489
1490#if defined(SPLIT_RSS_COUNTING)
1491void sync_mm_rss(struct mm_struct *mm);
1492#else
1493static inline void sync_mm_rss(struct mm_struct *mm)
1494{
1495}
1496#endif
1497
1498#ifndef __HAVE_ARCH_PTE_DEVMAP
1499static inline int pte_devmap(pte_t pte)
1500{
1501 return 0;
1502}
1503#endif
1504
1505int vma_wants_writenotify(struct vm_area_struct *vma);
1506
1507extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
1508 spinlock_t **ptl);
1509static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
1510 spinlock_t **ptl)
1511{
1512 pte_t *ptep;
1513 __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
1514 return ptep;
1515}
1516
1517#ifdef __PAGETABLE_PUD_FOLDED
1518static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
1519 unsigned long address)
1520{
1521 return 0;
1522}
1523#else
1524int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
1525#endif
1526
1527#if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)
1528static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
1529 unsigned long address)
1530{
1531 return 0;
1532}
1533
1534static inline void mm_nr_pmds_init(struct mm_struct *mm) {}
1535
1536static inline unsigned long mm_nr_pmds(struct mm_struct *mm)
1537{
1538 return 0;
1539}
1540
1541static inline void mm_inc_nr_pmds(struct mm_struct *mm) {}
1542static inline void mm_dec_nr_pmds(struct mm_struct *mm) {}
1543
1544#else
1545int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
1546
1547static inline void mm_nr_pmds_init(struct mm_struct *mm)
1548{
1549 atomic_long_set(&mm->nr_pmds, 0);
1550}
1551
1552static inline unsigned long mm_nr_pmds(struct mm_struct *mm)
1553{
1554 return atomic_long_read(&mm->nr_pmds);
1555}
1556
1557static inline void mm_inc_nr_pmds(struct mm_struct *mm)
1558{
1559 atomic_long_inc(&mm->nr_pmds);
1560}
1561
1562static inline void mm_dec_nr_pmds(struct mm_struct *mm)
1563{
1564 atomic_long_dec(&mm->nr_pmds);
1565}
1566#endif
1567
1568int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
1569int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
1570
1571/*
1572 * The following ifdef needed to get the 4level-fixup.h header to work.
1573 * Remove it when 4level-fixup.h has been removed.
1574 */
1575#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
1576static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
1577{
1578 return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
1579 NULL: pud_offset(pgd, address);
1580}
1581
1582static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
1583{
1584 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
1585 NULL: pmd_offset(pud, address);
1586}
1587#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
1588
1589#if USE_SPLIT_PTE_PTLOCKS
1590#if ALLOC_SPLIT_PTLOCKS
1591void __init ptlock_cache_init(void);
1592extern bool ptlock_alloc(struct page *page);
1593extern void ptlock_free(struct page *page);
1594
1595static inline spinlock_t *ptlock_ptr(struct page *page)
1596{
1597 return page->ptl;
1598}
1599#else /* ALLOC_SPLIT_PTLOCKS */
1600static inline void ptlock_cache_init(void)
1601{
1602}
1603
1604static inline bool ptlock_alloc(struct page *page)
1605{
1606 return true;
1607}
1608
1609static inline void ptlock_free(struct page *page)
1610{
1611}
1612
1613static inline spinlock_t *ptlock_ptr(struct page *page)
1614{
1615 return &page->ptl;
1616}
1617#endif /* ALLOC_SPLIT_PTLOCKS */
1618
1619static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1620{
1621 return ptlock_ptr(pmd_page(*pmd));
1622}
1623
1624static inline bool ptlock_init(struct page *page)
1625{
1626 /*
1627 * prep_new_page() initialize page->private (and therefore page->ptl)
1628 * with 0. Make sure nobody took it in use in between.
1629 *
1630 * It can happen if arch try to use slab for page table allocation:
1631 * slab code uses page->slab_cache, which share storage with page->ptl.
1632 */
1633 VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page);
1634 if (!ptlock_alloc(page))
1635 return false;
1636 spin_lock_init(ptlock_ptr(page));
1637 return true;
1638}
1639
1640/* Reset page->mapping so free_pages_check won't complain. */
1641static inline void pte_lock_deinit(struct page *page)
1642{
1643 page->mapping = NULL;
1644 ptlock_free(page);
1645}
1646
1647#else /* !USE_SPLIT_PTE_PTLOCKS */
1648/*
1649 * We use mm->page_table_lock to guard all pagetable pages of the mm.
1650 */
1651static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1652{
1653 return &mm->page_table_lock;
1654}
1655static inline void ptlock_cache_init(void) {}
1656static inline bool ptlock_init(struct page *page) { return true; }
1657static inline void pte_lock_deinit(struct page *page) {}
1658#endif /* USE_SPLIT_PTE_PTLOCKS */
1659
1660static inline void pgtable_init(void)
1661{
1662 ptlock_cache_init();
1663 pgtable_cache_init();
1664}
1665
1666static inline bool pgtable_page_ctor(struct page *page)
1667{
1668 if (!ptlock_init(page))
1669 return false;
1670 inc_zone_page_state(page, NR_PAGETABLE);
1671 return true;
1672}
1673
1674static inline void pgtable_page_dtor(struct page *page)
1675{
1676 pte_lock_deinit(page);
1677 dec_zone_page_state(page, NR_PAGETABLE);
1678}
1679
1680#define pte_offset_map_lock(mm, pmd, address, ptlp) \
1681({ \
1682 spinlock_t *__ptl = pte_lockptr(mm, pmd); \
1683 pte_t *__pte = pte_offset_map(pmd, address); \
1684 *(ptlp) = __ptl; \
1685 spin_lock(__ptl); \
1686 __pte; \
1687})
1688
1689#define pte_unmap_unlock(pte, ptl) do { \
1690 spin_unlock(ptl); \
1691 pte_unmap(pte); \
1692} while (0)
1693
1694#define pte_alloc(mm, pmd, address) \
1695 (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd, address))
1696
1697#define pte_alloc_map(mm, pmd, address) \
1698 (pte_alloc(mm, pmd, address) ? NULL : pte_offset_map(pmd, address))
1699
1700#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
1701 (pte_alloc(mm, pmd, address) ? \
1702 NULL : pte_offset_map_lock(mm, pmd, address, ptlp))
1703
1704#define pte_alloc_kernel(pmd, address) \
1705 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
1706 NULL: pte_offset_kernel(pmd, address))
1707
1708#if USE_SPLIT_PMD_PTLOCKS
1709
1710static struct page *pmd_to_page(pmd_t *pmd)
1711{
1712 unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
1713 return virt_to_page((void *)((unsigned long) pmd & mask));
1714}
1715
1716static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1717{
1718 return ptlock_ptr(pmd_to_page(pmd));
1719}
1720
1721static inline bool pgtable_pmd_page_ctor(struct page *page)
1722{
1723#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1724 page->pmd_huge_pte = NULL;
1725#endif
1726 return ptlock_init(page);
1727}
1728
1729static inline void pgtable_pmd_page_dtor(struct page *page)
1730{
1731#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1732 VM_BUG_ON_PAGE(page->pmd_huge_pte, page);
1733#endif
1734 ptlock_free(page);
1735}
1736
1737#define pmd_huge_pte(mm, pmd) (pmd_to_page(pmd)->pmd_huge_pte)
1738
1739#else
1740
1741static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1742{
1743 return &mm->page_table_lock;
1744}
1745
1746static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }
1747static inline void pgtable_pmd_page_dtor(struct page *page) {}
1748
1749#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
1750
1751#endif
1752
1753static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
1754{
1755 spinlock_t *ptl = pmd_lockptr(mm, pmd);
1756 spin_lock(ptl);
1757 return ptl;
1758}
1759
1760extern void free_area_init(unsigned long * zones_size);
1761extern void free_area_init_node(int nid, unsigned long * zones_size,
1762 unsigned long zone_start_pfn, unsigned long *zholes_size);
1763extern void free_initmem(void);
1764
1765/*
1766 * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK)
1767 * into the buddy system. The freed pages will be poisoned with pattern
1768 * "poison" if it's within range [0, UCHAR_MAX].
1769 * Return pages freed into the buddy system.
1770 */
1771extern unsigned long free_reserved_area(void *start, void *end,
1772 int poison, char *s);
1773
1774#ifdef CONFIG_HIGHMEM
1775/*
1776 * Free a highmem page into the buddy system, adjusting totalhigh_pages
1777 * and totalram_pages.
1778 */
1779extern void free_highmem_page(struct page *page);
1780#endif
1781
1782extern void adjust_managed_page_count(struct page *page, long count);
1783extern void mem_init_print_info(const char *str);
1784
1785extern void reserve_bootmem_region(unsigned long start, unsigned long end);
1786
1787/* Free the reserved page into the buddy system, so it gets managed. */
1788static inline void __free_reserved_page(struct page *page)
1789{
1790 ClearPageReserved(page);
1791 init_page_count(page);
1792 __free_page(page);
1793}
1794
1795static inline void free_reserved_page(struct page *page)
1796{
1797 __free_reserved_page(page);
1798 adjust_managed_page_count(page, 1);
1799}
1800
1801static inline void mark_page_reserved(struct page *page)
1802{
1803 SetPageReserved(page);
1804 adjust_managed_page_count(page, -1);
1805}
1806
1807/*
1808 * Default method to free all the __init memory into the buddy system.
1809 * The freed pages will be poisoned with pattern "poison" if it's within
1810 * range [0, UCHAR_MAX].
1811 * Return pages freed into the buddy system.
1812 */
1813static inline unsigned long free_initmem_default(int poison)
1814{
1815 extern char __init_begin[], __init_end[];
1816
1817 return free_reserved_area(&__init_begin, &__init_end,
1818 poison, "unused kernel");
1819}
1820
1821static inline unsigned long get_num_physpages(void)
1822{
1823 int nid;
1824 unsigned long phys_pages = 0;
1825
1826 for_each_online_node(nid)
1827 phys_pages += node_present_pages(nid);
1828
1829 return phys_pages;
1830}
1831
1832#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1833/*
1834 * With CONFIG_HAVE_MEMBLOCK_NODE_MAP set, an architecture may initialise its
1835 * zones, allocate the backing mem_map and account for memory holes in a more
1836 * architecture independent manner. This is a substitute for creating the
1837 * zone_sizes[] and zholes_size[] arrays and passing them to
1838 * free_area_init_node()
1839 *
1840 * An architecture is expected to register range of page frames backed by
1841 * physical memory with memblock_add[_node]() before calling
1842 * free_area_init_nodes() passing in the PFN each zone ends at. At a basic
1843 * usage, an architecture is expected to do something like
1844 *
1845 * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn,
1846 * max_highmem_pfn};
1847 * for_each_valid_physical_page_range()
1848 * memblock_add_node(base, size, nid)
1849 * free_area_init_nodes(max_zone_pfns);
1850 *
1851 * free_bootmem_with_active_regions() calls free_bootmem_node() for each
1852 * registered physical page range. Similarly
1853 * sparse_memory_present_with_active_regions() calls memory_present() for
1854 * each range when SPARSEMEM is enabled.
1855 *
1856 * See mm/page_alloc.c for more information on each function exposed by
1857 * CONFIG_HAVE_MEMBLOCK_NODE_MAP.
1858 */
1859extern void free_area_init_nodes(unsigned long *max_zone_pfn);
1860unsigned long node_map_pfn_alignment(void);
1861unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn,
1862 unsigned long end_pfn);
1863extern unsigned long absent_pages_in_range(unsigned long start_pfn,
1864 unsigned long end_pfn);
1865extern void get_pfn_range_for_nid(unsigned int nid,
1866 unsigned long *start_pfn, unsigned long *end_pfn);
1867extern unsigned long find_min_pfn_with_active_regions(void);
1868extern void free_bootmem_with_active_regions(int nid,
1869 unsigned long max_low_pfn);
1870extern void sparse_memory_present_with_active_regions(int nid);
1871
1872#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
1873
1874#if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \
1875 !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
1876static inline int __early_pfn_to_nid(unsigned long pfn,
1877 struct mminit_pfnnid_cache *state)
1878{
1879 return 0;
1880}
1881#else
1882/* please see mm/page_alloc.c */
1883extern int __meminit early_pfn_to_nid(unsigned long pfn);
1884/* there is a per-arch backend function. */
1885extern int __meminit __early_pfn_to_nid(unsigned long pfn,
1886 struct mminit_pfnnid_cache *state);
1887#endif
1888
1889extern void set_dma_reserve(unsigned long new_dma_reserve);
1890extern void memmap_init_zone(unsigned long, int, unsigned long,
1891 unsigned long, enum memmap_context);
1892extern void setup_per_zone_wmarks(void);
1893extern int __meminit init_per_zone_wmark_min(void);
1894extern void mem_init(void);
1895extern void __init mmap_init(void);
1896extern void show_mem(unsigned int flags);
1897extern long si_mem_available(void);
1898extern void si_meminfo(struct sysinfo * val);
1899extern void si_meminfo_node(struct sysinfo *val, int nid);
1900
1901extern __printf(3, 4)
1902void warn_alloc_failed(gfp_t gfp_mask, unsigned int order,
1903 const char *fmt, ...);
1904
1905extern void setup_per_cpu_pageset(void);
1906
1907extern void zone_pcp_update(struct zone *zone);
1908extern void zone_pcp_reset(struct zone *zone);
1909
1910/* page_alloc.c */
1911extern int min_free_kbytes;
1912extern int watermark_scale_factor;
1913
1914/* nommu.c */
1915extern atomic_long_t mmap_pages_allocated;
1916extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
1917
1918/* interval_tree.c */
1919void vma_interval_tree_insert(struct vm_area_struct *node,
1920 struct rb_root *root);
1921void vma_interval_tree_insert_after(struct vm_area_struct *node,
1922 struct vm_area_struct *prev,
1923 struct rb_root *root);
1924void vma_interval_tree_remove(struct vm_area_struct *node,
1925 struct rb_root *root);
1926struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root *root,
1927 unsigned long start, unsigned long last);
1928struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
1929 unsigned long start, unsigned long last);
1930
1931#define vma_interval_tree_foreach(vma, root, start, last) \
1932 for (vma = vma_interval_tree_iter_first(root, start, last); \
1933 vma; vma = vma_interval_tree_iter_next(vma, start, last))
1934
1935void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
1936 struct rb_root *root);
1937void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
1938 struct rb_root *root);
1939struct anon_vma_chain *anon_vma_interval_tree_iter_first(
1940 struct rb_root *root, unsigned long start, unsigned long last);
1941struct anon_vma_chain *anon_vma_interval_tree_iter_next(
1942 struct anon_vma_chain *node, unsigned long start, unsigned long last);
1943#ifdef CONFIG_DEBUG_VM_RB
1944void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
1945#endif
1946
1947#define anon_vma_interval_tree_foreach(avc, root, start, last) \
1948 for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
1949 avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
1950
1951/* mmap.c */
1952extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
1953extern int vma_adjust(struct vm_area_struct *vma, unsigned long start,
1954 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert);
1955extern struct vm_area_struct *vma_merge(struct mm_struct *,
1956 struct vm_area_struct *prev, unsigned long addr, unsigned long end,
1957 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
1958 struct mempolicy *, struct vm_userfaultfd_ctx);
1959extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
1960extern int split_vma(struct mm_struct *,
1961 struct vm_area_struct *, unsigned long addr, int new_below);
1962extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
1963extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
1964 struct rb_node **, struct rb_node *);
1965extern void unlink_file_vma(struct vm_area_struct *);
1966extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
1967 unsigned long addr, unsigned long len, pgoff_t pgoff,
1968 bool *need_rmap_locks);
1969extern void exit_mmap(struct mm_struct *);
1970
1971static inline int check_data_rlimit(unsigned long rlim,
1972 unsigned long new,
1973 unsigned long start,
1974 unsigned long end_data,
1975 unsigned long start_data)
1976{
1977 if (rlim < RLIM_INFINITY) {
1978 if (((new - start) + (end_data - start_data)) > rlim)
1979 return -ENOSPC;
1980 }
1981
1982 return 0;
1983}
1984
1985extern int mm_take_all_locks(struct mm_struct *mm);
1986extern void mm_drop_all_locks(struct mm_struct *mm);
1987
1988extern void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
1989extern struct file *get_mm_exe_file(struct mm_struct *mm);
1990
1991extern bool may_expand_vm(struct mm_struct *, vm_flags_t, unsigned long npages);
1992extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages);
1993
1994extern struct vm_area_struct *_install_special_mapping(struct mm_struct *mm,
1995 unsigned long addr, unsigned long len,
1996 unsigned long flags,
1997 const struct vm_special_mapping *spec);
1998/* This is an obsolete alternative to _install_special_mapping. */
1999extern int install_special_mapping(struct mm_struct *mm,
2000 unsigned long addr, unsigned long len,
2001 unsigned long flags, struct page **pages);
2002
2003extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
2004
2005extern unsigned long mmap_region(struct file *file, unsigned long addr,
2006 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff);
2007extern unsigned long do_mmap(struct file *file, unsigned long addr,
2008 unsigned long len, unsigned long prot, unsigned long flags,
2009 vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate);
2010extern int do_munmap(struct mm_struct *, unsigned long, size_t);
2011
2012static inline unsigned long
2013do_mmap_pgoff(struct file *file, unsigned long addr,
2014 unsigned long len, unsigned long prot, unsigned long flags,
2015 unsigned long pgoff, unsigned long *populate)
2016{
2017 return do_mmap(file, addr, len, prot, flags, 0, pgoff, populate);
2018}
2019
2020#ifdef CONFIG_MMU
2021extern int __mm_populate(unsigned long addr, unsigned long len,
2022 int ignore_errors);
2023static inline void mm_populate(unsigned long addr, unsigned long len)
2024{
2025 /* Ignore errors */
2026 (void) __mm_populate(addr, len, 1);
2027}
2028#else
2029static inline void mm_populate(unsigned long addr, unsigned long len) {}
2030#endif
2031
2032/* These take the mm semaphore themselves */
2033extern unsigned long vm_brk(unsigned long, unsigned long);
2034extern int vm_munmap(unsigned long, size_t);
2035extern unsigned long vm_mmap(struct file *, unsigned long,
2036 unsigned long, unsigned long,
2037 unsigned long, unsigned long);
2038
2039struct vm_unmapped_area_info {
2040#define VM_UNMAPPED_AREA_TOPDOWN 1
2041 unsigned long flags;
2042 unsigned long length;
2043 unsigned long low_limit;
2044 unsigned long high_limit;
2045 unsigned long align_mask;
2046 unsigned long align_offset;
2047};
2048
2049extern unsigned long unmapped_area(struct vm_unmapped_area_info *info);
2050extern unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);
2051
2052/*
2053 * Search for an unmapped address range.
2054 *
2055 * We are looking for a range that:
2056 * - does not intersect with any VMA;
2057 * - is contained within the [low_limit, high_limit) interval;
2058 * - is at least the desired size.
2059 * - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
2060 */
2061static inline unsigned long
2062vm_unmapped_area(struct vm_unmapped_area_info *info)
2063{
2064 if (info->flags & VM_UNMAPPED_AREA_TOPDOWN)
2065 return unmapped_area_topdown(info);
2066 else
2067 return unmapped_area(info);
2068}
2069
2070/* truncate.c */
2071extern void truncate_inode_pages(struct address_space *, loff_t);
2072extern void truncate_inode_pages_range(struct address_space *,
2073 loff_t lstart, loff_t lend);
2074extern void truncate_inode_pages_final(struct address_space *);
2075
2076/* generic vm_area_ops exported for stackable file systems */
2077extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
2078extern void filemap_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf);
2079extern int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf);
2080
2081/* mm/page-writeback.c */
2082int write_one_page(struct page *page, int wait);
2083void task_dirty_inc(struct task_struct *tsk);
2084
2085/* readahead.c */
2086#define VM_MAX_READAHEAD 128 /* kbytes */
2087#define VM_MIN_READAHEAD 16 /* kbytes (includes current page) */
2088
2089int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
2090 pgoff_t offset, unsigned long nr_to_read);
2091
2092void page_cache_sync_readahead(struct address_space *mapping,
2093 struct file_ra_state *ra,
2094 struct file *filp,
2095 pgoff_t offset,
2096 unsigned long size);
2097
2098void page_cache_async_readahead(struct address_space *mapping,
2099 struct file_ra_state *ra,
2100 struct file *filp,
2101 struct page *pg,
2102 pgoff_t offset,
2103 unsigned long size);
2104
2105/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
2106extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
2107
2108/* CONFIG_STACK_GROWSUP still needs to to grow downwards at some places */
2109extern int expand_downwards(struct vm_area_struct *vma,
2110 unsigned long address);
2111#if VM_GROWSUP
2112extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
2113#else
2114 #define expand_upwards(vma, address) (0)
2115#endif
2116
2117/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
2118extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
2119extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
2120 struct vm_area_struct **pprev);
2121
2122/* Look up the first VMA which intersects the interval start_addr..end_addr-1,
2123 NULL if none. Assume start_addr < end_addr. */
2124static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
2125{
2126 struct vm_area_struct * vma = find_vma(mm,start_addr);
2127
2128 if (vma && end_addr <= vma->vm_start)
2129 vma = NULL;
2130 return vma;
2131}
2132
2133static inline unsigned long vma_pages(struct vm_area_struct *vma)
2134{
2135 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
2136}
2137
2138/* Look up the first VMA which exactly match the interval vm_start ... vm_end */
2139static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
2140 unsigned long vm_start, unsigned long vm_end)
2141{
2142 struct vm_area_struct *vma = find_vma(mm, vm_start);
2143
2144 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
2145 vma = NULL;
2146
2147 return vma;
2148}
2149
2150#ifdef CONFIG_MMU
2151pgprot_t vm_get_page_prot(unsigned long vm_flags);
2152void vma_set_page_prot(struct vm_area_struct *vma);
2153#else
2154static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
2155{
2156 return __pgprot(0);
2157}
2158static inline void vma_set_page_prot(struct vm_area_struct *vma)
2159{
2160 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
2161}
2162#endif
2163
2164#ifdef CONFIG_NUMA_BALANCING
2165unsigned long change_prot_numa(struct vm_area_struct *vma,
2166 unsigned long start, unsigned long end);
2167#endif
2168
2169struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
2170int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
2171 unsigned long pfn, unsigned long size, pgprot_t);
2172int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
2173int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2174 unsigned long pfn);
2175int vm_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
2176 unsigned long pfn, pgprot_t pgprot);
2177int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
2178 pfn_t pfn);
2179int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);
2180
2181
2182struct page *follow_page_mask(struct vm_area_struct *vma,
2183 unsigned long address, unsigned int foll_flags,
2184 unsigned int *page_mask);
2185
2186static inline struct page *follow_page(struct vm_area_struct *vma,
2187 unsigned long address, unsigned int foll_flags)
2188{
2189 unsigned int unused_page_mask;
2190 return follow_page_mask(vma, address, foll_flags, &unused_page_mask);
2191}
2192
2193#define FOLL_WRITE 0x01 /* check pte is writable */
2194#define FOLL_TOUCH 0x02 /* mark page accessed */
2195#define FOLL_GET 0x04 /* do get_page on page */
2196#define FOLL_DUMP 0x08 /* give error on hole if it would be zero */
2197#define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */
2198#define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO
2199 * and return without waiting upon it */
2200#define FOLL_POPULATE 0x40 /* fault in page */
2201#define FOLL_SPLIT 0x80 /* don't return transhuge pages, split them */
2202#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
2203#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
2204#define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
2205#define FOLL_TRIED 0x800 /* a retry, previous pass started an IO */
2206#define FOLL_MLOCK 0x1000 /* lock present pages */
2207#define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */
2208
2209typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
2210 void *data);
2211extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
2212 unsigned long size, pte_fn_t fn, void *data);
2213
2214
2215#ifdef CONFIG_PAGE_POISONING
2216extern bool page_poisoning_enabled(void);
2217extern void kernel_poison_pages(struct page *page, int numpages, int enable);
2218extern bool page_is_poisoned(struct page *page);
2219#else
2220static inline bool page_poisoning_enabled(void) { return false; }
2221static inline void kernel_poison_pages(struct page *page, int numpages,
2222 int enable) { }
2223static inline bool page_is_poisoned(struct page *page) { return false; }
2224#endif
2225
2226#ifdef CONFIG_DEBUG_PAGEALLOC
2227extern bool _debug_pagealloc_enabled;
2228extern void __kernel_map_pages(struct page *page, int numpages, int enable);
2229
2230static inline bool debug_pagealloc_enabled(void)
2231{
2232 return _debug_pagealloc_enabled;
2233}
2234
2235static inline void
2236kernel_map_pages(struct page *page, int numpages, int enable)
2237{
2238 if (!debug_pagealloc_enabled())
2239 return;
2240
2241 __kernel_map_pages(page, numpages, enable);
2242}
2243#ifdef CONFIG_HIBERNATION
2244extern bool kernel_page_present(struct page *page);
2245#endif /* CONFIG_HIBERNATION */
2246#else /* CONFIG_DEBUG_PAGEALLOC */
2247static inline void
2248kernel_map_pages(struct page *page, int numpages, int enable) {}
2249#ifdef CONFIG_HIBERNATION
2250static inline bool kernel_page_present(struct page *page) { return true; }
2251#endif /* CONFIG_HIBERNATION */
2252static inline bool debug_pagealloc_enabled(void)
2253{
2254 return false;
2255}
2256#endif /* CONFIG_DEBUG_PAGEALLOC */
2257
2258#ifdef __HAVE_ARCH_GATE_AREA
2259extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);
2260extern int in_gate_area_no_mm(unsigned long addr);
2261extern int in_gate_area(struct mm_struct *mm, unsigned long addr);
2262#else
2263static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
2264{
2265 return NULL;
2266}
2267static inline int in_gate_area_no_mm(unsigned long addr) { return 0; }
2268static inline int in_gate_area(struct mm_struct *mm, unsigned long addr)
2269{
2270 return 0;
2271}
2272#endif /* __HAVE_ARCH_GATE_AREA */
2273
2274#ifdef CONFIG_SYSCTL
2275extern int sysctl_drop_caches;
2276int drop_caches_sysctl_handler(struct ctl_table *, int,
2277 void __user *, size_t *, loff_t *);
2278#endif
2279
2280void drop_slab(void);
2281void drop_slab_node(int nid);
2282
2283#ifndef CONFIG_MMU
2284#define randomize_va_space 0
2285#else
2286extern int randomize_va_space;
2287#endif
2288
2289const char * arch_vma_name(struct vm_area_struct *vma);
2290void print_vma_addr(char *prefix, unsigned long rip);
2291
2292void sparse_mem_maps_populate_node(struct page **map_map,
2293 unsigned long pnum_begin,
2294 unsigned long pnum_end,
2295 unsigned long map_count,
2296 int nodeid);
2297
2298struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
2299pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
2300pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
2301pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
2302pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
2303void *vmemmap_alloc_block(unsigned long size, int node);
2304struct vmem_altmap;
2305void *__vmemmap_alloc_block_buf(unsigned long size, int node,
2306 struct vmem_altmap *altmap);
2307static inline void *vmemmap_alloc_block_buf(unsigned long size, int node)
2308{
2309 return __vmemmap_alloc_block_buf(size, node, NULL);
2310}
2311
2312void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
2313int vmemmap_populate_basepages(unsigned long start, unsigned long end,
2314 int node);
2315int vmemmap_populate(unsigned long start, unsigned long end, int node);
2316void vmemmap_populate_print_last(void);
2317#ifdef CONFIG_MEMORY_HOTPLUG
2318void vmemmap_free(unsigned long start, unsigned long end);
2319#endif
2320void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
2321 unsigned long size);
2322
2323enum mf_flags {
2324 MF_COUNT_INCREASED = 1 << 0,
2325 MF_ACTION_REQUIRED = 1 << 1,
2326 MF_MUST_KILL = 1 << 2,
2327 MF_SOFT_OFFLINE = 1 << 3,
2328};
2329extern int memory_failure(unsigned long pfn, int trapno, int flags);
2330extern void memory_failure_queue(unsigned long pfn, int trapno, int flags);
2331extern int unpoison_memory(unsigned long pfn);
2332extern int get_hwpoison_page(struct page *page);
2333#define put_hwpoison_page(page) put_page(page)
2334extern int sysctl_memory_failure_early_kill;
2335extern int sysctl_memory_failure_recovery;
2336extern void shake_page(struct page *p, int access);
2337extern atomic_long_t num_poisoned_pages;
2338extern int soft_offline_page(struct page *page, int flags);
2339
2340
2341/*
2342 * Error handlers for various types of pages.
2343 */
2344enum mf_result {
2345 MF_IGNORED, /* Error: cannot be handled */
2346 MF_FAILED, /* Error: handling failed */
2347 MF_DELAYED, /* Will be handled later */
2348 MF_RECOVERED, /* Successfully recovered */
2349};
2350
2351enum mf_action_page_type {
2352 MF_MSG_KERNEL,
2353 MF_MSG_KERNEL_HIGH_ORDER,
2354 MF_MSG_SLAB,
2355 MF_MSG_DIFFERENT_COMPOUND,
2356 MF_MSG_POISONED_HUGE,
2357 MF_MSG_HUGE,
2358 MF_MSG_FREE_HUGE,
2359 MF_MSG_UNMAP_FAILED,
2360 MF_MSG_DIRTY_SWAPCACHE,
2361 MF_MSG_CLEAN_SWAPCACHE,
2362 MF_MSG_DIRTY_MLOCKED_LRU,
2363 MF_MSG_CLEAN_MLOCKED_LRU,
2364 MF_MSG_DIRTY_UNEVICTABLE_LRU,
2365 MF_MSG_CLEAN_UNEVICTABLE_LRU,
2366 MF_MSG_DIRTY_LRU,
2367 MF_MSG_CLEAN_LRU,
2368 MF_MSG_TRUNCATED_LRU,
2369 MF_MSG_BUDDY,
2370 MF_MSG_BUDDY_2ND,
2371 MF_MSG_UNKNOWN,
2372};
2373
2374#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
2375extern void clear_huge_page(struct page *page,
2376 unsigned long addr,
2377 unsigned int pages_per_huge_page);
2378extern void copy_user_huge_page(struct page *dst, struct page *src,
2379 unsigned long addr, struct vm_area_struct *vma,
2380 unsigned int pages_per_huge_page);
2381#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
2382
2383extern struct page_ext_operations debug_guardpage_ops;
2384extern struct page_ext_operations page_poisoning_ops;
2385
2386#ifdef CONFIG_DEBUG_PAGEALLOC
2387extern unsigned int _debug_guardpage_minorder;
2388extern bool _debug_guardpage_enabled;
2389
2390static inline unsigned int debug_guardpage_minorder(void)
2391{
2392 return _debug_guardpage_minorder;
2393}
2394
2395static inline bool debug_guardpage_enabled(void)
2396{
2397 return _debug_guardpage_enabled;
2398}
2399
2400static inline bool page_is_guard(struct page *page)
2401{
2402 struct page_ext *page_ext;
2403
2404 if (!debug_guardpage_enabled())
2405 return false;
2406
2407 page_ext = lookup_page_ext(page);
2408 return test_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
2409}
2410#else
2411static inline unsigned int debug_guardpage_minorder(void) { return 0; }
2412static inline bool debug_guardpage_enabled(void) { return false; }
2413static inline bool page_is_guard(struct page *page) { return false; }
2414#endif /* CONFIG_DEBUG_PAGEALLOC */
2415
2416#if MAX_NUMNODES > 1
2417void __init setup_nr_node_ids(void);
2418#else
2419static inline void setup_nr_node_ids(void) {}
2420#endif
2421
2422#endif /* __KERNEL__ */
2423#endif /* _LINUX_MM_H */